diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index e8758962b4..2f09d661aa 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -172,6 +172,16 @@ jobs: cd build ninja test-runtime ./test-runtime + - name: Test Tools + env: + RUST_BACKTRACE: full + run: | + cmake --build build --target shards + SH=`pwd`/build/shards + + echo "Running language formatter tests" + cd shards/lang/src/tests + $SH test # Minimize disk usage to prevent the next steps getting stuck due to no disk space - name: Minimize disk usage env: diff --git a/cmake/Modules.cmake b/cmake/Modules.cmake index c538642ca0..7cab3c31ae 100644 --- a/cmake/Modules.cmake +++ b/cmake/Modules.cmake @@ -1,4 +1,5 @@ option(SHARDS_WITH_EVERYTHING "Enables all modules, disabling this will only build common modules and exclude experimental ones" ON) +option(SHARDS_NO_RUST_UNION "Disables rust union build" OFF) function(is_module_enabled OUTPUT_VARIABLE MODULE_ID) if(${SHARDS_WITH_EVERYTHING}) @@ -132,68 +133,77 @@ function(shards_generate_rust_union TARGET_NAME) list(APPEND RUST_TARGETS ${UNION_EXTRA_RUST_TARGETS}) - set(GEN_RUST_PATH ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) - set(GEN_RUST_SRC_PATH ${GEN_RUST_PATH}/src) - set(CARGO_TOML ${GEN_RUST_PATH}/Cargo.toml) - set(LIB_RS_TMP ${GEN_RUST_SRC_PATH}/lib.rs.tmp) - set(LIB_RS ${GEN_RUST_SRC_PATH}/lib.rs) - - file(MAKE_DIRECTORY ${GEN_RUST_SRC_PATH}) - file(WRITE ${LIB_RS_TMP} "// This file is generated by CMake\n\n") - - # Add all the rust targets to the Cargo.toml - foreach(RUST_TARGET ${RUST_TARGETS}) - get_property(RUST_PROJECT_PATH TARGET ${RUST_TARGET} PROPERTY RUST_PROJECT_PATH) - get_property(RUST_NAME TARGET ${RUST_TARGET} PROPERTY RUST_NAME) - - get_property(RUST_FEATURES TARGET ${RUST_TARGET} PROPERTY RUST_FEATURES) - unset(RUST_FEATURES_STRING) - unset(RUST_FEATURES_STRING1) - if(RUST_FEATURES) - unset(RUST_FEATURES_QUOTED) - foreach(FEATURE ${RUST_FEATURES}) - list(APPEND RUST_FEATURES_QUOTED \"${FEATURE}\") - endforeach() - list(JOIN RUST_FEATURES_QUOTED "," RUST_FEATURES_STRING) - set(RUST_FEATURES_STRING1 ", features = [${RUST_FEATURES_STRING}]") - endif() + # Disable union + if(SHARDS_NO_RUST_UNION) + add_library(${TARGET_NAME} INTERFACE) + foreach(RUST_TARGET ${RUST_TARGETS}) + message(STATUS "RUST TARGET: ${RUST_TARGET}") + target_link_libraries(${TARGET_NAME} INTERFACE ${RUST_TARGET}) + endforeach() + else() + set(GEN_RUST_PATH ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) + set(GEN_RUST_SRC_PATH ${GEN_RUST_PATH}/src) + set(CARGO_TOML ${GEN_RUST_PATH}/Cargo.toml) + set(LIB_RS_TMP ${GEN_RUST_SRC_PATH}/lib.rs.tmp) + set(LIB_RS ${GEN_RUST_SRC_PATH}/lib.rs) + + file(MAKE_DIRECTORY ${GEN_RUST_SRC_PATH}) + file(WRITE ${LIB_RS_TMP} "// This file is generated by CMake\n\n") + + # Add all the rust targets to the Cargo.toml + foreach(RUST_TARGET ${RUST_TARGETS}) + get_property(RUST_PROJECT_PATH TARGET ${RUST_TARGET} PROPERTY RUST_PROJECT_PATH) + get_property(RUST_NAME TARGET ${RUST_TARGET} PROPERTY RUST_NAME) + + get_property(RUST_FEATURES TARGET ${RUST_TARGET} PROPERTY RUST_FEATURES) + unset(RUST_FEATURES_STRING) + unset(RUST_FEATURES_STRING1) + if(RUST_FEATURES) + unset(RUST_FEATURES_QUOTED) + foreach(FEATURE ${RUST_FEATURES}) + list(APPEND RUST_FEATURES_QUOTED \"${FEATURE}\") + endforeach() + list(JOIN RUST_FEATURES_QUOTED "," RUST_FEATURES_STRING) + set(RUST_FEATURES_STRING1 ", features = [${RUST_FEATURES_STRING}]") + endif() - message(VERBOSE "${TARGET_NAME}: Adding rust target ${RUST_TARGET} (path: ${RUST_PROJECT_PATH}, name: ${RUST_NAME}, features: ${RUST_FEATURES_STRING})") + message(VERBOSE "${TARGET_NAME}: Adding rust target ${RUST_TARGET} (path: ${RUST_PROJECT_PATH}, name: ${RUST_NAME}, features: ${RUST_FEATURES_STRING})") - file(RELATIVE_PATH RELATIVE_RUST_PROJECT_PATH ${GEN_RUST_PATH} ${RUST_PROJECT_PATH}) - string(APPEND CARGO_CRATE_DEPS "${RUST_NAME} = { path = \"${RELATIVE_RUST_PROJECT_PATH}\"${RUST_FEATURES_STRING1} }\n") + file(RELATIVE_PATH RELATIVE_RUST_PROJECT_PATH ${GEN_RUST_PATH} ${RUST_PROJECT_PATH}) + string(APPEND CARGO_CRATE_DEPS "${RUST_NAME} = { path = \"${RELATIVE_RUST_PROJECT_PATH}\"${RUST_FEATURES_STRING1} }\n") - string(REPLACE "-" "_" RUST_NAME_ID ${RUST_NAME}) - file(APPEND ${LIB_RS_TMP} "pub use ${RUST_NAME_ID};\n") + string(REPLACE "-" "_" RUST_NAME_ID ${RUST_NAME}) + file(APPEND ${LIB_RS_TMP} "pub use ${RUST_NAME_ID};\n") - get_property(TARGET_INTERFACE_LINK_LIBRARIES TARGET ${RUST_TARGET} PROPERTY INTERFACE_LINK_LIBRARIES) - list(APPEND COMBINED_INTERFACE_LINK_LIBRARIES ${TARGET_INTERFACE_LINK_LIBRARIES}) + get_property(TARGET_INTERFACE_LINK_LIBRARIES TARGET ${RUST_TARGET} PROPERTY INTERFACE_LINK_LIBRARIES) + list(APPEND COMBINED_INTERFACE_LINK_LIBRARIES ${TARGET_INTERFACE_LINK_LIBRARIES}) - get_property(RUST_DEPENDS TARGET ${RUST_TARGET} PROPERTY RUST_DEPENDS) - list(APPEND COMBINED_RUST_DEPENDS ${RUST_DEPENDS}) + get_property(RUST_DEPENDS TARGET ${RUST_TARGET} PROPERTY RUST_DEPENDS) + list(APPEND COMBINED_RUST_DEPENDS ${RUST_DEPENDS}) - get_property(RUST_ENVIRONMENT TARGET ${RUST_TARGET} PROPERTY RUST_ENVIRONMENT) - list(APPEND COMBINED_RUST_ENVIRONMENT ${RUST_ENVIRONMENT}) - endforeach() + get_property(RUST_ENVIRONMENT TARGET ${RUST_TARGET} PROPERTY RUST_ENVIRONMENT) + list(APPEND COMBINED_RUST_ENVIRONMENT ${RUST_ENVIRONMENT}) + endforeach() - # The generated create name - set(CARGO_CRATE_NAME ${TARGET_NAME}) + # The generated create name + set(CARGO_CRATE_NAME ${TARGET_NAME}) - file(COPY_FILE ${LIB_RS_TMP} ${LIB_RS} ONLY_IF_DIFFERENT) - configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Cargo.toml.in" ${CARGO_TOML}) + file(COPY_FILE ${LIB_RS_TMP} ${LIB_RS} ONLY_IF_DIFFERENT) + configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Cargo.toml.in" ${CARGO_TOML}) - # Add the rust library - add_rust_library(NAME ${TARGET_NAME} - PROJECT_PATH ${GEN_RUST_PATH} - DEPENDS ${COMBINED_RUST_DEPENDS} - ENVIRONMENT ${COMBINED_RUST_ENVIRONMENT}) - add_library(${TARGET_NAME} ALIAS ${TARGET_NAME}-rust) + # Add the rust library + add_rust_library(NAME ${TARGET_NAME} + PROJECT_PATH ${GEN_RUST_PATH} + DEPENDS ${COMBINED_RUST_DEPENDS} + ENVIRONMENT ${COMBINED_RUST_ENVIRONMENT}) + add_library(${TARGET_NAME} ALIAS ${TARGET_NAME}-rust) - list(REMOVE_DUPLICATES COMBINED_INTERFACE_LINK_LIBRARIES) + list(REMOVE_DUPLICATES COMBINED_INTERFACE_LINK_LIBRARIES) - if(COMBINED_INTERFACE_LINK_LIBRARIES) - set_property(TARGET ${TARGET_NAME}-rust APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${COMBINED_INTERFACE_LINK_LIBRARIES}) - message(VERBOSE "${TARGET_NAME}: Linking against ${COMBINED_INTERFACE_LINK_LIBRARIES}") + if(COMBINED_INTERFACE_LINK_LIBRARIES) + set_property(TARGET ${TARGET_NAME}-rust APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${COMBINED_INTERFACE_LINK_LIBRARIES}) + message(VERBOSE "${TARGET_NAME}: Linking against ${COMBINED_INTERFACE_LINK_LIBRARIES}") + endif() endif() endfunction() diff --git a/cmake/Root.cmake b/cmake/Root.cmake index 087c2ef288..d5f7c0604d 100644 --- a/cmake/Root.cmake +++ b/cmake/Root.cmake @@ -47,7 +47,6 @@ add_subdirectory(${SHARDS_DIR}/shards/lang src/lang) # Rust projects add_subdirectory(${SHARDS_DIR}/shards/rust src/rust) -add_subdirectory(${SHARDS_DIR}/shards/rust_macro src/rust_macro) # Modules set(SHARDS_MODULE_ROOT ${SHARDS_DIR}/shards/modules) diff --git a/cmake/Rust.cmake b/cmake/Rust.cmake index 15fa54a122..855991b078 100644 --- a/cmake/Rust.cmake +++ b/cmake/Rust.cmake @@ -282,7 +282,7 @@ function(add_rust_library) # Add default required libraries for windows if(WIN32) - target_link_libraries(${RUST_TARGET_NAME} INTERFACE NtDll) + target_link_libraries(${RUST_TARGET_NAME} INTERFACE NtDll Userenv) endif() endfunction() diff --git a/shards/egui/Cargo.toml b/shards/egui/Cargo.toml index e1e73c5809..7d4b6abcd0 100644 --- a/shards/egui/Cargo.toml +++ b/shards/egui/Cargo.toml @@ -15,7 +15,7 @@ egui = { version = "0.22.0", features = ["persistence"] } egui_commonmark = { version = "0.7.3" } egui_dock = { version = "0.6.1" } egui_extras = { version = "0.22.0" } -egui_memory_editor = { version = "0.2.4" } +egui_memory_editor = { version = "=0.2.5" } syntect = { version = "5.0.0", default-features = false, features = [ "default-fancy", ] } diff --git a/shards/lang/Cargo.toml b/shards/lang/Cargo.toml index f2ca8bf983..14cb2a4bf8 100644 --- a/shards/lang/Cargo.toml +++ b/shards/lang/Cargo.toml @@ -15,7 +15,7 @@ serde = { version = "1.0", features = ["derive"] } shards = { path = "../rust" } hex = { version = "0.4.3" } nanoid = "0.4.0" -clap = "4.3.9" +clap = { version = "4.4.7", features = ["derive"] } profiling = { version = "1", default-features = false } compile-time-crc32 = "0.1.2" lazy_static = "1.4.0" diff --git a/shards/lang/src/ast_visitor.rs b/shards/lang/src/ast_visitor.rs new file mode 100644 index 0000000000..cf6283deb1 --- /dev/null +++ b/shards/lang/src/ast_visitor.rs @@ -0,0 +1,507 @@ +use std::cell::RefCell; +type Rule = crate::ast::Rule; +use crate::error::*; +use pest::iterators::Pair; + +pub trait Visitor { + fn v_pipeline(&mut self, pair: Pair, inner: T); + fn v_stmt(&mut self, pair: Pair, inner: T); + fn v_value(&mut self, pair: Pair); + fn v_assign( + &mut self, + pair: Pair, + op: Pair, + to: Pair, + inner: T, + ); + fn v_func(&mut self, pair: Pair, name: Pair, inner: T); + fn v_param(&mut self, pair: Pair, kw: Option>, inner: T); + fn v_seq(&mut self, pair: Pair, inner: T); + fn v_table(&mut self, pair: Pair, inner: T); + fn v_table_val( + &mut self, + pair: Pair, + key: Pair, + inner_key: TK, + inner_val: TV, + ); + fn v_eval_expr( + &mut self, + pair: Pair, + inner_pair: Pair, + inner: T, + ); + fn v_expr(&mut self, pair: Pair, inner_pair: Pair, inner: T); + fn v_shards(&mut self, pair: Pair, inner: T); + fn v_take_table(&mut self, pair: Pair); + fn v_take_seq(&mut self, pair: Pair); +} + +fn process_take_seq(pair: Pair, v: &mut V) -> Result<(), Error> { + v.v_take_seq(pair); + Ok(()) +} + +fn process_param(pair: Pair, v: &mut V) -> Result<(), Error> { + let span = pair.as_span(); + if pair.as_rule() != Rule::Param { + return Err(fmt_errp("Expected a Param rule", &pair)); + } + + let mut inner = pair.clone().into_inner(); + let first = inner + .next() + .ok_or(fmt_err("Expected a ParamName or Value in Param", &span))?; + + let mut result: Option> = None; + if first.clone().as_rule() == Rule::ParamName { + v.v_param(pair, Some(first), |v| { + result = Some((|| { + process_value( + inner + .next() + .ok_or(fmt_err("Expected a Value in Param", &span))? + .into_inner() + .next() + .ok_or(fmt_err("Expected a Value in Param", &span))?, + v, + )?; + Ok(()) + })()); + }); + } else { + v.v_param(pair, None, |v| { + result = Some((|| { + process_value( + first + .into_inner() + .next() + .ok_or(fmt_err("Expected a Value in Param", &span))?, + v, + )?; + Ok(()) + })()); + }); + } + result.expect("Visitor didn't call v_param inner") +} + +fn process_params(pair: Pair, v: &mut V) -> Result, Error> { + pair.into_inner().map(|x| process_param(x, v)).collect() +} + +fn process_function(pair: Pair, v: &mut V) -> Result<(), Error> { + let span = pair.as_span(); + + let mut inner = pair.clone().into_inner(); + let exp: Pair<'_, Rule> = inner + .next() + .ok_or(fmt_err("Expected a Name or Const in Shard", &span))?; + + let mut result: Result<(), Error> = Ok(()); + v.v_func(pair.clone(), exp.clone(), |v| { + result = (|| { + match exp.as_rule() { + Rule::UppIden => { + // Definitely a Shard! + let next = inner.next(); + match next { + Some(pair) => { + if pair.as_rule() == Rule::Params { + Some(process_params(pair, v)?) + } else { + return Err(fmt_errp("Expected Params in Shard", &pair)); + } + } + None => None, + }; + Ok(()) + } + Rule::VarName => { + let next = inner.next(); + match next { + Some(pair) => { + if pair.as_rule() == Rule::Params { + Some(process_params(pair, v)?) + } else { + return Err(fmt_errp("Expected Params in Shard", &pair)); + } + } + None => None, + }; + + Ok(()) + } + _ => Err(fmt_err( + format!("Unexpected rule {:?} in Function.", exp.as_rule()), + &span, + )), + } + })(); + }); + result +} + +fn process_sequence(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Option> = None; + + let span = pair.as_span(); + v.v_seq(pair.clone(), |v| { + result = Some((|| { + pair + .into_inner() + .map(|value| { + process_value( + value + .into_inner() + .next() + .ok_or(fmt_err("Expected a Value in the sequence", &span))?, + v, + ) + }) + .collect::, _>>()?; + Ok(()) + })()); + }); + result.expect("Visitor didn't call v_seq inner") +} + +fn process_eval_expr(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Option> = None; + + let span = pair.as_span(); + let inner = pair + .clone() + .into_inner() + .next() + .ok_or(fmt_err("Expected a Sequence in Value", &span))?; + + v.v_eval_expr(pair, inner.clone(), |v| { + result = Some((|| { + process_sequence_no_visit(inner, v)?; + Ok(()) + })()); + }); + result.expect("Visitor didn't call v_eval_expr inner") +} + +fn process_expr(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Option> = None; + + let span = pair.as_span(); + let inner = pair + .clone() + .into_inner() + .next() + .ok_or(fmt_err("Expected a Sequence in Value", &span))?; + + v.v_expr(pair, inner.clone(), |v| { + result = Some((|| { + process_sequence_no_visit(inner, v)?; + Ok(()) + })()); + }); + result.expect("Visitor didn't call v_expr inner") +} + +fn process_shards(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Option> = None; + + let span = pair.as_span(); + let contents = pair + .clone() + .into_inner() + .next() + .ok_or(fmt_err("Expected an expression, but found none.", &span))?; + + v.v_shards(pair.clone(), |v| { + result = Some((|| { + process_sequence_no_visit(contents, v)?; + Ok(()) + })()); + }); + result.expect("Visitor didn't call v_shards inner") +} + +fn process_sequence_no_visit( + pair: Pair, + v: &mut V, +) -> Result<(), Error> { + for stmt in pair.into_inner() { + process_statement(stmt, v)?; + } + Ok(()) +} + +fn process_value(pair: Pair, v: &mut V) -> Result<(), Error> { + let span = pair.as_span(); + + let matched: Result = { + let pair = pair.clone(); + match pair.as_rule() { + Rule::ConstValue => { + // unwrap the inner rule + let pair = pair.into_inner().next().unwrap(); // parsed qed + process_value(pair, v)?; + Ok(true) + } + Rule::Seq => { + process_sequence(pair, v)?; + Ok(true) + } + Rule::Table => { + process_table(pair, v)?; + Ok(true) + } + Rule::Shards => { + process_shards(pair, v)?; + Ok(true) + } + Rule::Shard => { + let _f = process_function(pair, v)?; + Ok(true) + } + Rule::EvalExpr => { + process_eval_expr(pair, v)?; + Ok(true) + } + Rule::Expr => { + process_expr(pair, v)?; + Ok(true) + } + Rule::TakeTable => { + process_take_table(pair, v)?; + Ok(true) + } + Rule::TakeSeq => { + process_take_seq(pair, v)?; + Ok(true) + } + Rule::Func => { + process_function(pair, v)?; + Ok(true) + } + _ => Ok(false), + } + }; + if matched? { + return Ok(()); + } + + let span = pair.as_span(); + match pair.as_rule() { + Rule::None => Ok(()), + Rule::Boolean => Ok(()), + Rule::VarName => Ok(()), + Rule::Enum => Ok(()), + Rule::Number => Ok(()), + Rule::String => Ok(()), + Rule::Iden => Ok(()), + _ => Err(fmt_err( + format!("Unexpected rule ({:?}) in Value", pair.as_rule()), + &span, + )), + }?; + + // This is an atomic value (single token) + v.v_value(pair.clone()); + + Ok(()) +} + +fn process_table(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Option> = None; + let span = pair.as_span(); + + v.v_table(pair.clone(), |v| { + result = Some((|| { + pair + .into_inner() + .map(|pair| { + assert_eq!(pair.as_rule(), Rule::TableEntry); + + let mut inner = pair.into_inner(); + + let key: Pair<'_, Rule> = inner.next().unwrap(); // should not fail + assert_eq!(key.as_rule(), Rule::TableKey); + let key = key + .into_inner() + .next() + .ok_or(fmt_err("Expected a Table key", &span))?; + + let value = inner + .next() + .ok_or(fmt_err("Expected a value in TableEntry", &span))?; + + let mut key_result: Option> = None; + let mut val_result: Option> = None; + v.v_table_val( + value.clone(), + key.clone(), + |v| { + key_result = Some((|| { + match key.as_rule() { + Rule::Iden | Rule::None => process_value(key, v)?, + Rule::Value => process_value( + key.into_inner().next().unwrap(), // parsed qed + v, + )?, + _ => unreachable!(), + }; + Ok(()) + })()); + }, + |v| { + val_result = Some((|| { + process_value( + value + .into_inner() + .next() + .ok_or(fmt_err("Expected a value in TableEntry", &span))?, + v, + )?; + Ok(()) + })()); + }, + ); + key_result.expect("Visitor didn't call v_table_val key inner")?; + val_result.expect("Visitor didn't call v_table_val value inner")?; + + Ok(()) + }) + .collect::, Error>>()?; + Ok(()) + })()); + }); + result.expect("Visitor didn't call v_table inner") +} + +fn process_take_table(pair: Pair, v: &mut V) -> Result<(), Error> { + v.v_take_table(pair); + Ok(()) +} + +fn process_pipeline(pair: Pair, v: &mut V) -> Result<(), Error> { + if pair.as_rule() != Rule::Pipeline { + return Err(fmt_errp( + "Expected a Pipeline rule, but found a different rule.", + &pair, + )); + } + + let mut result: Result<(), Error> = Ok(()); + v.v_pipeline(pair.clone(), |v| { + result = (|| { + let span = pair.as_span(); + for pair in pair.into_inner() { + let rule = pair.as_rule(); + match rule { + Rule::EvalExpr => { + process_eval_expr(pair, v)?; + } + Rule::Expr => { + process_expr(pair, v)?; + } + Rule::Shard => process_function(pair, v)?, + Rule::Func => process_function(pair, v)?, + Rule::TakeTable => process_take_table(pair, v)?, + Rule::TakeSeq => { + let _pair = process_take_seq(pair, v)?; + } + Rule::ConstValue => { + let _v = process_value(pair, v)?; + } + Rule::Enum => { + let _v = process_value(pair, v)?; + } + Rule::Shards => { + let _inner = process_shards(pair, v)?; + } + _ => { + return Err(fmt_err( + format!("Unexpected rule ({:?}) in Pipeline.", rule), + &span, + )) + } + } + } + Ok(()) + })(); + }); + result +} + +fn process_assignment(pair: Pair, v: &mut V) -> Result<(), Error> { + if pair.as_rule() != Rule::Assignment { + return Err(fmt_errp( + "Expected an Assignment rule, but found a different rule.", + &pair, + )); + } + + let span = pair.as_span(); + let mut inner = pair.clone().into_inner(); + + let pipeline = if let Some(next) = inner.peek() { + if next.as_rule() == Rule::Pipeline { + inner.next() + } else { + None + } + } else { + None + }; + + let assignment_op = inner.next().ok_or(fmt_err( + "Expected an AssignmentOp in Assignment, but found none.", + &span, + ))?; + + let iden = inner.next().ok_or(fmt_err( + "Expected an Identifier in Assignment, but found none.", + &span, + ))?; + + let mut result: Result<(), Error> = Ok(()); + v.v_assign(pair.clone(), assignment_op, iden, |v: &mut V| { + result = (|| { + if let Some(pipeline) = pipeline { + process_pipeline(pipeline, v)? + // } else { + // v.v_pipeline(pipeline, |v| {}); + // } + } + Ok(()) + })(); + }); + result +} + +fn process_statement(pair: Pair, v: &mut V) -> Result<(), Error> { + let mut result: Result<(), Error> = Ok(()); + v.v_stmt(pair.clone(), |v| { + result = (|| { + let rule = pair.as_rule(); + match rule { + Rule::Assignment => process_assignment(pair, v)?, + Rule::Pipeline => process_pipeline(pair, v)?, + _ => return Err(fmt_errp("Unexpected rule in Statement.", &pair)), + } + Ok(()) + })(); + }); + result +} + +pub fn process(code: &str, v: &mut V) -> Result<(), Error> { + let successful_parse = crate::read::parse(code).map_err(|x| x.message)?; + let root = successful_parse.into_iter().next().unwrap(); + if root.as_rule() != Rule::Program { + return Err("Expected a Program rule, but found a different rule.".into()); + } + + let inner = root.into_inner().next().unwrap(); + process_sequence_no_visit(inner, v)?; + + Ok(()) +} diff --git a/shards/lang/src/cli.rs b/shards/lang/src/cli.rs index 851c380b8c..9006d47ce7 100644 --- a/shards/lang/src/cli.rs +++ b/shards/lang/src/cli.rs @@ -1,13 +1,15 @@ use crate::ast::Sequence; -use crate::eval; +use crate::error::Error; use crate::read::{get_dependencies, read_with_env, ReadEnv}; +use crate::{eval, formatter}; use crate::{eval::eval, eval::new_cancellation_token, read::read}; -use clap::{arg, Arg, ArgMatches, Command}; +use clap::{arg, Arg, ArgMatches, Command, Parser}; use shards::core::Core; use shards::types::Mesh; -use shards::{fourCharacterCode, shlog, SHCore, SHARDS_CURRENT_ABI}; +use shards::{fourCharacterCode, shlog, shlog_error, SHCore, SHARDS_CURRENT_ABI}; use std::collections::HashMap; use std::ffi::CStr; +use std::fs; use std::io::Write; use std::os::raw::c_char; use std::path::Path; @@ -18,6 +20,82 @@ extern "C" { fn shardsInterface(version: u32) -> *mut SHCore; } +#[derive(Debug, clap::Subcommand)] +enum Commands { + /// Formats a shards file + Format { + /// The file to format + #[arg(value_hint = clap::ValueHint::FilePath)] + file: String, + /// Run the formatter on the file directly + /// by default the output will go to stdout + #[arg(long, short = 'i', action)] + inline: bool, + /// Optinally an output file name + #[arg(long, short = 'o')] + output: Option, + }, + /// Run formatter tests + Test {}, + /// Reads and executes a Shards file + New { + /// The script to execute + #[arg(value_hint = clap::ValueHint::FilePath)] + file: String, + /// Decompress help strings before running the script + #[arg(long, short = 'd', default_value = "false", action)] + decompress_strings: bool, + #[arg(num_args = 0..)] + args: Vec, + }, + /// Reads and builds a binary AST Shards file + Build { + /// The script to evaluate + #[arg(value_hint = clap::ValueHint::FilePath)] + file: String, + /// The output file to write to + #[arg(long, short = 'o', default_value = "out.sho")] + output: String, + /// Output as JSON ast + #[arg(long, short = 'j', action)] + json: bool, + /// The depfile to write, in makefile readable format + #[arg(long, short = 'd')] + depfile: Option, + }, + AST { + /// The script to evaluate + #[arg(value_hint = clap::ValueHint::FilePath)] + file: String, + /// The output file to write to + #[arg(long, short = 'o', default_value = "out.sho")] + output: String, + }, + /// Loads and executes a binary Shards file + Load { + /// The binary Shards file to execute + #[arg(value_hint = clap::ValueHint::FilePath)] + file: String, + /// Decompress help strings before running the script + #[arg(long, short = 'd', default_value = "false", action)] + decompress_strings: bool, + #[arg(num_args = 0..)] + args: Vec, + }, + + #[command(external_subcommand)] + External(Vec), +} + +#[derive(Debug, clap::Parser)] +#[command(name = "Shards", version = "0.1")] +#[command(about = "Shards command line tools and executor.")] +#[command(author = "Fragcolor Team")] +struct Cli { + #[command(subcommand)] + command: Commands, +} + #[no_mangle] pub extern "C" fn shards_process_args( argc: i32, @@ -48,103 +126,97 @@ pub extern "C" fn shards_process_args( .collect() }; - let matches = Command::new("Shards") - .version("0.1") - .about("Shards command line tools and executor.") - .author("Fragcolor Team") - .allow_external_subcommands(true) - .subcommand( - Command::new("new") - .about("Reads and executes a Shards file.") - .arg(arg!( "The script to execute")) - .arg( - Arg::new("decompress-strings") - .long("decompress-strings") - .help("Decompress help strings before running the script") - .default_value("false"), - ) - .arg(Arg::new("args").num_args(0..)), - ) - .subcommand( - Command::new("build") - .about("Reads and builds a binary AST Shards file.") - .arg(arg!( "The script to evaluate")) - .arg( - Arg::new("output") - .long("output") - .short('o') - .help("The output file to write to") - .default_value("out.sho"), - ) - .arg( - Arg::new("depfile") - .long("depfile") - .short('d') - .help("The depfile to write, in makefile readable format") - .default_value(""), - ), - ) - .subcommand( - Command::new("ast") - .about("Reads and outputs a JSON AST Shards file.") - .arg(arg!( "The script to evaluate")) - .arg( - Arg::new("output") - .long("output") - .short('o') - .help("The output file to write to") - .default_value("out.json"), - ) - .arg( - Arg::new("depfile") - .long("depfile") - .short('d') - .help("The depfile to write, in makefile readable format") - .default_value(None), - ), - ) - .subcommand( - Command::new("load") - .about("Loads and executes a binary Shards file.") - .arg(arg!( "The binary Shards file to execute")) - .arg( - Arg::new("decompress-strings") - .long("decompress-strings") - .help("Decompress help strings before running the script") - .default_value("false"), - ) - .arg(Arg::new("args").num_args(0..)), - ) - // Add your arguments here - .get_matches_from(args); - - let res = match matches.subcommand() { - Some(("new", matches)) => execute(matches, cancellation_token), - Some(("build", matches)) => build(matches, false), - Some(("ast", matches)) => build(matches, true), - Some(("load", matches)) => load(matches, cancellation_token), - Some((_external, _matches)) => return 99, - _ => Ok(()), + let cli = Cli::parse_from(args); + + // Init Core interface when not running external commands + match &cli.command { + Commands::External(_) => {} + _ => unsafe { + shards::core::Core = shardsInterface(SHARDS_CURRENT_ABI as u32); + }, + }; + + let res: Result<_, Error> = match &cli.command { + Commands::Build { + file, + output, + depfile, + json, + } => build(file, &output, depfile.as_deref(), *json), + Commands::AST { file, output } => build(file, &output, None, true), + Commands::Load { + file, + decompress_strings, + args, + } => load(file, args, *decompress_strings, cancellation_token), + Commands::New { + file, + decompress_strings, + args, + } => execute(file, *decompress_strings, args, cancellation_token), + Commands::Format { + file, + output, + inline, + } => format(file, output, *inline), + Commands::Test {} => formatter::run_tests(), + Commands::External(_args) => { + return 99; + } }; if let Err(e) = res { - shlog!("Error: {}", e); + shlog_error!("Error: {}", e); 1 } else { 0 } } -fn load(matches: &ArgMatches, cancellation_token: Arc) -> Result<(), &'static str> { - // we need to do this here or old path will fail - unsafe { - shards::core::Core = shardsInterface(SHARDS_CURRENT_ABI as u32); +fn format(file: &str, output: &Option, inline: bool) -> Result<(), Error> { + if output.is_some() && inline { + return Err("Cannot use both -i and -o".into()); } + let mut in_str = if file == "-" { + std::io::read_to_string(std::io::stdin()).unwrap() + } else { + fs::read_to_string(file.clone())? + }; + + // add new line at the end of the file to be able to parse it correctly + in_str.push('\n'); + + if inline { + let mut buf = std::io::BufWriter::new(Vec::new()); + let mut v = formatter::FormatterVisitor::new(&mut buf, &in_str); + + crate::ast_visitor::process(&in_str, &mut v)?; + + fs::write(file, &buf.into_inner()?[..])?; + } else { + let mut out_stream: Box = if let Some(out) = output { + Box::new(fs::File::create(out)?) + } else { + Box::new(std::io::stdout()) + }; + + let mut v = formatter::FormatterVisitor::new(out_stream.as_mut(), &in_str); + crate::ast_visitor::process(&in_str, &mut v)?; + } + + std::io::stdout().flush()?; + + Ok(()) +} + +fn load( + file: &str, + args: &Vec, + decompress_strings: bool, + cancellation_token: Arc, +) -> Result<(), Error> { shlog!("Loading file"); - let file = matches - .get_one::("FILE") - .ok_or("A binary Shards file to parse")?; shlog!("Parsing binary file: {}", file); let ast = { @@ -168,30 +240,28 @@ fn load(matches: &ArgMatches, cancellation_token: Arc) -> Result<(), bincode::deserialize(&file_content).unwrap() }; - execute_seq(matches, ast, cancellation_token) + Ok(execute_seq(&args, ast, cancellation_token)?) } fn execute_seq( - matches: &ArgMatches, + args: &Vec, ast: Sequence, cancellation_token: Arc, ) -> Result<(), &'static str> { let mut defines = HashMap::new(); - let args = matches.get_many::("args"); - if let Some(args) = args { - for arg in args { - shlog!("arg: {}", arg); - // find the first column and split it, the rest is the value - let mut split = arg.split(':'); - let key = split.next().unwrap(); - // value should be all the rest, could contain ':' even - let value = split.collect::>().join(":"); - // finally unescape the value if needed - let value = value.replace("\\:", ":"); - // and remove quotes if quoted - let value = value.trim_matches('"'); - defines.insert(key.to_owned(), value.to_owned()); - } + + for arg in args { + shlog!("arg: {}", arg); + // find the first column and split it, the rest is the value + let mut split = arg.split(':'); + let key = split.next().unwrap(); + // value should be all the rest, could contain ':' even + let value = split.collect::>().join(":"); + // finally unescape the value if needed + let value = value.replace("\\:", ":"); + // and remove quotes if quoted + let value = value.trim_matches('"'); + defines.insert(key.to_owned(), value.to_owned()); } let wire = { @@ -235,13 +305,7 @@ fn execute_seq( } } -fn build(matches: &ArgMatches, as_json: bool) -> Result<(), &'static str> { - // we need to do this here or old path will fail - unsafe { - shards::core::Core = shardsInterface(SHARDS_CURRENT_ABI as u32); - } - - let file = matches.get_one::("FILE").ok_or("A file to parse")?; +fn build(file: &str, output: &str, depfile: Option<&str>, as_json: bool) -> Result<(), Error> { shlog!("Parsing file: {}", file); let (deps, ast) = { @@ -276,10 +340,6 @@ fn build(matches: &ArgMatches, as_json: bool) -> Result<(), &'static str> { (deps, ast) }; - let output = matches - .get_one::("output") - .ok_or("An output file to write to")?; - // write sequence to file { let mut file = std::fs::File::create(output).unwrap(); @@ -301,7 +361,7 @@ fn build(matches: &ArgMatches, as_json: bool) -> Result<(), &'static str> { } } - if let Some(out_dep_file) = matches.get_one::("depfile") { + if let Some(out_dep_file) = depfile { let mut file = std::fs::File::create(out_dep_file).unwrap(); let mut writer = std::io::BufWriter::new(&mut file); @@ -316,25 +376,11 @@ fn build(matches: &ArgMatches, as_json: bool) -> Result<(), &'static str> { } fn execute( - matches: &clap::ArgMatches, + file: &str, + decompress_strings: bool, + args: &Vec, cancellation_token: Arc, -) -> Result<(), &'static str> { - // we need to do this here or old path will fail - unsafe { - shards::core::Core = shardsInterface(SHARDS_CURRENT_ABI as u32); - } - - let decompress = matches.get_one::("decompress-strings").unwrap(); - if decompress == "true" { - shlog!("Decompressing strings"); - unsafe { - (*Core).decompressStrings.unwrap()(); - } - } - - let file = matches - .get_one::("FILE") - .ok_or("A file to evaluate")?; +) -> Result<(), Error> { shlog!("Evaluating file: {}", file); let ast = { @@ -356,5 +402,5 @@ fn execute( })? }; - Ok(execute_seq(matches, ast.sequence, cancellation_token)?) + Ok(execute_seq(args, ast.sequence, cancellation_token)?) } diff --git a/shards/lang/src/error.rs b/shards/lang/src/error.rs new file mode 100644 index 0000000000..02a5926011 --- /dev/null +++ b/shards/lang/src/error.rs @@ -0,0 +1,12 @@ +use pest::RuleType; +use std::fmt::Display; + +pub type Error = Box; + +pub fn fmt_err(msg: M, r: &pest::Span) -> Error { + format!("{}: {}", r.as_str(), msg).to_string().into() +} + +pub fn fmt_errp<'i, M: Display, R: RuleType>(msg: M, pair: &pest::iterators::Pair<'i, R>) -> Error { + fmt_err(msg, &pair.as_span()) +} diff --git a/shards/lang/src/formatter.rs b/shards/lang/src/formatter.rs new file mode 100644 index 0000000000..0eb8a8df0b --- /dev/null +++ b/shards/lang/src/formatter.rs @@ -0,0 +1,742 @@ +use std::{ + fs::{self, File}, + io::Read, + path::{Path, PathBuf}, +}; + +use crate::ast_visitor::Visitor; +use pest::iterators::Pair; + +pub type Rule = crate::ast::Rule; + +// Identifies the last written value type +#[derive(PartialEq, Debug)] +enum FormatterTop { + None, + Atom, + Comment, + LineFunc, +} + +#[derive(Clone, Copy, Default, Debug)] +struct CollectionStyling { + start_line: usize, + // The indent for the first item of this collections, subsequent items will use the same indent + indent: Option, + // Joined collection, where braces are connected to the items e.g.: + // {a:1 + // b:2} + is_joined: Option, +} + +#[derive(Clone, Copy, Debug)] +enum Context { + Unknown, + Pipeline, + Seq(CollectionStyling), + Table(CollectionStyling), +} + +pub struct Options { + pub indent: usize, +} + +impl Default for Options { + fn default() -> Self { + Self { indent: 2 } + } +} + +pub struct FormatterVisitor<'a> { + out: &'a mut dyn std::io::Write, + options: Options, + top: FormatterTop, + line_length: usize, + line_counter: usize, + depth: usize, + last_char: usize, + context_stack: Vec, + + input: String, +} + +enum UserLine { + Newline, + Comment(String), +} + +#[derive(Default)] +struct UserStyling { + lines: Vec, +} + +struct QuoteState { + start: usize, + num_quotes: i32, +} + +impl<'a> FormatterVisitor<'a> { + pub fn new(out: &'a mut dyn std::io::Write, input: &str) -> Self { + Self { + out, + options: Options::default(), + top: FormatterTop::None, + line_length: 0, + line_counter: 0, + depth: 0, + input: input.into(), + last_char: 0, + context_stack: vec![Context::Unknown], + } + } + + fn get_context(&self) -> Context { + *self.context_stack.last().unwrap() + } + + fn get_context_mut(&mut self) -> &mut Context { + self.context_stack.last_mut().unwrap() + } + + fn determine_collection_styling(&self, pair: &Pair) -> CollectionStyling { + let mut styling = CollectionStyling::default(); + // let (start_line, _start_col) = pair.line_col(); + styling.start_line = self.line_counter; + // let mut inner: pest::iterators::Pairs<'_, crate::ast::Rule> = pair.clone().into_inner(); + // if inner.len() > 0 { + // // let item = inner.next().unwrap(); + // // let (item_line, item_col) = item.line_col(); + // // if item_line == start_line { + // // let spacing = i32::max(0, item_col as i32 - start_col as i32) as usize; + // // let base_indent = self.get_indent_opts(); + // // styling.indent = Some(base_indent + spacing); + // // } + // } + styling + } + + fn get_indent_opts(&self) -> usize { + let ctx = self.get_context(); + match ctx { + Context::Pipeline | Context::Unknown => self.depth * self.options.indent, + Context::Seq(s) | Context::Table(s) => { + if let Some(i) = s.indent { + i + } else { + self.depth * self.options.indent + } + } + } + } + + fn with_context(&mut self, ctx: Context, inner: F) -> Context { + self.context_stack.push(ctx); + inner(self); + self.context_stack.pop().unwrap() + } + + fn set_last_char(&mut self, ptr: usize) { + self.last_char = ptr; + } + + fn extract_styling(&self, until: usize) -> Option { + let from = self.last_char; + if until <= from { + return None; + } + + let mut us: UserStyling = UserStyling::default(); + let mut comment_start: Option = None; + let interpolated = &self.input[from..until]; + for (i, c) in interpolated.chars().enumerate() { + if c == ';' && comment_start.is_none() { + comment_start = Some(i + from + 1); + } else if c == '\n' { + if let Some(start) = comment_start { + let comment = &self.input[start..i + from]; + let comment_str = comment.trim_start().to_string(); + us.lines.push(UserLine::Comment(comment_str)); + comment_start = None; + } else { + us.lines.push(UserLine::Newline); + } + } + } + if let Some(start) = comment_start { + let comment = &self.input[start..until]; + us.lines.push(UserLine::Comment(comment.into())); + } + if !us.lines.is_empty() { + return Some(us); + } + None + } + + fn interpolate(&mut self, new_pair: &Pair) { + self.interpolate_at_pos(new_pair.as_span().start()); + } + + // Will interpolate any user-defined comments and newline styling + // up until the given position + fn interpolate_at_pos(&mut self, ptr: usize) { + self.interpolate_at_pos_ext(ptr, false); + } + + fn interpolate_at_pos_ext(&mut self, ptr: usize, strip_final_newline: bool) { + if let Some(us) = self.extract_styling(ptr) { + for (i, line) in us.lines.iter().enumerate() { + match line { + UserLine::Newline => { + if i == us.lines.len() - 1 && strip_final_newline { + continue; + } + self.newline(); + } + UserLine::Comment(line) => { + self.write(&format!("; {}", line), FormatterTop::Comment); + if i < us.lines.len() - 1 { + self.newline(); + } + } + } + } + } + self.set_last_char(ptr); + } + + fn write_raw(&mut self, s: &str) { + for c in s.chars() { + if c == '\n' { + self.line_length = 0; + } else { + self.line_length += 1; + } + } + self.out.write_all(s.as_bytes()).unwrap(); + } + + fn write(&mut self, s: &str, top: FormatterTop) { + self.write_pre_space(); + self.write_raw(s); + self.top = top; + } + + fn newline(&mut self) { + self.write_raw("\n"); + self.line_length = 0; + self.line_counter += 1; + for _ in 0..self.get_indent_opts() { + self.write_raw(" "); + } + self.top = FormatterTop::None; + } + + // Insert a newline before any closing bracket for tables/seq/pipelines/etc. + // if the opening bracket is on the same line as an item, the closing bracked will also be on the same line as the last item + fn opt_pre_closing_newline(&mut self, ctx: Context) { + match ctx { + Context::Seq(s) | Context::Table(s) => { + if s.is_joined.is_none() { + // Empty collection, no newline + return; + } + if s.is_joined.is_some_and(|x| x) { + // no newline + return; + } + } + _ => {} + } + self.newline(); + } + + fn get_next_atom_col(&self) -> usize { + match self.top { + FormatterTop::Comment => self.get_indent_opts(), + FormatterTop::Atom => self.line_length + 1, + _ => self.line_length, + } + } + + fn write_pre_space(&mut self) { + match self.top { + FormatterTop::None => {} + FormatterTop::Atom => { + self.write_raw(" "); + } + FormatterTop::Comment => { + self.newline(); + } + FormatterTop::LineFunc => { + self.newline(); + } + } + self.top = FormatterTop::None; + } + + // Write while spacing between previous atoms + fn write_atom(&mut self, s: &str) { + self.write(s, FormatterTop::Atom); + } + + // Write without spacing between atomics (for symbols and such) + fn write_joined(&mut self, s: &str) { + match self.top { + FormatterTop::Atom | FormatterTop::None => { + self.write_raw(s); + } + FormatterTop::Comment | FormatterTop::LineFunc => { + self.newline(); + self.write_raw(s); + } + } + self.top = FormatterTop::Atom; + } + + fn filter<'b>(&mut self, v: &'b str) -> String { + let mut result = String::new(); + let mut comment = false; + let mut quote_open: Option = None; + let mut quote_close: Option = None; + for (i, c) in v.chars().enumerate() { + if !comment && c == '"' { + match &mut quote_open { + None => { + quote_open = Some(QuoteState { + start: i, + num_quotes: 1, + }); + } + Some(qo) => { + if qo.num_quotes < 3 && (i - 1) == qo.start { + qo.num_quotes += 1; + qo.start = i; + } else { + match &mut quote_close { + None => { + quote_close = Some(QuoteState { + start: i, + num_quotes: 1, + }); + } + Some(qc) => { + qc.num_quotes += 1; + if qc.num_quotes == qo.num_quotes { + quote_close = None; + quote_open = None; + } + } + } + } + } + }; + } else { + quote_close = None; + } + if quote_open.is_some() { + result.push(c); + continue; + } + if c == ';' { + comment = true; + } + if c == '\n' || c == '\r' { + comment = false; + continue; + } + if c == ' ' || c == '\t' { + continue; // Ignore whitespace + } + if !comment { + result.push(c); + } + } + result + } + + fn write_func_after_open(&mut self, pair: &Pair, inner: F) { + let start_line = self.line_counter; + let omit_indent = omit_shard_param_indent(pair.clone()); + if omit_indent { + inner(self); + } else { + self.depth += 1; + inner(self); + self.depth -= 1; + } + + if !omit_indent && start_line != self.line_counter { + self.newline(); + } + self.write_joined(")"); + } + + fn update_collection_first_item(&mut self) { + let ll = self.get_next_atom_col(); + let lc = self.line_counter; + match self.get_context_mut() { + Context::Seq(s) | Context::Table(s) => { + if s.indent.is_none() { + s.indent = Some(ll); + s.is_joined = Some(lc == s.start_line); + } + } + _ => {} + } + } +} + +// Checks if this is a function without any parameters +fn omit_shard_params(func: Pair) -> bool { + let mut inner = func.clone().into_inner(); + let _name = inner.next().unwrap(); + if let Some(params) = inner.next() { + let params: pest::iterators::Pairs<'_, Rule> = params.into_inner(); + let num_params = params.len(); + if num_params == 0 { + return true; + } + } else { + return true; + } + return false; +} + +fn omit_builtin_params(func: Pair) -> bool { + let mut inner = func.clone().into_inner(); + let _name = inner.next().unwrap(); + if let Some(_) = inner.next() { + return false; + } + return true; +} + +// Checks if all param starting points are on the same line as the function name +// Examples are: +// Once({... , If(Something {..., etc. +fn omit_shard_param_indent(func: Pair) -> bool { + let mut inner = func.clone().into_inner(); + let _name = inner.next().unwrap(); + let start_line = _name.line_col().0; + if let Some(params) = inner.next() { + let params = params.into_inner(); + let end_line = params + .clone() + .last() + .map(|x| x.as_span().end_pos().line_col().0) + .unwrap_or(start_line); + for param in params { + let mut param_inner = param.into_inner(); + let v = if param_inner.len() == 2 { + param_inner.next(); + param_inner.next() + } else { + param_inner.next() + } + .unwrap() + .into_inner() + .next() + .unwrap(); + + let col = v.line_col().0; + if col != start_line && col != end_line { + return false; + } + } + } + return true; +} + +impl<'a> Visitor for FormatterVisitor<'a> { + fn v_pipeline(&mut self, pair: Pair, inner: T) { + self.with_context(Context::Pipeline, |s| { + s.interpolate(&pair); + inner(s); + }); + } + fn v_stmt(&mut self, pair: Pair, inner: T) { + self.interpolate(&pair); + inner(self); + } + fn v_value(&mut self, pair: Pair) { + self.interpolate(&pair); + self.update_collection_first_item(); + + let ctx = self.get_context(); + if self.top == FormatterTop::Atom { + if let Context::Pipeline = ctx { + self.write_atom("|"); + } + } + + let inner = pair.clone().into_inner(); + if inner.len() > 0 { + self.write_atom(inner.as_str()); + let last = inner.last().unwrap(); + self.set_last_char(last.as_span().end()); + } else { + self.write_atom(pair.as_str()); + self.set_last_char(pair.as_span().end()); + } + } + fn v_assign( + &mut self, + pair: Pair, + op: Pair, + to: Pair, + inner: T, + ) { + self.interpolate(&pair); + inner(self); + let op = self.filter(op.as_str()); + let to = self.filter(to.as_str()); + self.write_atom(&format!("{} {}", op, to)); + } + fn v_func(&mut self, pair: Pair, name: Pair, inner: T) { + let ctx = self.get_context(); + self.with_context(Context::Unknown, |_self| { + _self.interpolate_at_pos(pair.as_span().start()); + + if _self.top == FormatterTop::Atom { + if let Context::Pipeline = ctx { + _self.write_atom("|"); + } + } + + let name_str = _self.filter(name.as_str()); + match pair.as_rule() { + Rule::Func => { + if omit_builtin_params(pair.clone()) { + // This is most likely a @define + _self.write_atom(&format!("@{}", name_str)); + } else { + _self.write(&format!("@{}(", name_str), FormatterTop::None); + _self.write_func_after_open(&pair, inner); + if name_str == "wire" + || name_str == "define" + || name_str == "template" + || name_str == "mesh" + || name_str == "schedule" + { + _self.top = FormatterTop::LineFunc; + } + } + } + _ => { + if omit_shard_params(pair.clone()) { + _self.write_atom(&format!("{}", name_str)); + } else { + _self.write(&format!("{}(", name_str), FormatterTop::None); + _self.write_func_after_open(&pair, inner); + } + } + } + + let last = pair.clone().into_inner().last().unwrap(); + let last = if last.as_rule() == Rule::VarName { + // In case this is an argumentless builtin '@something', expand the identifier to get the last token before whitespace + last.into_inner().last().unwrap() + } else { + last + }; + _self.set_last_char(last.as_span().end()); + }); + } + + fn v_param(&mut self, pair: Pair, kw: Option>, inner: T) { + self.with_context(Context::Unknown, |_self| { + _self.interpolate(&pair); + if let Some(kw) = kw { + let kw_str = _self.filter(kw.as_str()); + _self.write_atom(&format!("{}", kw_str)); + _self.interpolate_at_pos(kw.as_span().end()); + } + inner(_self); + }); + } + fn v_expr(&mut self, pair: Pair, inner_pair: Pair, inner: T) { + self.interpolate(&pair); + self.write("(", FormatterTop::None); + let starting_line = self.line_counter; + self.depth += 1; + { + inner(self); + self.interpolate_at_pos_ext(inner_pair.as_span().end(), true); + } + self.depth -= 1; + if self.line_counter != starting_line { + self.newline(); + } + self.write_joined(")"); + } + fn v_eval_expr( + &mut self, + pair: Pair, + inner_pair: Pair, + inner: T, + ) { + self.interpolate(&pair); + self.write("#(", FormatterTop::None); + let starting_line = self.line_counter; + self.depth += 1; + { + inner(self); + self.interpolate_at_pos_ext(inner_pair.as_span().end(), true); + } + self.depth -= 1; + if self.line_counter != starting_line { + self.newline(); + } + self.write_joined(")"); + } + fn v_shards(&mut self, pair: Pair, inner: T) { + self.with_context(Context::Pipeline, |_self| { + let start = pair.as_span().start(); + _self.interpolate_at_pos(start); + _self.write("{", FormatterTop::None); + let starting_line = _self.line_counter; + + _self.depth += 1; + { + inner(_self); + let end = pair.as_span().end(); + _self.interpolate_at_pos_ext(end, true); + } + _self.depth -= 1; + if _self.line_counter != starting_line { + _self.newline(); + } + _self.write_joined("}"); + }); + } + fn v_table(&mut self, pair: Pair, inner: T) { + self.interpolate(&pair); + self.write("{", FormatterTop::None); + let ctx = Context::Table(self.determine_collection_styling(&pair)); + let ctx = self.with_context(ctx, |_self| { + _self.depth += 1; + inner(_self); + _self.interpolate_at_pos_ext(pair.as_span().end(), true); + _self.depth -= 1; + }); + self.opt_pre_closing_newline(ctx); + self.write_joined("}"); + } + fn v_seq(&mut self, pair: Pair, inner: T) { + self.interpolate(&pair); + self.write("[", FormatterTop::None); + + let ctx = Context::Seq(self.determine_collection_styling(&pair)); + let ctx = self.with_context(ctx, |_self| { + _self.depth += 1; + inner(_self); + _self.interpolate_at_pos_ext(pair.as_span().end(), true); + _self.depth -= 1; + }); + self.opt_pre_closing_newline(ctx); + self.write_joined("]"); + } + fn v_table_val( + &mut self, + pair: Pair, + _key: Pair, + inner_key: TK, + inner_val: TV, + ) { + self.interpolate(&_key); + inner_key(self); + self.write_joined(":"); + self.interpolate(&pair); + inner_val(self); + } + fn v_take_seq(&mut self, pair: Pair) { + let str = self.filter(pair.as_str()); + self.write_atom(&str); + } + fn v_take_table(&mut self, pair: Pair) { + let str = self.filter(pair.as_str()); + self.write_atom(&str); + } +} + +pub fn format_str(input: &str) -> Result { + let mut buf = std::io::BufWriter::new(Vec::new()); + let mut v = FormatterVisitor::new(&mut buf, &input); + + crate::ast_visitor::process(&input, &mut v)?; + + Ok(String::from_utf8(buf.into_inner()?)?) +} + +fn strequal_ignore_line_endings(a: &str, b: &str) -> bool { + let a = a.replace("\r\n", "\n"); + let b = b.replace("\r\n", "\n"); + a == b +} + +fn format_file_validate(input_path: &Path) -> Result<(), crate::error::Error> { + let input_str = fs::read_to_string(input_path)?; + let expected = input_path.with_extension("expected.shs"); + + if let Ok(f) = File::open(expected.clone()) { + eprintln!("Validating test {:?} against {:?}", input_path, expected); + + let mut f = f; + let mut expected_str = String::new(); + f.read_to_string(&mut expected_str)?; + + let formatted = format_str(&input_str)?; + if !strequal_ignore_line_endings(&formatted, &expected_str) { + return Err(format!("Test output does not match, was: {}", formatted).into()); + } + + let reformatted = format_str(&formatted)?; + if !strequal_ignore_line_endings(&reformatted, &formatted) { + return Err( + format!( + "Formatted output changes after formatting twice: {}", + reformatted + ) + .into(), + ); + } + } else { + eprintln!("Generating expected result for test {:?}", input_path); + let formatted = format_str(&input_str)?; + fs::write(expected, formatted)?; + } + + Ok(()) +} + +pub fn run_tests() -> Result<(), crate::error::Error> { + let mut any_failure = false; + let test_files = fs::read_dir(".")?; + for file in test_files { + let p = &file?.path(); + if p + .file_name() + .is_some_and(|x| x.to_str().unwrap().ends_with(".expected.shs")) + { + continue; + } + + if !p + .extension() + .is_some_and(|x| x.to_ascii_lowercase() == "shs") + { + continue; + } + + if let Err(e) = format_file_validate(p) { + eprintln!("Test {:?} failed: {}", p, e); + any_failure = true + } + } + + if any_failure { + Err("Test failures".into()) + } else { + Ok(()) + } +} diff --git a/shards/lang/src/lib.rs b/shards/lang/src/lib.rs index 909ce74f94..4efb7f52a6 100644 --- a/shards/lang/src/lib.rs +++ b/shards/lang/src/lib.rs @@ -7,8 +7,10 @@ extern crate clap; mod ast; mod cli; mod eval; -// mod print; mod read; +mod error; +mod ast_visitor; +mod formatter; use crate::ast::*; @@ -23,7 +25,6 @@ use shards::core::register_shard; use shards::shlog_error; use shards::types::Var; use shards::SHString; -// use print::print_ast; use std::ops::Deref; diff --git a/shards/lang/src/read.rs b/shards/lang/src/read.rs index fb0614391b..89118ab589 100644 --- a/shards/lang/src/read.rs +++ b/shards/lang/src/read.rs @@ -842,6 +842,16 @@ fn process_params(pair: Pair, env: &mut ReadEnv) -> Result, Sha pair.into_inner().map(|x| process_param(x, env)).collect() } +pub fn parse(code: &str) -> Result, ShardsError> { + ShardsParser::parse(Rule::Program, code).map_err(|e| { + ( + format!("Failed to parse file: {}", e), + LineInfo { line: 0, column: 0 }, + ) + .into() + }) +} + pub fn read_with_env(code: &str, env: &mut ReadEnv) -> Result { let successful_parse: pest::iterators::Pairs<'_, Rule> = { ShardsParser::parse(Rule::Program, code).map_err(|e| { diff --git a/shards/lang/src/tests/test1.expected.shs b/shards/lang/src/tests/test1.expected.shs new file mode 100644 index 0000000000..3089a77d23 --- /dev/null +++ b/shards/lang/src/tests/test1.expected.shs @@ -0,0 +1,7 @@ +[1 2 3] = seq + +seq:0 | something | Log("First") +seq:2 | Log("Last") + +{a: 1 b: 2} = table +table:a | Log("Table a") \ No newline at end of file diff --git a/shards/lang/src/tests/test1.shs b/shards/lang/src/tests/test1.shs new file mode 100644 index 0000000000..c12faaa1ed --- /dev/null +++ b/shards/lang/src/tests/test1.shs @@ -0,0 +1,7 @@ +[ 1 2 3] =seq + +seq: 0 something Log("First") +seq: 2 Log("Last") + +{a:1 b:2}=table +table:a Log("Table a") \ No newline at end of file diff --git a/shards/lang/src/tests/test10.expected.shs b/shards/lang/src/tests/test10.expected.shs new file mode 100644 index 0000000000..9522272f2b --- /dev/null +++ b/shards/lang/src/tests/test10.expected.shs @@ -0,0 +1,14 @@ +@macro(if [cond yes no] { + Sequence(pipelines) + cond | If(IsAny([true "true" 1]) { + {Pipeline: {blocks: [{content: + {Shard: {name: {name: "Sub" namespaces: []} + params: [{name: none value: (@ast(yes) | FromJson)}]}}}]}} + } { + {Pipeline: {blocks: [{content: + {Shard: {name: {name: "Sub" namespaces: []} + params: [{name: none value: (@ast(no) | FromJson)}]}}}]}} + } + ) >> pipelines + {statements: pipelines} | ToJson +}) \ No newline at end of file diff --git a/shards/lang/src/tests/test10.shs b/shards/lang/src/tests/test10.shs new file mode 100644 index 0000000000..6ae127f543 --- /dev/null +++ b/shards/lang/src/tests/test10.shs @@ -0,0 +1,15 @@ +@macro(if [cond yes no] { + Sequence(pipelines) + cond | If(IsAny([true "true" 1]) { + {Pipeline: {blocks: [{content: + {Shard: {name: {name: "Sub" namespaces: []} + params:[{name: none value: (@ast(yes) | FromJson)}] + }}}]}} + } { + {Pipeline: {blocks: [{content: + {Shard: {name: {name: "Sub" namespaces: []} + params:[{name: none value: (@ast(no) | FromJson)}] + }}}]}} + }) >> pipelines + {statements: pipelines} | ToJson +}) diff --git a/shards/lang/src/tests/test11.expected.shs b/shards/lang/src/tests/test11.expected.shs new file mode 100644 index 0000000000..92d9e6965d --- /dev/null +++ b/shards/lang/src/tests/test11.expected.shs @@ -0,0 +1,6 @@ +; Leading comment + +{j: [1 2 + 3] + v: 200 + k: 2} \ No newline at end of file diff --git a/shards/lang/src/tests/test11.shs b/shards/lang/src/tests/test11.shs new file mode 100644 index 0000000000..20151457c3 --- /dev/null +++ b/shards/lang/src/tests/test11.shs @@ -0,0 +1,7 @@ +; Leading comment + +{ j: [ 1 2 +3 ] + v: 200 +k :2 +} \ No newline at end of file diff --git a/shards/lang/src/tests/test2.expected.shs b/shards/lang/src/tests/test2.expected.shs new file mode 100644 index 0000000000..4df3dd41f3 --- /dev/null +++ b/shards/lang/src/tests/test2.expected.shs @@ -0,0 +1,6 @@ +{ + ; wires + ; magically scheduled in fbl.cpp + ; seek for auto wire = getScriptWire(key); + main: start +} \ No newline at end of file diff --git a/shards/lang/src/tests/test2.shs b/shards/lang/src/tests/test2.shs new file mode 100644 index 0000000000..0d32a6ef89 --- /dev/null +++ b/shards/lang/src/tests/test2.shs @@ -0,0 +1,6 @@ +{ + ; wires + ; magically scheduled in fbl.cpp + ; seek for auto wire = getScriptWire(key); + main: start +} diff --git a/shards/lang/src/tests/test3.expected.shs b/shards/lang/src/tests/test3.expected.shs new file mode 100644 index 0000000000..e049e0f0e5 --- /dev/null +++ b/shards/lang/src/tests/test3.expected.shs @@ -0,0 +1 @@ +fbl/default-lighting-feature \ No newline at end of file diff --git a/shards/lang/src/tests/test3.shs b/shards/lang/src/tests/test3.shs new file mode 100644 index 0000000000..e049e0f0e5 --- /dev/null +++ b/shards/lang/src/tests/test3.shs @@ -0,0 +1 @@ +fbl/default-lighting-feature \ No newline at end of file diff --git a/shards/lang/src/tests/test4.expected.shs b/shards/lang/src/tests/test4.expected.shs new file mode 100644 index 0000000000..7c6571271f --- /dev/null +++ b/shards/lang/src/tests/test4.expected.shs @@ -0,0 +1,3 @@ +@define(client-viz-colors [ + @color(0x9a9a9a) ; Grey +]) \ No newline at end of file diff --git a/shards/lang/src/tests/test4.shs b/shards/lang/src/tests/test4.shs new file mode 100644 index 0000000000..5be8bba388 --- /dev/null +++ b/shards/lang/src/tests/test4.shs @@ -0,0 +1,3 @@ +@define(client-viz-colors [ + @color(0x9a9a9a) ; Grey +]) diff --git a/shards/lang/src/tests/test5.expected.shs b/shards/lang/src/tests/test5.expected.shs new file mode 100644 index 0000000000..42aad59e00 --- /dev/null +++ b/shards/lang/src/tests/test5.expected.shs @@ -0,0 +1,6 @@ +@template(get-client-viz-color [username] { + @client-viz-colors = colors + colors | Take(( + username | Hash | Take(0) | Math.Abs | Math.Mod((Count(colors))) + )) +}) \ No newline at end of file diff --git a/shards/lang/src/tests/test5.shs b/shards/lang/src/tests/test5.shs new file mode 100644 index 0000000000..071cf631f2 --- /dev/null +++ b/shards/lang/src/tests/test5.shs @@ -0,0 +1,5 @@ +@template(get-client-viz-color [username] { + @client-viz-colors = colors + colors | Take(( + username | Hash | Take(0) | Math.Abs | Math.Mod((Count(colors))))) +}) \ No newline at end of file diff --git a/shards/lang/src/tests/test6.expected.shs b/shards/lang/src/tests/test6.expected.shs new file mode 100644 index 0000000000..6f314303fa --- /dev/null +++ b/shards/lang/src/tests/test6.expected.shs @@ -0,0 +1,7 @@ +@define(assets-categories [ + @category-shards-script + @category-textures + @category-meshes + @category-audio + ; @category-shaders +]) \ No newline at end of file diff --git a/shards/lang/src/tests/test6.shs b/shards/lang/src/tests/test6.shs new file mode 100644 index 0000000000..6f314303fa --- /dev/null +++ b/shards/lang/src/tests/test6.shs @@ -0,0 +1,7 @@ +@define(assets-categories [ + @category-shards-script + @category-textures + @category-meshes + @category-audio + ; @category-shaders +]) \ No newline at end of file diff --git a/shards/lang/src/tests/test7.expected.shs b/shards/lang/src/tests/test7.expected.shs new file mode 100644 index 0000000000..f4e544523b --- /dev/null +++ b/shards/lang/src/tests/test7.expected.shs @@ -0,0 +1 @@ +{Shard: {}} \ No newline at end of file diff --git a/shards/lang/src/tests/test7.shs b/shards/lang/src/tests/test7.shs new file mode 100644 index 0000000000..9fa46c5ecc --- /dev/null +++ b/shards/lang/src/tests/test7.shs @@ -0,0 +1,2 @@ +{Shard: {} +} \ No newline at end of file diff --git a/shards/lang/src/tests/test8.expected.shs b/shards/lang/src/tests/test8.expected.shs new file mode 100644 index 0000000000..3e43d6585b --- /dev/null +++ b/shards/lang/src/tests/test8.expected.shs @@ -0,0 +1,6 @@ +{content: + 0} +{ + content: + 0 +} \ No newline at end of file diff --git a/shards/lang/src/tests/test8.shs b/shards/lang/src/tests/test8.shs new file mode 100644 index 0000000000..11d7a657b8 --- /dev/null +++ b/shards/lang/src/tests/test8.shs @@ -0,0 +1,7 @@ +{content: +0 +} +{ +content: +0 +} \ No newline at end of file diff --git a/shards/lang/src/tests/test9.expected.shs b/shards/lang/src/tests/test9.expected.shs new file mode 100644 index 0000000000..c3086e9a4a --- /dev/null +++ b/shards/lang/src/tests/test9.expected.shs @@ -0,0 +1,3 @@ +@define(main-db #( + output +)) \ No newline at end of file diff --git a/shards/lang/src/tests/test9.shs b/shards/lang/src/tests/test9.shs new file mode 100644 index 0000000000..c3086e9a4a --- /dev/null +++ b/shards/lang/src/tests/test9.shs @@ -0,0 +1,3 @@ +@define(main-db #( + output +)) \ No newline at end of file diff --git a/shards/log/log.cpp b/shards/log/log.cpp index 5d214773ef..f9ecc9a8b2 100644 --- a/shards/log/log.cpp +++ b/shards/log/log.cpp @@ -54,7 +54,7 @@ void redirectAll(const std::vector &sinks) { static void setupDefaultLogger(const std::string &fileName = "shards.log") { auto dist_sink = std::make_shared(); - auto sink1 = std::make_shared(); + auto sink1 = std::make_shared(); dist_sink->add_sink(sink1); // Setup log file diff --git a/shards/rust_macro/CMakeLists.txt b/shards/rust_macro/CMakeLists.txt deleted file mode 100644 index 0bee7590b8..0000000000 --- a/shards/rust_macro/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_rust_library( - NAME shards-macro - PROJECT_PATH ${CMAKE_CURRENT_LIST_DIR} -) diff --git a/shards/union/CMakeLists.txt b/shards/union/CMakeLists.txt index 5a28eb4425..ea1f7dec17 100644 --- a/shards/union/CMakeLists.txt +++ b/shards/union/CMakeLists.txt @@ -1,6 +1,6 @@ shards_generate_union(shards-cpp-union) shards_generate_rust_union(shards-rust-union - RUST_TARGETS shards-rust shards-macro-rust + RUST_TARGETS shards-rust ) add_library(shards-union INTERFACE) @@ -8,4 +8,6 @@ add_library(shards-union INTERFACE) # Insert dependency that fixes some C++ that call exposed rust functions target_link_libraries(shards-union INTERFACE shards-cpp-union shards-rust-union) -rust_copy_cargo_lock(shards-rust-union Cargo.lock) +if(NOT SHARDS_NO_RUST_UNION) + rust_copy_cargo_lock(shards-rust-union Cargo.lock) +endif()