Moved number and identifier parsing to the tokenization step

.gitignore

@@ -1,3 +1,6 @@
 /target
 *.lock
 .vscode
+*.svg
+*.data
+*.old

Cargo.toml

@@ -12,6 +12,7 @@ license = "MIT"
 console = "0.15.7" # for terminal colors
 # chumsky = "0.9.2"
 ariadne = "0.3.0" # for nice errors
+num-bigint = "0.4.3"
 
 #lsp
 lsp-server = {version = "0.7.1", optional = true}

multiply_add.sus

@@ -11,7 +11,7 @@ module ExactlyOne : bool[Size] inputs -> bool exactlyOne {
 		bool[Size] atLeastTwos;
 		atLeastOnes[0] = inputs[0];
 		atLeastTwos[0] = false;
-		for i : 1..Size {
+		for i in 1:Size {
 			atLeastOnes[i] = atLeastOnes[i-1] | inputs[i];
 			atLeastTwos[i] = atLeastTwos[i-1] | atLeastOnes[i-1] & inputs[i];
 		}
@@ -34,7 +34,12 @@ module MultiplyAdd : i32 a, i32 b, i32 c -> i32 result {
 	result = tmp + cooo * 30 + 5;
 	array_subscript = tmp;
 	@
-	int[5] myVar = {1,2,3,4,5};
+	int[5] myVar;
+	myVar[0] = 0;
+	myVar[1] = 1;
+	myVar[2] = 2;
+	myVar[3] = 3;
+	myVar[4] = 4;
 	@
 	{
 		//[
@@ -60,7 +65,8 @@ module multiply_add : i32 a, i32 b, i32 c -> i32 result, double double_result {
 	double_result = cvt_to_double(result);
 }
 
-module blur : i32'0 a -> i32'1 result : timeline (a -> /) .. (a -> r)* {
+timeline (a -> /) .. (a -> r)*
+module blur : i32'0 a -> i32'1 result {
 	state prev = a;
 	#
 	loop {
@@ -70,7 +76,8 @@ module blur : i32'0 a -> i32'1 result : timeline (a -> /) .. (a -> r)* {
 	}
 }
 
-module my_complex_operation : i32'0 a -> i32'9 result : timeline (a -> /) .. (a -> r)* {
+timeline (a -> /) .. (a -> r)*
+module my_complex_operation : i32'0 a -> i32'9 result {
 	state prev'0 = a;
 	state tmp2'6;
 	#
@@ -82,11 +89,20 @@ module my_complex_operation : i32'0 a -> i32'9 result : timeline (a -> /) .. (a
 	}
 }
 
-module seq_adder : i32'0 a -> i32'9 result : timeline (a -> /)* .. (/ -> r) {
-	state sum = a;
+timeline (a -> r) .. (/ -> r)
+module dwiogo : bool[512] data -> bool[256] out {
+    state bool[256] save = data[256:511];
+	out = data[0:255];
+	#
+	out = save;
+}
+
+timeline (a -> /)* .. (/ -> r)
+module seq_adder : i32 a -> i32 result {
+	state int sum = a;
 	#
 	loop {
-		sum = @@(sum + a);
+		sum = sum + a;
 		#
 	}
 	result = sum;

src/ast.rs

@@ -1,5 +1,5 @@
 
-use crate::tokenizer::{TokenTypeIdx};
+use crate::tokenizer::{TokenTypeIdx, TokenExtraInfo};
 use core::ops::Range;
 
 // Token span. Indices are INCLUSIVE
@@ -71,11 +71,17 @@ impl From<usize> for Span {
     }
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct IdentifierToken {
+    pub position : usize,
+    pub name_idx : TokenExtraInfo
+}
+
 #[derive(Debug)]
 pub struct SignalDeclaration {
     pub span : Span,
     pub typ : SpanExpression,
-    pub name_token : usize,
+    pub name : IdentifierToken,
     pub identifier_type : IdentifierType
 }
 
@@ -88,9 +94,9 @@ pub struct Interface {
 
 #[derive(Debug)]
 pub enum Expression {
-    Named(usize),
-    BoolConstant(bool, usize),
-    Constant(usize),
+    Named(TokenExtraInfo),
+    Constant(TokenExtraInfo),
+    BoolConstant(bool),
     UnaryOp(Box<(TokenTypeIdx, usize/*Operator token */, SpanExpression)>),
     BinOp(Box<(SpanExpression, TokenTypeIdx, usize/*Operator token */, SpanExpression)>),
     Array(Vec<SpanExpression>), // first[second, third, ...]
@@ -120,7 +126,7 @@ pub enum Statement {
 #[derive(Debug)]
 pub struct Module {
     pub span : Span,
-    pub name : usize,
+    pub name : IdentifierToken,
     pub interface : Interface,
     pub code : Vec<SpanStatement>
 }
@@ -131,25 +137,21 @@ pub struct ASTRoot {
 }
 
 pub struct VariableContext<'prev, 'ast> where 'prev : 'ast {
-    locals : Vec<(&'ast str, &'ast SignalDeclaration)>,
+    locals : Vec<&'ast SignalDeclaration>,
     prev : Option<&'prev VariableContext<'prev, 'ast>>
 }
 
 impl<'prev, 'ast> VariableContext<'prev, 'ast> {
-    pub fn get_declaration_for_str(&self, name : &str) -> Option<&'ast SignalDeclaration> {
-        for (n, decl) in &self.locals {
-            if **n == *name {
+    pub fn get_declaration_for(&self, name : TokenExtraInfo) -> Option<&'ast SignalDeclaration> {
+        for decl in &self.locals {
+            if decl.name.name_idx == name {
                 return Some(decl);
             }
         }
-        self.prev?.get_declaration_for_str(name)
+        self.prev?.get_declaration_for(name)
     }
-    pub fn get_declaration_for(&self, tok_idx : usize, token_spans : &[CharSpan], file_text : &str) -> Option<&'ast SignalDeclaration> {
-        self.get_declaration_for_str(file_text.get(token_spans[tok_idx].as_range()).unwrap())
-    }
-    pub fn add_declaration(&mut self, new_local : &'ast SignalDeclaration, token_spans : &[CharSpan], file_text : &'ast str) {
-        let name = file_text.get(token_spans[new_local.name_token].as_range()).unwrap();
-        self.locals.push((name, new_local));
+    pub fn add_declaration(&mut self, new_local : &'ast SignalDeclaration) {
+        self.locals.push(new_local);
     }
     pub fn new_initial() -> Self {
         Self{locals : Vec::new(), prev : None}
@@ -159,49 +161,51 @@ impl<'prev, 'ast> VariableContext<'prev, 'ast> {
     }
 }
 
-pub fn for_each_identifier_in_expression<F>(expr : &Expression, func : &mut F) where F: FnMut(usize) -> () {
+pub fn for_each_identifier_in_expression<F>((expr, span) : &SpanExpression, func : &mut F) where F: FnMut(IdentifierToken) -> () {
     match expr {
-        Expression::Named(token) => func(*token),
-        Expression::BoolConstant(_, _) => {},
-        Expression::Constant(_) => {},
+        Expression::Named(id) => {
+            assert!(span.0 == span.1);
+            func(IdentifierToken{name_idx : *id, position : span.0})
+        },
+        Expression::BoolConstant(_v) => {},
+        Expression::Constant(_v) => {},
         Expression::UnaryOp(b) => {
             let (_operator, _operator_pos, right) = &**b;
-            for_each_identifier_in_expression(&right.0, func);
+            for_each_identifier_in_expression(&right, func);
         }
         Expression::BinOp(b) => {
             let (left, _operator, _operator_pos, right) = &**b;
-            for_each_identifier_in_expression(&left.0, func);
-            for_each_identifier_in_expression(&right.0, func);
+            for_each_identifier_in_expression(&left, func);
+            for_each_identifier_in_expression(&right, func);
         },
         Expression::Array(args) | Expression::FuncCall(args) => {
-            for (a_expr, _a_span) in args {
-                for_each_identifier_in_expression(a_expr, func);
+            for arg in args {
+                for_each_identifier_in_expression(arg, func);
             }
         }
     }
 }
 
 #[allow(unused_variables)]
 pub trait ASTWalker {
-    fn visit_module_name(&mut self, module_name : usize) {}
+    fn visit_module_name(&mut self, module_name : IdentifierToken) {}
     fn visit_declaration(&mut self, decl : &SignalDeclaration, context : &VariableContext) {}
     fn visit_expression(&mut self, expr : &SpanExpression, context : &VariableContext) {}
     fn visit_assignment(&mut self, to : &SpanExpression, expr : &SpanExpression, context : &VariableContext) {}
 }
 
-fn walk_ast_code_block<W : ASTWalker>(walker : &mut W, code_block : &[SpanStatement], token_spans : &[CharSpan], file_text : &str, outer_context : &VariableContext) {
+fn walk_ast_code_block<W : ASTWalker>(walker : &mut W, code_block : &[SpanStatement], outer_context : &VariableContext) {
     let mut local_context = VariableContext::new_extend(outer_context);
     for statement in code_block {
         match &statement.0 {
             Statement::Declare(decl) => {
-                local_context.add_declaration(decl, token_spans, file_text);
+                local_context.add_declaration(decl);
                 walker.visit_declaration(&decl, &local_context);
             }
             Statement::DeclareAssign(decl, expr) => {
-                local_context.add_declaration(decl, token_spans, file_text);
+                local_context.add_declaration(decl);
                 walker.visit_declaration(decl, &local_context);
-                let tok = decl.name_token;
-                let tmp_local_expr = (Expression::Named(tok), Span::from(tok));
+                let tmp_local_expr = (Expression::Named(decl.name.name_idx), Span::from(decl.name.position));
                 walker.visit_assignment(expr, &tmp_local_expr, &local_context);
                 walker.visit_expression(expr, &local_context);
                 walker.visit_expression(&tmp_local_expr, &local_context);
@@ -215,7 +219,7 @@ fn walk_ast_code_block<W : ASTWalker>(walker : &mut W, code_block : &[SpanStatem
                 walker.visit_expression(expr, &local_context);
             }
             Statement::Block(code) => {
-                walk_ast_code_block(walker, &code, token_spans, file_text, &local_context);
+                walk_ast_code_block(walker, &code, &local_context);
             }
             Statement::PipelineStage(_pos) => {
 
@@ -227,20 +231,20 @@ fn walk_ast_code_block<W : ASTWalker>(walker : &mut W, code_block : &[SpanStatem
     }
 }
 
-pub fn walk_ast<W : ASTWalker>(walker : &mut W, ast : &ASTRoot, token_spans : &[CharSpan], file_text : &str, global_context : &VariableContext) {
+pub fn walk_ast<W : ASTWalker>(walker : &mut W, ast : &ASTRoot, global_context : &VariableContext) {
     for module in &ast.modules {
         walker.visit_module_name(module.name);
         let mut local_context = VariableContext::new_extend(global_context);
         for decl in &module.interface.inputs {
             walker.visit_declaration(decl, &local_context);
-            local_context.add_declaration(decl, token_spans, file_text);
+            local_context.add_declaration(decl);
         }
         for decl in &module.interface.outputs {
             walker.visit_declaration(decl, &local_context);
-            local_context.add_declaration(decl, token_spans, file_text);
+            local_context.add_declaration(decl);
         }
 
-        walk_ast_code_block(walker, &module.code, token_spans, file_text, &local_context);
+        walk_ast_code_block(walker, &module.code, &local_context);
     }
 }
 

src/dev_aid/lsp.rs

@@ -200,10 +200,10 @@ fn do_syntax_highlight(file_data : &LoadedFile, full_parse : &FullParseResult) -
     let ide_tokens = create_token_ide_info(file_text, &full_parse);
 
     let mut semantic_tokens_acc = SemanticTokensDeltaAccumulator{prev_line : 0, prev_col : 0, semantic_tokens : Vec::new()};
-    semantic_tokens_acc.semantic_tokens.reserve(full_parse.token_spans.len());
+    semantic_tokens_acc.semantic_tokens.reserve(full_parse.tokens.token_spans.len());
 
     for (idx, tok) in ide_tokens.iter().enumerate() {
-        let tok_file_pos = full_parse.token_spans[idx];
+        let tok_file_pos = full_parse.tokens.token_spans[idx];
 
         let typ = get_semantic_token_type_from_ide_token(tok);
         let mod_bits = get_modifiers_for_token(tok);

src/dev_aid/syntax_highlighting.rs

@@ -101,19 +101,19 @@ struct NameColoringWalker<'a> {
 }
 
 impl<'a> ASTWalker for NameColoringWalker<'a> {
-    fn visit_module_name(&mut self, module_name : usize) {
-        self.ide_token_info[module_name].typ = IDETokenType::Identifier(IDEIdentifierType::Interface);
+    fn visit_module_name(&mut self, module_name : IdentifierToken) {
+        self.ide_token_info[module_name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Interface);
     }
     fn visit_declaration(&mut self, decl : &SignalDeclaration, _context : &VariableContext) {
-        for_each_identifier_in_expression(&decl.typ.0, &mut |tok_idx| {
-            self.ide_token_info[tok_idx].typ = IDETokenType::Identifier(IDEIdentifierType::Type);
+        for_each_identifier_in_expression(&decl.typ, &mut |name| {
+            self.ide_token_info[name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Type);
         });
-        self.ide_token_info[decl.name_token].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
+        self.ide_token_info[decl.name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
     }
     fn visit_expression(&mut self, expr : &SpanExpression, context : &VariableContext) {
-        for_each_identifier_in_expression(&expr.0, &mut |tok_idx| {
-            if let Some(tok_decl) = context.get_declaration_for(tok_idx, self.token_spans, self.file_text) {
-                self.ide_token_info[tok_idx].typ = IDETokenType::Identifier(IDEIdentifierType::Value(tok_decl.identifier_type));
+        for_each_identifier_in_expression(&expr, &mut |name| {
+            if let Some(tok_decl) = context.get_declaration_for(name.name_idx) {
+                self.ide_token_info[name.position].typ = IDETokenType::Identifier(IDEIdentifierType::Value(tok_decl.identifier_type));
             }
         });
         //self.ide_token_info[decl.name_token].typ = IDETokenType::Identifier(IDEIdentifierType::Value(decl.identifier_type));
@@ -123,31 +123,32 @@ impl<'a> ASTWalker for NameColoringWalker<'a> {
 fn walk_name_color(ast : &ASTRoot, token_spans : &[CharSpan], file_text : &str, result : &mut [IDEToken]) {
     let mut walker = NameColoringWalker{ide_token_info : result, token_spans : token_spans, file_text : file_text};
 
-    walk_ast(&mut walker, ast, token_spans, file_text, &VariableContext::new_initial());
+    walk_ast(&mut walker, ast, &VariableContext::new_initial());
 }
 
 pub fn create_token_ide_info<'a>(file_text : &str, parsed: &FullParseResult) -> Vec<IDEToken> {
     let mut result : Vec<IDEToken> = Vec::new();
-    result.reserve(parsed.token_types.len());
+    result.reserve(parsed.tokens.len());
 
-    for &t in &parsed.token_types {
-        let initial_typ = if is_keyword(t) {
+    for t in &parsed.tokens.tokens {
+        let tok_typ = t.get_type();
+        let initial_typ = if is_keyword(tok_typ) {
             IDETokenType::Keyword
-        } else if is_bracket(t) != IsBracket::NotABracket {
+        } else if is_bracket(tok_typ) != IsBracket::NotABracket {
             IDETokenType::InvalidBracket // Brackets are initially invalid. They should be overwritten by the token_hierarchy step. The ones that don't get overwritten are invalid
-        } else if is_symbol(t) {
-            if t == kw("@") {
+        } else if is_symbol(tok_typ) {
+            if tok_typ == kw("@") {
                 IDETokenType::PipelineStage
-            } else if t == kw("#") {
+            } else if tok_typ == kw("#") {
                 IDETokenType::TimelineStage
             } else {
                 IDETokenType::Operator
             }
-        } else if is_identifier(t) {
+        } else if is_identifier(tok_typ) {
             IDETokenType::Identifier(IDEIdentifierType::Unknown)
-        } else if is_number(t) {
+        } else if is_number(tok_typ) {
             IDETokenType::Number
-        } else if is_comment(t) {
+        } else if is_comment(tok_typ) {
             IDETokenType::Comment
         } else {
             IDETokenType::Invalid
@@ -158,7 +159,7 @@ pub fn create_token_ide_info<'a>(file_text : &str, parsed: &FullParseResult) ->
 
     add_ide_bracket_depths_recursive(&mut result, 0, &parsed.token_hierarchy);
 
-    walk_name_color(&parsed.ast, &parsed.token_spans, file_text, &mut result);
+    walk_name_color(&parsed.ast, &parsed.tokens.token_spans, file_text, &mut result);
 
     result
 }
@@ -172,12 +173,12 @@ pub fn syntax_highlight_file(file_path : &str) {
         err.pretty_print_error(&file_path, &file_text)
     }
 
-    print_tokens(&file_text, &full_parse.token_spans);
+    print_tokens(&file_text, &full_parse.tokens.token_spans);
 
     let ide_tokens = create_token_ide_info(&file_text, &full_parse);
 
 
-    pretty_print(&file_text, &full_parse.token_spans, &ide_tokens);
+    pretty_print(&file_text, &full_parse.tokens.token_spans, &ide_tokens);
 
     println!("{:?}", full_parse.ast);
 }

src/errors.rs

@@ -117,8 +117,8 @@ pub fn error_unexpected_tree_node(expected : &[TokenTypeIdx], found : Option<&To
         None => {
             error_basic(Span::from(unexpected_eof_idx), format!("Unexpected End of Scope while parsing {context}. Expected {expected_list_str}"))
         },
-        Some(TokenTreeNode::PlainToken(typ, pos)) => {
-            let tok_typ_name = get_token_type_name(*typ);
+        Some(TokenTreeNode::PlainToken(tok, pos)) => {
+            let tok_typ_name = get_token_type_name(tok.get_type());
             error_basic(Span::from(*pos), format!("Unexpected Token '{tok_typ_name}' while parsing {context}. Expected {expected_list_str}"))
         },
         Some(TokenTreeNode::Block(typ, _, span)) => {