Fixes and improvements to serialization.

There were multiple issues with comment and newline lexing. Extended printing functions to support Strings with %S flag (captial 'S'). Allows for length detection. Also made it so that precision for strings is the string length.
Ed94 · Aug 9, 2023 · b5fa864 · b5fa864
1 parent bb35444
commit b5fa864
Show file tree

Hide file tree

Showing 9 changed files with 223 additions and 224 deletions.
diff --git a/project/components/ast.cpp b/project/components/ast.cpp
diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp
@@ -292,11 +292,13 @@ namespace Parser
 
 				if ( current == '\n' )
 				{
-					token.Type = TokType::NewLine;
-					token.Length ++;
 					move_forward();
 
+					token.Type = TokType::NewLine;
+					token.Length++;
+
 					Tokens.append( token );
+					// log_fmt( "NewLine: %d\n", token.Line );
 					continue;
 				}
 			}
@@ -370,14 +372,14 @@ namespace Parser
 
 							if ( current == '\r' )
 							{
-								// move_forward();
-								// token.Length++;
+								move_forward();
+								token.Length++;
 							}
 
 							if ( current == '\n' )
 							{
-								// move_forward();
-								// token.Length++;
+								move_forward();
+								token.Length++;
 								break;
 							}
 
@@ -502,13 +504,13 @@ namespace Parser
 
 						if ( current == '\r' )
 						{
-							// move_forward();
+							move_forward();
 							// content.Length++;
 						}
 
 						if ( current == '\n' )
 						{
-							// move_forward();
+							move_forward();
 							// content.Length++;
 							break;
 						}
@@ -880,7 +882,8 @@ namespace Parser
 							Tokens.append( token );
 
 							move_forward();
-							Token content = { scanner, 1, TokType::Comment, line, column, false };
+							// move_forward();
+							Token content = { scanner, 0, TokType::Comment, line, column, false };
 
 							bool star   = current == '*';
 							bool slash  = scanner[1] == '/';
@@ -1044,15 +1047,13 @@ namespace Parser
 					token.Length++;
 				}
 
-				if ( current == '\r' )
+				if ( current == '\r' && scanner[1] == '\n' )
 				{
 					move_forward();
-					// token.Length++;
 				}
-				if ( current == '\n' )
+				else if ( current == '\n' )
 				{
 					move_forward();
-					// token.Length++;
 				}
 			}
 			else
@@ -1326,10 +1327,9 @@ Code parse_static_assert()
 
 	content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text;
 
-	// content.Text = str_fmt_buf( "%.*s\n", content.Length, content.Text );
-	// content.Length++;
+	char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text );
 
-	assert->Content = get_cached_string( content );
+	assert->Content = get_cached_string( to_StrC( result ) );
 	assert->Name	= assert->Content;
 
 	Context.pop();
@@ -2197,7 +2197,7 @@ CodeVar parse_variable_after_name(
 			eat( currtok.Type );
 		}
 
-		expr_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)expr_tok.Text;
+		expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text;
 		bitfield_expr   = untyped_str( expr_tok );
 	}
 
@@ -2262,7 +2262,9 @@ Code parse_simple_preprocess( Parser::TokType which )
 		tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text;
 	}
 
-	Code result = untyped_str( tok );
+	char const* content = str_fmt_buf( "%.*s\n", tok.Length, tok.Text );
+
+	Code result = untyped_str( to_StrC( content ) );
 	Context.Scope->Name = tok;
 
 	if ( str_compare( Context.Scope->Prev->ProcName.Ptr, "parse_typedef", Context.Scope->Prev->ProcName.Len ) != 0 )
@@ -2520,7 +2522,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::Token name = Pa
 
 		Context.Scope->Start = currtok_noskip;
 
-		if ( currtok.Type == TokType::Preprocess_Hash )
+		if ( currtok_noskip.Type == TokType::Preprocess_Hash )
 			eat( TokType::Preprocess_Hash );
 
 		switch ( currtok_noskip.Type )
@@ -2906,7 +2908,7 @@ CodeBody parse_global_nspace( CodeT which )
 
 		Context.Scope->Start = currtok_noskip;
 
-		if ( currtok.Type == TokType::Preprocess_Hash )
+		if ( currtok_noskip.Type == TokType::Preprocess_Hash )
 			eat( TokType::Preprocess_Hash );
 
 		switch ( currtok_noskip.Type )

diff --git a/project/components/interface.upfront.cpp b/project/components/interface.upfront.cpp
@@ -1059,13 +1059,17 @@ CodePreprocessCond def_preprocess_cond( EPreprocessCond type, StrC expr )
 	switch (type)
 	{
 		case EPreprocessCond::If:
-			result->Type = ECode::Preprocess_If;
+			result->Type = Preprocess_If;
+		break;
 		case EPreprocessCond::IfDef:
 			result->Type = Preprocess_IfDef;
+		break;
 		case EPreprocessCond::IfNotDef:
 			result->Type = Preprocess_IfNotDef;
+		break;
 		case EPreprocessCond::ElIf:
 			result->Type = Preprocess_ElIf;
+		break;
 	}
 
 	return result;

diff --git a/project/dependencies/printing.cpp b/project/dependencies/printing.cpp
@@ -47,7 +47,8 @@ internal sw _print_string( char* text, sw max_len, _format_info* info, char cons
 	}
 
 	if ( info && info->precision >= 0 )
-		len = str_len( str, info->precision );
+		// Made the design decision for this library that precision is the length of the string.
+		len = info->precision;
 	else
 		len = str_len( str );
 
@@ -410,6 +411,15 @@ neverinline sw str_fmt_va( char* text, sw max_len, char const* fmt, va_list va )
 				len = _print_string( text, remaining, &info, va_arg( va, char* ) );
 				break;
 
+			case 'S':
+			{
+				String gen_str = String { va_arg( va, char*) };
+
+				info.precision = gen_str.length();
+				len            = _print_string( text, remaining, &info, gen_str );
+			}
+			break;
+
 			case 'r' :
 				len = _print_repeated_char( text, remaining, &info, va_arg( va, int ) );
 				break;

diff --git a/project/dependencies/string.hpp b/project/dependencies/string.hpp
@@ -226,6 +226,11 @@ struct String
 		return header.Capacity - header.Length;
 	}
 
+	char& back()
+	{
+		return Data[ length() - 1 ];
+	}
+
 	sw capacity() const
 	{
 		Header const&

diff --git a/project/helpers/helper.hpp b/project/helpers/helper.hpp
@@ -53,8 +53,8 @@ CodeBody gen_ecode( char const* path )
 	)));
 #pragma pop_macro( "local_persist" )
 
-	CodeNS nspace = def_namespace( name(ECode), def_namespace_body( args( enum_code, to_str ) ) );
-	CodeUsing    code_t  = def_using( name(CodeT), def_type( name(ECode::Type) ) );
+	CodeNS    nspace = def_namespace( name(ECode), def_namespace_body( args( enum_code, to_str ) ) );
+	CodeUsing code_t = def_using( name(CodeT), def_type( name(ECode::Type) ) );
 
 	return def_global_body( args( nspace, code_t ) );
 }
@@ -324,8 +324,8 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path )
 #pragma pop_macro( "do_once_start" )
 #pragma pop_macro( "do_once_end" )
 
-	CodeNS nspace     = def_namespace( name(ETokType), def_namespace_body( args( attribute_entires_def, enum_code, to_str, to_type ) ) );
-	CodeUsing     td_toktype = def_using( name(TokType), def_type( name(ETokType::Type) ) );
+	CodeNS    nspace     = def_namespace( name(ETokType), def_namespace_body( args( attribute_entires_def, enum_code, to_str, to_type ) ) );
+	CodeUsing td_toktype = def_using( name(TokType), def_type( name(ETokType::Type) ) );
 
 	return def_global_body( args( nspace, td_toktype ) );
 }

diff --git a/scripts/test.gen_run.ps1 b/scripts/test.gen_run.ps1
@@ -23,6 +23,9 @@ $path_gen        = Join-Path $path_test gen
 $path_test_build = Join-Path $path_test build
 $path_gen_build  = Join-Path $path_gen  build
 
+# Invoke-Expression "& $(Join-Path $PSScriptRoot 'bootstrap.ci.ps1')"
+# Invoke-Expression "& $(Join-Path $PSScriptRoot 'singleheader.ci.ps1')"
+
 write-host "`n`nBuilding Test`n"
 
 if ( -not( Test-Path $path_gen_build ) )

diff --git a/test/SOA.cpp b/test/SOA.cpp
@@ -118,9 +118,9 @@ void check_SOA()
 {
 	log_fmt("\ncheck_SOA:");
 	gen::init();
-	
+
 	Builder soa_test = Builder::open( "SOA.gen.hpp" );
-	
+
 	soa_test.print( parse_using( code(
 		using u16 = unsigned short;
 	)));

diff --git a/test/test.cpp b/test/test.cpp
@@ -16,9 +16,9 @@ int gen_main()
 
 	// check_sanity();
 
-	check_SOA();
+	// check_SOA();
 
-	// check_singleheader_ast();
+	check_singleheader_ast();
 
 	return 0;
 }