From 75b1d42ccaef6b883f3b9107e352798e26835595 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 11 Feb 2025 18:59:09 -0500 Subject: [PATCH 01/19] changes based off of metadesk genc_c11 --- base/components/code_serialization.cpp | 4 +- base/components/inlines.hpp | 2 +- base/components/interface.cpp | 13 ++++- base/components/interface.hpp | 3 + base/components/parser.cpp | 81 ++++++++++++++++++++------ base/components/parser_types.hpp | 26 ++++++--- base/dependencies/macros.hpp | 5 -- base/dependencies/memory.hpp | 2 +- gen_c_library/c_library.refactor | 3 + scripts/helpers/vendor_toolchain.ps1 | 2 +- 10 files changed, 102 insertions(+), 39 deletions(-) diff --git a/base/components/code_serialization.cpp b/base/components/code_serialization.cpp index 9ec168b..39dacb5 100644 --- a/base/components/code_serialization.cpp +++ b/base/components/code_serialization.cpp @@ -38,13 +38,13 @@ void body_to_strbuilder_export( CodeBody body, StrBuilder* result ) GEN_ASSERT(result != nullptr); strbuilder_append_fmt( result, "export\n{\n" ); - Code curr = cast(Code, body); + Code curr = body->Front; s32 left = body->NumEntries; while ( left-- ) { code_to_strbuilder_ref(curr, result); // strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) ); - ++curr; + curr = curr->Next; } strbuilder_append_fmt( result, "};\n" ); diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index 5bf6bf5..45e572c 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -38,7 +38,7 @@ void body_to_strbuilder_ref( CodeBody body, StrBuilder* result ) { code_to_strbuilder_ref(curr, result); // strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) ); - ++curr; + curr = curr->Next; } } diff --git a/base/components/interface.cpp b/base/components/interface.cpp index b3eaf45..7d65eda 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -283,6 +283,15 @@ void init(Context* ctx) ctx->InitSize_Fallback_Allocator_Bucket_Size = megabytes(8); } + if (ctx->InitSize_StrCacheTable == 0) + { + ctx->InitSize_StrCacheTable = kilobytes(8); + } + if (ctx->InitSize_MacrosTable == 0) + { + ctx->InitSize_MacrosTable = kilobytes(8); + } + // Override the current context (user has to put it back if unwanted). _ctx = ctx; @@ -311,11 +320,11 @@ void init(Context* ctx) } // Setup the hash tables { - ctx->StrCache = hashtable_init(StrCached, ctx->Allocator_DyanmicContainers); + ctx->StrCache = hashtable_init_reserve(StrCached, ctx->Allocator_DyanmicContainers, ctx->InitSize_StrCacheTable); if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); - ctx->Macros = hashtable_init(Macro, ctx->Allocator_DyanmicContainers); + ctx->Macros = hashtable_init_reserve(Macro, ctx->Allocator_DyanmicContainers, ctx->InitSize_MacrosTable); if (ctx->Macros.Hashes == nullptr || ctx->Macros.Entries == nullptr) { GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" ); } diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 7578f1c..da0a1b3 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -64,6 +64,9 @@ struct Context u32 InitSize_LexerTokens; u32 SizePer_StringArena; + u32 InitSize_StrCacheTable; + u32 InitSize_MacrosTable; + // TODO(Ed): Symbol Table // Keep track of all resolved symbols (naemspaced identifiers) diff --git a/base/components/parser.cpp b/base/components/parser.cpp index b064e04..b786613 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1443,6 +1443,22 @@ CodeFn parse_function_after_name( } // ( ) + // Check for trailing specifiers... + CodeAttributes post_rt_attributes = parse_attributes(); + if (post_rt_attributes) + { + if (attributes) + { + StrBuilder merged = strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content); + attributes->Content = cache_str(strbuilder_to_str(merged)); + } + else + { + attributes = post_rt_attributes; + } + } + // ( ) + CodeBody body = NullCode; CodeComment inline_cmt = NullCode; if ( check( Tok_BraceCurly_Open ) ) @@ -1481,17 +1497,19 @@ CodeFn parse_function_after_name( if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); - // ( ) ; + // ( ) < = 0 or delete > ; } - else + + + if (body == nullptr) { Token stmt_end = currtok; eat( Tok_Statement_End ); - // ( ) ; + // ( ) < = 0 or delete > ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); - // ( ) ; + // ( ) < = 0 or delete > ; } StrBuilder @@ -1766,7 +1784,7 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Preprocess_Macro_Expr: { - if (tok_is_attribute(currtok)) + if ( ! tok_is_attribute(currtok)) { log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); return InvalidCode; @@ -2049,11 +2067,20 @@ Token parse_identifier( bool* possible_member_function ) Macro* macro = lookup_macro(currtok.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); + b32 is_decarator = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Identifier_Decorator ); // Typename can be: '::' // If that is the case first option will be Tok_Access_StaticSymbol below - if (check(Tok_Identifier) || accept_as_identifier) - eat( Tok_Identifier ); + if (check(Tok_Identifier) || accept_as_identifier) + { + if (is_decarator) { + Code name_macro = parse_simple_preprocess(currtok.Type); + name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; + } + else { + eat(Tok_Identifier); + } + } // parse_template_args( & name ); @@ -4714,21 +4741,37 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function ) else if ( currtok.Type == Tok_Decl_Class || currtok.Type == Tok_Decl_Enum || currtok.Type == Tok_Decl_Struct || currtok.Type == Tok_Decl_Union ) { - switch (currtok.Type) { - case Tok_Decl_Class : tag = Tag_Class; break; - case Tok_Decl_Enum : tag = Tag_Enum; break; - case Tok_Decl_Struct : tag = Tag_Struct; break; - case Tok_Decl_Union : tag = Tag_Union; break; - default: - break; + Token next = nexttok; + + if (next.Type == Tok_Identifier) + { + switch (currtok.Type) { + case Tok_Decl_Class : tag = Tag_Class; break; + case Tok_Decl_Enum : tag = Tag_Enum; break; + case Tok_Decl_Struct : tag = Tag_Struct; break; + case Tok_Decl_Union : tag = Tag_Union; break; + default: + break; + } + eat( currtok.Type ); + // + + name = parse_identifier(nullptr); + // } - eat( currtok.Type ); - // + else if (next.Type == Tok_BraceCurly_Open) + { + name = currtok; + // We have an inplace definition, we need to consume that... - name = parse_identifier(nullptr); + // TODO(Ed): we need to add a way for AST_CodeTypename to track an implace definition.. + b32 const inplace = true; + Code indplace_def = cast(Code, parser_parse_struct(inplace)); - // name.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )name.Text; - // eat( Tok_Identifier ); + // For now we lose the structural information, + name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; + // + } _ctx->parser.Scope->Name = name.Text; // } diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 2393945..a2bd57a 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -168,26 +168,36 @@ Str macrotype_to_str( MacroType type ) enum EMacroFlags : u16 { - MF_Functional = bit(0), // Macro has parameters (args expected to be passed) - MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. + // Macro has parameters (args expected to be passed) + MF_Functional = bit(0), + + // Expects to assign a braced scope to its body. + MF_Expects_Body = bit(1), // lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one. - // ^^^ This is a kludge because we don't support push/pop macro pragmas rn. + // This is a kludge because we don't support push/pop macro pragmas rn. MF_Allow_As_Identifier = bit(2), + // When parsing identifiers, it will allow the consumption of the macro parameters (as its expected to be a part of constructing the identifier) + // Example of a decarator macro from stb_sprintf.h: + // STBSP__PUBLICDEC int STB_SPRINTF_DECORATE(sprintf)(char* buf, char const *fmt, ...) STBSP__ATTRIBUTE_FORMAT(2,3); + // ^^ STB_SPRINTF_DECORATE is decorating sprintf + MF_Identifier_Decorator = bit(3), + // lex__eat wil treat this macro as an attribute if the parser attempts to consume it as one. - // ^^^ This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc) + // This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc) // TODO(Ed): We can keep the MF_Allow_As_Attribute flag for macros, however, we need to add the ability of AST_Attributes to chain themselves. // Its thats already a thing in the standard language anyway // & it would allow UE_DEPRECATED, (UE_PROPERTY / UE_FUNCTION) to chain themselves as attributes of a resolved member function/variable definition - MF_Allow_As_Attribute = bit(3), + MF_Allow_As_Attribute = bit(4), // When a macro is encountered after attributes and specifiers while parsing a function, or variable: - // It will consume the macro and treat it as resolving the definition. (Yes this is for Unreal Engine) + // It will consume the macro and treat it as resolving the definition. // (MUST BE OF MT_Statement TYPE) - MF_Allow_As_Definition = bit(4), + MF_Allow_As_Definition = bit(5), - MF_Allow_As_Specifier = bit(5), // Created for Unreal's PURE_VIRTUAL + // Created for Unreal's PURE_VIRTUAL + MF_Allow_As_Specifier = bit(6), MF_Null = 0, MF_UnderlyingType = GEN_U16_MAX, diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index c8ad2d7..24b3163 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -198,21 +198,16 @@ #ifndef forceinline # if GEN_COMPILER_MSVC # define forceinline __forceinline -# define neverinline __declspec( noinline ) # elif GEN_COMPILER_GCC # define forceinline inline __attribute__((__always_inline__)) -# define neverinline __attribute__( ( __noinline__ ) ) # elif GEN_COMPILER_CLANG # if __has_attribute(__always_inline__) # define forceinline inline __attribute__((__always_inline__)) -# define neverinline __attribute__( ( __noinline__ ) ) # else # define forceinline -# define neverinline # endif # else # define forceinline -# define neverinline # endif #endif diff --git a/base/dependencies/memory.hpp b/base/dependencies/memory.hpp index eaad19d..70e5508 100644 --- a/base/dependencies/memory.hpp +++ b/base/dependencies/memory.hpp @@ -138,7 +138,7 @@ constexpr AllocatorInfo heap( void ) { AllocatorInfo allocator = { heap_allocato #define malloc( sz ) alloc( heap(), sz ) //! Helper to free memory allocated by heap allocator. -#define mfree( ptr ) free( heap(), ptr ) +#define mfree( ptr ) allocator_free( heap(), ptr ) struct VirtualMemory { diff --git a/gen_c_library/c_library.refactor b/gen_c_library/c_library.refactor index 4e1d247..96021a1 100644 --- a/gen_c_library/c_library.refactor +++ b/gen_c_library/c_library.refactor @@ -53,6 +53,7 @@ word enum_underlying, gen_enum_underlying word nullptr, gen_nullptr word struct_init, gen_struct_init word hash, gen_hash +word txt, gen_txt // Basic Types @@ -410,6 +411,8 @@ namespace var_, gen_var_ word _ctx, gen__ctx +word get_context, gen_get_context + word init, gen_init word deinit, gen_deinit word reset, gen_reset diff --git a/scripts/helpers/vendor_toolchain.ps1 b/scripts/helpers/vendor_toolchain.ps1 index f1f1baa..9d5f005 100644 --- a/scripts/helpers/vendor_toolchain.ps1 +++ b/scripts/helpers/vendor_toolchain.ps1 @@ -404,7 +404,7 @@ if ( $vendor -match "msvc" ) $flag_optimize_intrinsics = '/Oi' $flag_optimized_debug_forceinline = '/d2Obforceinline' $flag_optimized_debug = '/Zo' - $flag_ + # $flag_ # $flag_out_name = '/OUT:' $flag_path_interm = '/Fo' $flag_path_debug = '/Fd' From ef72d27f3ed275fa36c707a58be46829a165c768 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 11 Feb 2025 19:24:42 -0500 Subject: [PATCH 02/19] made optionals for c11 actually work... --- base/components/interface.upfront.cpp | 76 ++++++++++++++++++++------- base/dependencies/macros.hpp | 8 +++ gen_c_library/c_library.cpp | 7 ++- 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 94af5f7..b30864b 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -473,8 +473,10 @@ CodeComment def_comment( Str content ) return (CodeComment) result; } -CodeConstructor def_constructor( Opts_def_constructor p ) +CodeConstructor def_constructor( Opts_def_constructor opt ) { + Opts_def_constructor p = get_optional(opt); + if ( p.params && p.params->Type != CT_Parameters ) { log_failure("gen::def_constructor: params must be of Parameters type - %s", code_debug_str((Code)p.params)); GEN_DEBUG_TRAP(); @@ -510,8 +512,10 @@ CodeConstructor def_constructor( Opts_def_constructor p ) return result; } -CodeClass def_class( Str name, Opts_def_struct p ) +CodeClass def_class( Str name, Opts_def_struct opt ) { + Opts_def_struct p = get_optional(opt); + if ( ! name_check( def_class, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -561,8 +565,10 @@ CodeClass def_class( Str name, Opts_def_struct p ) return result; } -CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) +CodeDefine def_define( Str name, MacroType type, Opts_def_define opt ) { + Opts_def_define p = get_optional(opt); + if ( ! name_check( def_define, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -585,8 +591,10 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) return result; } -CodeDestructor def_destructor( Opts_def_destructor p ) +CodeDestructor def_destructor( Opts_def_destructor opt ) { + Opts_def_destructor p = get_optional(opt); + if ( p.specifiers && p.specifiers->Type != CT_Specifiers ) { log_failure( "gen::def_destructor: specifiers was not a 'Specifiers' type: %s", code_debug_str(p.specifiers) ); GEN_DEBUG_TRAP(); @@ -619,8 +627,10 @@ CodeDestructor def_destructor( Opts_def_destructor p ) return result; } -CodeEnum def_enum( Str name, Opts_def_enum p ) +CodeEnum def_enum( Str name, Opts_def_enum opt ) { + Opts_def_enum p = get_optional(opt); + if ( ! name_check( def_enum, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -742,8 +752,10 @@ CodeFriend def_friend( Code declaration ) return result; } -CodeFn def_function( Str name, Opts_def_function p ) +CodeFn def_function( Str name, Opts_def_function opt ) { + Opts_def_function p = get_optional(opt); + if ( ! name_check( def_function, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -802,8 +814,10 @@ CodeFn def_function( Str name, Opts_def_function p ) return result; } -CodeInclude def_include( Str path, Opts_def_include p ) +CodeInclude def_include( Str path, Opts_def_include opt ) { + Opts_def_include p = get_optional(opt); + if ( path.Len <= 0 || path.Ptr == nullptr ) { log_failure( "gen::def_include: Invalid path provided - %d" ); GEN_DEBUG_TRAP(); @@ -821,8 +835,10 @@ CodeInclude def_include( Str path, Opts_def_include p ) return result; } -CodeModule def_module( Str name, Opts_def_module p ) +CodeModule def_module( Str name, Opts_def_module opt ) { + Opts_def_module p = get_optional(opt); + if ( ! name_check( def_module, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -835,8 +851,10 @@ CodeModule def_module( Str name, Opts_def_module p ) return result; } -CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p ) +CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace opt ) { + Opts_def_namespace p = get_optional(opt); + if ( ! name_check( def_namespace, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -859,8 +877,10 @@ CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p ) return result; } -CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p ) +CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator opt ) { + Opts_def_operator p = get_optional(opt); + if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) { log_failure( "gen::def_operator: PlatformAttributes was provided but its not of attributes type: %s", code_debug_str(p.attributes) ); GEN_DEBUG_TRAP(); @@ -926,8 +946,10 @@ CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p ) return result; } -CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p ) +CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast opt ) { + Opts_def_operator_cast p = get_optional(opt); + if ( ! null_check( def_operator_cast, type )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -959,8 +981,10 @@ CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p ) return result; } -CodeParams def_param( CodeTypename type, Str name, Opts_def_param p ) +CodeParams def_param( CodeTypename type, Str name, Opts_def_param opt ) { + Opts_def_param p = get_optional(opt); + if ( ! name_check( def_param, name ) || ! null_check( def_param, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1034,8 +1058,10 @@ CodeSpecifiers def_specifier( Specifier spec ) return result; } -CodeStruct def_struct( Str name, Opts_def_struct p ) +CodeStruct def_struct( Str name, Opts_def_struct opt ) { + Opts_def_struct p = get_optional(opt); + if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) { log_failure( "gen::def_struct: attributes was not a `PlatformAttributes` type - %s", code_debug_str(cast(Code, p.attributes)) ); GEN_DEBUG_TRAP(); @@ -1076,8 +1102,10 @@ CodeStruct def_struct( Str name, Opts_def_struct p ) return result; } -CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template p ) +CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template opt ) { + Opts_def_template p = get_optional(opt); + if ( ! null_check( def_template, declaration ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1108,8 +1136,10 @@ CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_templat return result; } -CodeTypename def_type( Str name, Opts_def_type p ) +CodeTypename def_type( Str name, Opts_def_type opt ) { + Opts_def_type p = get_optional(opt); + if ( ! name_check( def_type, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1143,8 +1173,10 @@ CodeTypename def_type( Str name, Opts_def_type p ) return result; } -CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p ) +CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef opt ) { + Opts_def_typedef p = get_optional(opt); + if ( ! null_check( def_typedef, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1206,8 +1238,10 @@ CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p ) return result; } -CodeUnion def_union( Str name, CodeBody body, Opts_def_union p ) +CodeUnion def_union( Str name, CodeBody body, Opts_def_union opt ) { + Opts_def_union p = get_optional(opt); + if ( ! null_check( def_union, body ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1233,8 +1267,10 @@ CodeUnion def_union( Str name, CodeBody body, Opts_def_union p ) return result; } -CodeUsing def_using( Str name, CodeTypename type, Opts_def_using p ) +CodeUsing def_using( Str name, CodeTypename type, Opts_def_using opt ) { + Opts_def_using p = get_optional(opt); + if ( ! name_check( def_using, name ) || null_check( def_using, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1274,8 +1310,10 @@ CodeUsing def_using_namespace( Str name ) return result; } -CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable p ) +CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable opt ) { + Opts_def_variable p = get_optional(opt); + if ( ! name_check( def_variable, name ) || ! null_check( def_variable, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index 24b3163..b6cd161 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -314,4 +314,12 @@ # define GEN_OPITMIZE_MAPPINGS_END #endif +#ifndef get_optional +# if GEN_COMPILER_C +# define get_optional(opt) opt ? *opt : (typeof(*opt)){0} +# else +# define get_optional(opt) opt +# endif +#endif + #pragma endregion Macros diff --git a/gen_c_library/c_library.cpp b/gen_c_library/c_library.cpp index 591d79c..0d0b4e5 100644 --- a/gen_c_library/c_library.cpp +++ b/gen_c_library/c_library.cpp @@ -1159,6 +1159,8 @@ R"(#define ( code ) _Generic( (code), \ Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len - prefix.Len }; Str new_name = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str(); + opt_param->ValueType->Specs = def_specifier(Spec_Ptr); + // Resolve define's arguments b32 has_args = fn->Params->NumEntries > 1; StrBuilder params_str = StrBuilder::make_reserve(_ctx->Allocator_Temp, 32); @@ -1172,10 +1174,10 @@ R"(#define ( code ) _Generic( (code), \ } char const* tmpl_fn_macro = nullptr; if (params_str.length() > 0 ) { - tmpl_fn_macro= "#define ( ... ) ( () { __VA_ARGS__ } )\n"; + tmpl_fn_macro= "#define ( ... ) ( & () { __VA_ARGS__ } )\n"; } else { - tmpl_fn_macro= "#define ( ... ) ( () { __VA_ARGS__ } )\n"; + tmpl_fn_macro= "#define ( ... ) ( & () { __VA_ARGS__ } )\n"; } Code fn_macro = untyped_str(token_fmt( "def_name", fn->Name @@ -1504,6 +1506,7 @@ R"(#define ( code ) _Generic( (code), \ Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len - prefix.Len }; Str new_name = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str(); fn->Name = cache_str(new_name); + opt_param->ValueType->Specs = def_specifier(Spec_Ptr); } src_upfront.append(fn); } From 727b54c341910158058b8fcebf5c300cd2a37be4 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 15 Feb 2025 21:22:33 -0500 Subject: [PATCH 03/19] Added CodeSpecifiers::index_of for cpp, fix token bug with () operator in parse_operator_after_ret_type --- base/components/code_types.hpp | 1 + base/components/parser.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index 212cff8..49e6b48 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -253,6 +253,7 @@ struct CodeSpecifiers Using_Code( CodeSpecifiers ); bool append( Specifier spec ) { return specifiers_append(* this, spec); } s32 has( Specifier spec ) { return specifiers_has(* this, spec); } + s32 index_of( Specifier spec ) { return specifiers_index_of(* this, spec); } s32 remove( Specifier to_remove ) { return specifiers_remove(* this, to_remove); } StrBuilder to_strbuilder() { return specifiers_to_strbuilder(* this ); } void to_strbuilder( StrBuilder& result ) { return specifiers_to_strbuilder_ref(* this, & result); } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index b786613..b685406 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -2390,7 +2390,10 @@ CodeOperator parse_operator_after_ret_type( case '(': { if ( currtok.Text.Ptr[1] == ')' ) + { op = Op_FunctionCall; + eat(Tok_Paren_Open); + } else op = Op_Invalid; From 844d431e1cdcbc5df266bd047f5a9ee071c1014a Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 10:50:36 -0500 Subject: [PATCH 04/19] Setup Context's logger and fallback. (Not yet used in implementation). --- base/components/interface.cpp | 14 +++++- base/components/interface.hpp | 76 +++++++++++++++++++++++++++----- base/components/types.hpp | 10 ----- base/dependencies/filesystem.hpp | 2 +- base/dependencies/macros.hpp | 18 ++++++-- base/dependencies/memory.hpp | 8 +--- base/dependencies/printing.hpp | 13 ------ 7 files changed, 95 insertions(+), 46 deletions(-) diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 7d65eda..cfc67ee 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -71,6 +71,14 @@ void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, return nullptr; } +internal +void fallback_logger(LogEntry entry) +{ + GEN_ASSERT(entry.msg.Len > 0); + GEN_ASSERT(entry.msg.Ptr); + log_fmt("%S: %S", loglevel_to_str(entry.level), entry.msg); +} + internal void define_constants() { @@ -292,6 +300,10 @@ void init(Context* ctx) ctx->InitSize_MacrosTable = kilobytes(8); } + if (ctx->Logger == nullptr) { + ctx->Logger = & fallback_logger; + } + // Override the current context (user has to put it back if unwanted). _ctx = ctx; @@ -307,7 +319,7 @@ void init(Context* ctx) } // Setup the code pool and code entries arena. { - Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, sizeof(AST) ); + Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, size_of(AST) ); if ( code_pool.PhysicalStart == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the code pool" ); array_append( ctx->CodePools, code_pool ); diff --git a/base/components/interface.hpp b/base/components/interface.hpp index da0a1b3..0f58ba4 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -15,23 +15,34 @@ \▓▓▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ */ -#if 0 enum LogLevel : u32 { - Info, - Warning, - Panic, + LL_Null, + LL_Note, + LL_Warning, + LL_Error, + LL_Fatal, + LL_UnderlyingType = GEN_U32_MAX, }; +Str loglevel_to_str(LogLevel level) +{ + local_persist + Str lookup[] = { + txt("Info"), + txt("Warning"), + txt("Panic"), + }; + return lookup[level]; +} + struct LogEntry { - Str msg; - u32 line_num; - void* data; + Str msg; + LogLevel level; }; -typedef void LoggerCallback(LogEntry entry); -#endif +typedef void LoggerProc(LogEntry entry); // Note(Ed): This is subject to heavily change // with upcoming changes to the library's fallback (default) allocations strategy; @@ -70,6 +81,10 @@ struct Context // TODO(Ed): Symbol Table // Keep track of all resolved symbols (naemspaced identifiers) +// Logging + + LoggerProc* Logger; + // Parser // Used by the lexer to persistently treat all these identifiers as preprocessor defines. @@ -107,6 +122,45 @@ struct Context // An implicit context interface will be provided instead as wrapper procedures as convience. GEN_API extern Context* _ctx; +// By default this library will either crash or exit if an error is detected while generating codes. +// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur. +#ifdef GEN_DONT_USE_FATAL + #define log_failure log_fmt +#else + #define log_failure GEN_FATAL +#endif + +// TODO(Ed): Swap all usage of this with logger_fmt (then rename logger_fmt to log_fmt) +inline +ssize log_fmt(char const* fmt, ...) +{ + ssize res; + va_list va; + + va_start(va, fmt); + res = c_str_fmt_out_va(fmt, va); + va_end(va); + + return res; +} + +inline +void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...) +{ + local_persist thread_local + PrintF_Buffer buf = struct_init(PrintF_Buffer, {0}); + + va_list va; + va_start(va, fmt); + ssize res = c_str_fmt_va(buf, GEN_PRINTF_MAXLEN, fmt, va) -1; + va_end(va); + + StrBuilder msg = strbuilder_make_length(ctx->Allocator_Temp, buf, res); + + LogEntry entry = { strbuilder_to_str(msg), level }; + ctx->Logger(entry); +} + // Initialize the library. There first ctx initialized must exist for lifetime of other contextes that come after as its the one that GEN_API void init(Context* ctx); @@ -117,7 +171,7 @@ GEN_API void deinit(Context* ctx); // Retrieves the active context (not usually needed, but here in case...) GEN_API Context* get_context(); -// Clears the allocations, but doesn't free the memoery, then calls init() again. +// Clears the allocations, but doesn't free the memory, then calls init() again. // Ease of use. GEN_API void reset(Context* ctx); @@ -398,7 +452,7 @@ GEN_API ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list v //! Do not use directly. Use the token_fmt macro instead. Str token_fmt_impl( ssize, ... ); -GEN_API Code untyped_str( Str content); +GEN_API Code untyped_str ( Str content); GEN_API Code untyped_fmt ( char const* fmt, ... ); GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ); diff --git a/base/components/types.hpp b/base/components/types.hpp index 18762ea..e64a706 100644 --- a/base/components/types.hpp +++ b/base/components/types.hpp @@ -19,16 +19,6 @@ */ -using LogFailType = ssize(*)(char const*, ...); - -// By default this library will either crash or exit if an error is detected while generating codes. -// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur. -#ifdef GEN_DONT_USE_FATAL - #define log_failure log_fmt -#else - #define log_failure GEN_FATAL -#endif - enum AccessSpec : u32 { AccessSpec_Default, diff --git a/base/dependencies/filesystem.hpp b/base/dependencies/filesystem.hpp index 9d3676a..c9026b0 100644 --- a/base/dependencies/filesystem.hpp +++ b/base/dependencies/filesystem.hpp @@ -187,7 +187,7 @@ struct FileContents { AllocatorInfo allocator; void* data; - ssize size; + ssize size; }; constexpr b32 file_zero_terminate = true; diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index b6cd161..ec2106c 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -298,10 +298,20 @@ # define GEN_PARAM_DEFAULT #endif -#if GEN_COMPILER_CPP - #define struct_init(type, value) {value} -#else - #define struct_init(type, value) {value} +#ifndef struct_init +# if GEN_COMPILER_CPP +# define struct_init(type, value) value +# else +# define struct_init(type, value) (type) value +# endif +#endif + +#ifndef struct_zero +# if GEN_COMPILER_CPP +# define struct_zero(type) {} +# else +# define struct_zero(type) (type) {0} +# endif #endif #if 0 diff --git a/base/dependencies/memory.hpp b/base/dependencies/memory.hpp index 70e5508..2a7713d 100644 --- a/base/dependencies/memory.hpp +++ b/base/dependencies/memory.hpp @@ -134,12 +134,6 @@ GEN_API void* heap_allocator_proc( void* allocator_data, AllocType type, ssize s //! The heap allocator backed by operating system's memory manager. constexpr AllocatorInfo heap( void ) { AllocatorInfo allocator = { heap_allocator_proc, nullptr }; return allocator; } -//! Helper to allocate memory using heap allocator. -#define malloc( sz ) alloc( heap(), sz ) - -//! Helper to free memory allocated by heap allocator. -#define mfree( ptr ) allocator_free( heap(), ptr ) - struct VirtualMemory { void* data; @@ -185,6 +179,8 @@ void arena_check (Arena* arena); void arena_free (Arena* arena); ssize arena_size_remaining(Arena* arena, ssize alignment); +// TODO(Ed): Add arena_pos, arena_pop, and arena_pop_to + struct Arena { AllocatorInfo Backing; diff --git a/base/dependencies/printing.hpp b/base/dependencies/printing.hpp index 5ddc20e..2550254 100644 --- a/base/dependencies/printing.hpp +++ b/base/dependencies/printing.hpp @@ -26,17 +26,4 @@ GEN_API ssize c_str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ); constexpr char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; -inline -ssize log_fmt(char const* fmt, ...) -{ - ssize res; - va_list va; - - va_start(va, fmt); - res = c_str_fmt_out_va(fmt, va); - va_end(va); - - return res; -} - #pragma endregion Printing From 3b81eea6884b6dd9bd1103c870e4eedd8681fe6e Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 10:50:55 -0500 Subject: [PATCH 05/19] Doing some initial prep for parser overhaul --- base/components/ast.hpp | 5 +-- base/components/interface.hpp | 46 +++++++++++++++------------ base/components/interface.parsing.cpp | 23 ++++++++++++++ base/components/parser.cpp | 11 ++++--- base/components/parser_types.hpp | 21 +++++++----- gen_c_library/c_library.refactor | 2 +- 6 files changed, 71 insertions(+), 37 deletions(-) diff --git a/base/components/ast.hpp b/base/components/ast.hpp index 2c9f9af..0041456 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -406,7 +406,8 @@ struct AST Code PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal) }; }; - StrCached Content; // Attributes, Comment, Execution, Include + StrCached Content; // Attributes, Comment, Execution, Include + // TokenSlice Content; // TODO(Ed): Use a token slice for content struct { Specifier ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers Code NextSpecs; // Specifiers; If ArrSpecs is full, then NextSpecs is used. @@ -422,7 +423,7 @@ struct AST Code Next; Code Back; }; - Token* Token; // Reference to starting token, only available if it was derived from parsing. + Token* Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice. Code Parent; CodeType Type; // CodeFlag CodeFlags; diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 0f58ba4..bf74f1c 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -391,37 +391,41 @@ forceinline CodeBody def_union_body ( s32 num, Code* codes ) #pragma region Parsing -#if 0 -struct StackNode +struct ParseStackNode { - StackNode* Prev; - - Token Start; - Token Name; // The name of the AST node (if parsed) - Str FailedProc; // The name of the procedure that failed + ParseStackNode* Prev; + + TokenSlice tokens; + Token* Start; + Str Name; // The name of the AST node (if parsed) + Str ProcName; // The name of the procedure + Code code; // Relevant AST node + // TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling. }; -// Stack nodes are allocated the error's allocator -struct Error +struct ParseMessage { - StrBuilder message; - StackNode* context_stack; + ParseMessage* Next; + ParseStackNode* Scope; + Str Log; + LogLevel Level; }; struct ParseInfo { - Arena FileMem; - Arena TokMem; - Arena CodeMem; - - FileContents FileContent; - Array Tokens; - Array Errors; - // Errors are allocated to a dedicated general arena. + ParseMessage* messages; + LexedInfo lexed; + Code result; }; -CodeBody parse_file( Str path ); -#endif +struct Opts_parse +{ + AllocatorInfo backing_msgs; + AllocatorInfo backing_tokens; + AllocatorInfo backing_ast; +}; + +ParseInfo wip_parse_str( LexedInfo lexed, Opts_parse opts GEN_PARAM_DEFAULT ); GEN_API CodeClass parse_class ( Str class_def ); GEN_API CodeConstructor parse_constructor ( Str constructor_def ); diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index d5f4bef..cad87eb 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -8,6 +8,29 @@ // Publically Exposed Interface +ParseInfo wip_parse_str(LexedInfo lexed, Opts_parse opts) +{ + TokArray toks; + if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) { + check_parse_args(lexed.text); + toks = lex(lexed.text); + + TokenSlice slice = { toks.Arr, scast(s32, array_num(toks.Arr)) }; + lexed.tokens = slice; + } + ParseInfo info = struct_zero(ParseInfo); + info.lexed = lexed; + + // TODO(Ed): ParseInfo should be set to the parser context. + + _ctx->parser.Tokens = toks; + push_scope(); + CodeBody result = parse_global_nspace(CT_Global_Body); + parser_pop(& _ctx->parser); + + return info; +} + CodeClass parse_class( Str def ) { check_parse_args( def ); diff --git a/base/components/parser.cpp b/base/components/parser.cpp index b685406..709723d 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -11,7 +11,7 @@ constexpr bool lex_dont_skip_formatting = false; constexpr bool lex_skip_formatting = true; -void parser_push( ParseContext* ctx, StackNode* node ) +void parser_push( ParseContext* ctx, ParseStackNode* node ) { node->Prev = ctx->Scope; ctx->Scope = node; @@ -60,7 +60,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) else strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column ); - StackNode* curr_scope = ctx.Scope; + ParseStackNode* curr_scope = ctx.Scope; s32 level = 0; do { @@ -181,9 +181,10 @@ bool _check_parse_args( Str def, char const* func_name ) # define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) # define check( Type_ ) ( left && currtok.Type == Type_ ) -# define push_scope() \ - Str null_name = {}; \ - StackNode scope = { nullptr, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \ +// TODO(Ed): Don't do this anymore, we need a better initializer. +# define push_scope() \ + Str null_name = {}; \ + ParseStackNode scope = { nullptr, {nullptr, 0}, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ), { nullptr} }; \ parser_push( & _ctx->parser, & scope ) #pragma endregion Helper Macros diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index a2bd57a..8969740 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -97,6 +97,12 @@ struct TokArray s32 Idx; }; +struct TokenSlice +{ + Token* Ptr; + s32 Num; +}; + struct LexContext { Str content; @@ -108,19 +114,18 @@ struct LexContext Token token; }; -struct StackNode +struct LexedInfo { - StackNode* Prev; - - Token* Start; - Str Name; // The name of the AST node (if parsed) - Str ProcName; // The name of the procedure + Str text; + TokenSlice tokens; }; +typedef struct ParseStackNode ParseStackNode; + struct ParseContext { - TokArray Tokens; - StackNode* Scope; + TokArray Tokens; + ParseStackNode* Scope; }; enum MacroType : u16 diff --git a/gen_c_library/c_library.refactor b/gen_c_library/c_library.refactor index 96021a1..4c3da39 100644 --- a/gen_c_library/c_library.refactor +++ b/gen_c_library/c_library.refactor @@ -535,7 +535,7 @@ namespace Lexer_, gen_Lexer_ word LexContext, gen_LexContext word lex, gen_lex -word StackNode, gen_StackNode +word ParseStackNode, gen_ParseStackNode word ParseContext, gen_ParseContext // namespace parse_, gen_parse_ From 6d531fdf97806eb09c6775c02cc661b267433fec Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 11:08:24 -0500 Subject: [PATCH 06/19] Fixing intellisense --- base/auxiliary/builder.hpp | 2 +- base/auxiliary/scanner.hpp | 2 +- base/components/ast.hpp | 4 +++- base/components/interface.hpp | 4 ++-- base/components/interface.parsing.cpp | 2 +- base/components/lexer.cpp | 2 +- base/components/parser.cpp | 2 +- base/components/parser_types.hpp | 3 +-- base/dependencies/debug.hpp | 5 ++--- base/dependencies/printing.cpp | 2 +- base/dependencies/printing.hpp | 2 +- base/dependencies/strings.hpp | 2 ++ base/gen.hpp | 3 --- 13 files changed, 17 insertions(+), 18 deletions(-) diff --git a/base/auxiliary/builder.hpp b/base/auxiliary/builder.hpp index 65d4d5b..f8c18c7 100644 --- a/base/auxiliary/builder.hpp +++ b/base/auxiliary/builder.hpp @@ -3,7 +3,7 @@ # include "helpers/push_ignores.inline.hpp" # include "components/header_start.hpp" # include "components/types.hpp" -# include "components/gen/ecode.hpp" +# include "components/gen/ecodetypes.hpp" # include "components/gen/eoperator.hpp" # include "components/gen/especifier.hpp" # include "components/ast.hpp" diff --git a/base/auxiliary/scanner.hpp b/base/auxiliary/scanner.hpp index 72a57b3..fbeaf11 100644 --- a/base/auxiliary/scanner.hpp +++ b/base/auxiliary/scanner.hpp @@ -3,7 +3,7 @@ # include "helpers/push_ignores.inline.hpp" # include "components/header_start.hpp" # include "components/types.hpp" -# include "components/gen/ecode.hpp" +# include "components/gen/ecodetypes.hpp" # include "components/gen/eoperator.hpp" # include "components/gen/especifier.hpp" # include "components/ast.hpp" diff --git a/base/components/ast.hpp b/base/components/ast.hpp index 0041456..bcac8ae 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -1,7 +1,9 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once +#include "basic_types.hpp" +#include "strings.hpp" #include "types.hpp" -#include "gen/ecode.hpp" +#include "gen/ecodetypes.hpp" #include "gen/eoperator.hpp" #include "gen/especifier.hpp" #endif diff --git a/base/components/interface.hpp b/base/components/interface.hpp index bf74f1c..97e14f2 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -395,11 +395,11 @@ struct ParseStackNode { ParseStackNode* Prev; - TokenSlice tokens; + TokenSlice Tokens; Token* Start; Str Name; // The name of the AST node (if parsed) Str ProcName; // The name of the procedure - Code code; // Relevant AST node + Code CodeRel; // Relevant AST node // TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling. }; diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index cad87eb..69acb69 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #include "interface.upfront.cpp" #include "lexer.cpp" #include "parser.cpp" diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index f501f70..6f4ca55 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -1,7 +1,7 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once #include "interface.upfront.cpp" -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #endif StrBuilder tok_to_strbuilder(Token tok) diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 709723d..d470021 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #include "parser_case_macros.cpp" #include "interface.upfront.cpp" #include "lexer.cpp" diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 8969740..fb4fc9a 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -1,7 +1,7 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once #include "types.hpp" -#include "gen/ecode.hpp" +#include "gen/ecodetypes.hpp" #include "gen/eoperator.hpp" #include "gen/especifier.hpp" #include "gen/etoktype.hpp" @@ -110,7 +110,6 @@ struct LexContext char const* scanner; s32 line; s32 column; - // StringTable defines; Token token; }; diff --git a/base/dependencies/debug.hpp b/base/dependencies/debug.hpp index cb2970a..90a8334 100644 --- a/base/dependencies/debug.hpp +++ b/base/dependencies/debug.hpp @@ -1,9 +1,8 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once -# include "dependencies/platform.hpp" -# include "dependencies/macros.hpp" -# include "basic_types.hpp" +# include "platform.hpp" # include "macros.hpp" +# include "basic_types.hpp" #endif #pragma region Debug diff --git a/base/dependencies/printing.cpp b/base/dependencies/printing.cpp index 684f0ea..1e11be5 100644 --- a/base/dependencies/printing.cpp +++ b/base/dependencies/printing.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once -# include "strbuilder_ops.cpp" +# include "string_ops.cpp" #endif #pragma region Printing diff --git a/base/dependencies/printing.hpp b/base/dependencies/printing.hpp index 2550254..be5754c 100644 --- a/base/dependencies/printing.hpp +++ b/base/dependencies/printing.hpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once -# include "strbuilder_ops.hpp" +# include "string_ops.hpp" #endif #pragma region Printing diff --git a/base/dependencies/strings.hpp b/base/dependencies/strings.hpp index 866e944..69cead9 100644 --- a/base/dependencies/strings.hpp +++ b/base/dependencies/strings.hpp @@ -1,5 +1,7 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once +# include "basic_types.hpp" +# include "memory.hpp" # include "hashing.hpp" #endif diff --git a/base/gen.hpp b/base/gen.hpp index 8d3d5a9..33acd5b 100644 --- a/base/gen.hpp +++ b/base/gen.hpp @@ -11,9 +11,6 @@ #include "helpers/push_ignores.inline.hpp" #include "components/header_start.hpp" -// Has container defines pushed -#include "gen.dep.hpp" - GEN_NS_BEGIN #include "components/types.hpp" From 7ea90ef3490e744b83e575c88694b8851f342ebc Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 11:22:00 -0500 Subject: [PATCH 07/19] intellisense still broken... --- base/components/ast.hpp | 7 +------ base/components/gen/ast_inlines.hpp | 2 +- base/components/inlines.hpp | 2 +- base/components/static_data.cpp | 2 +- base/components/types.hpp | 14 +++++++++++++- base/dependencies/debug.hpp | 2 -- base/dependencies/strings.hpp | 2 -- 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/base/components/ast.hpp b/base/components/ast.hpp index bcac8ae..13c679e 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -1,11 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "basic_types.hpp" -#include "strings.hpp" -#include "types.hpp" -#include "gen/ecodetypes.hpp" -#include "gen/eoperator.hpp" -#include "gen/especifier.hpp" +#include "parser_types.hpp" #endif /* diff --git a/base/components/gen/ast_inlines.hpp b/base/components/gen/ast_inlines.hpp index 7ec01aa..2ccb518 100644 --- a/base/components/gen/ast_inlines.hpp +++ b/base/components/gen/ast_inlines.hpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "components/types.hpp" +#include "inlines.hpp" #endif // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index 45e572c..4489516 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "interface.hpp" +#include "constants.hpp" #endif #pragma region Serialization diff --git a/base/components/static_data.cpp b/base/components/static_data.cpp index 696b5e2..81c21a6 100644 --- a/base/components/static_data.cpp +++ b/base/components/static_data.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "../gen.hpp" +#include "interface.hpp" #endif #pragma region StaticData diff --git a/base/components/types.hpp b/base/components/types.hpp index e64a706..73a0aaa 100644 --- a/base/components/types.hpp +++ b/base/components/types.hpp @@ -1,6 +1,18 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "header_start.hpp" +#include "dependencies/platform.hpp" +#include "dependencies/macros.hpp" +#include "dependencies/basic_types.hpp" +#include "dependencies/debug.hpp" +#include "dependencies/memory.hpp" +#include "dependencies/string_ops.hpp" +#include "dependencies/printing.hpp" +#include "dependencies/containers.hpp" +#include "dependencies/hashing.hpp" +#include "dependencies/strings.hpp" +#include "dependencies/filesystem.hpp" +#include "dependencies/timing.hpp" +#include "dependencies/parsing.hpp" #endif /* diff --git a/base/dependencies/debug.hpp b/base/dependencies/debug.hpp index 90a8334..6efbeb4 100644 --- a/base/dependencies/debug.hpp +++ b/base/dependencies/debug.hpp @@ -1,7 +1,5 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once -# include "platform.hpp" -# include "macros.hpp" # include "basic_types.hpp" #endif diff --git a/base/dependencies/strings.hpp b/base/dependencies/strings.hpp index 69cead9..866e944 100644 --- a/base/dependencies/strings.hpp +++ b/base/dependencies/strings.hpp @@ -1,7 +1,5 @@ #ifdef INTELLISENSE_DIRECTIVES # pragma once -# include "basic_types.hpp" -# include "memory.hpp" # include "hashing.hpp" #endif From 26623075add09f91e3dde411df6533f0cab37983 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 12:10:11 -0500 Subject: [PATCH 08/19] started to work on modularizing parser code paths --- base/components/ast.hpp | 2 +- base/components/gen/ast_inlines.hpp | 2 +- base/components/interface.hpp | 19 ++++++++++++------- base/components/interface.parsing.cpp | 2 +- base/components/interface.untyped.cpp | 13 +++++++++++++ base/components/lexer.cpp | 5 ++++- base/components/parser.cpp | 25 ++++++++++++++++++------- base/components/parser_types.hpp | 27 +++++++++++++++++++++------ base/dependencies/macros.hpp | 2 +- base/dependencies/strings.hpp | 2 +- 10 files changed, 73 insertions(+), 26 deletions(-) diff --git a/base/components/ast.hpp b/base/components/ast.hpp index 13c679e..ae2513e 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -404,7 +404,7 @@ struct AST }; }; StrCached Content; // Attributes, Comment, Execution, Include - // TokenSlice Content; // TODO(Ed): Use a token slice for content + TokenSlice ContentToks; // TODO(Ed): Use a token slice for content struct { Specifier ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers Code NextSpecs; // Specifiers; If ArrSpecs is full, then NextSpecs is used. diff --git a/base/components/gen/ast_inlines.hpp b/base/components/gen/ast_inlines.hpp index 2ccb518..7ec01aa 100644 --- a/base/components/gen/ast_inlines.hpp +++ b/base/components/gen/ast_inlines.hpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "inlines.hpp" +#include "components/types.hpp" #endif // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 97e14f2..29495d6 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -15,7 +15,7 @@ \▓▓▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ */ -enum LogLevel : u32 +enum LogLevel //: u32 { LL_Null, LL_Note, @@ -24,14 +24,17 @@ enum LogLevel : u32 LL_Fatal, LL_UnderlyingType = GEN_U32_MAX, }; +typedef enum LogLevel LogLevel; Str loglevel_to_str(LogLevel level) { local_persist Str lookup[] = { - txt("Info"), - txt("Warning"), - txt("Panic"), + { "Null", sizeof("Null") - 1 }, + { "Note", sizeof("Note") - 1 }, + { "Warning", sizeof("Info") - 1 }, + { "Error", sizeof("Error") - 1 }, + { "Fatal", sizeof("Fatal") - 1 }, }; return lookup[level]; } @@ -148,7 +151,7 @@ inline void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...) { local_persist thread_local - PrintF_Buffer buf = struct_init(PrintF_Buffer, {0}); + PrintF_Buffer buf = struct_zero(PrintF_Buffer); va_list va; va_start(va, fmt); @@ -403,6 +406,7 @@ struct ParseStackNode // TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling. }; +typedef struct ParseMessage ParseMessage; struct ParseMessage { ParseMessage* Next; @@ -418,14 +422,14 @@ struct ParseInfo Code result; }; -struct Opts_parse +struct ParseOpts { AllocatorInfo backing_msgs; AllocatorInfo backing_tokens; AllocatorInfo backing_ast; }; -ParseInfo wip_parse_str( LexedInfo lexed, Opts_parse opts GEN_PARAM_DEFAULT ); +ParseInfo wip_parse_str( LexedInfo lexed, ParseOpts* opts GEN_PARAM_DEFAULT ); GEN_API CodeClass parse_class ( Str class_def ); GEN_API CodeConstructor parse_constructor ( Str constructor_def ); @@ -459,6 +463,7 @@ Str token_fmt_impl( ssize, ... ); GEN_API Code untyped_str ( Str content); GEN_API Code untyped_fmt ( char const* fmt, ... ); GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ); +GEN_API Code untyped_toks ( TokenSlice tokens ); #pragma endregion Untyped text diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index 69acb69..0c10c39 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -8,7 +8,7 @@ // Publically Exposed Interface -ParseInfo wip_parse_str(LexedInfo lexed, Opts_parse opts) +ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts) { TokArray toks; if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) { diff --git a/base/components/interface.untyped.cpp b/base/components/interface.untyped.cpp index 69d74bb..3e05369 100644 --- a/base/components/interface.untyped.cpp +++ b/base/components/interface.untyped.cpp @@ -176,3 +176,16 @@ Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ) return result; } + +Code untyped_toks( TokenSlice tokens ) +{ + if ( tokens.Num == 0 ) { + log_failure( "untyped_toks: empty token slice" ); + return InvalidCode; + } + Code + result = make_code(); + result->Type = CT_Untyped; + result->ContentToks = tokens; + return result; +} diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 6f4ca55..0fc46b8 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -564,9 +564,12 @@ void lex_found_token( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, ctx->token ); } +// TODO(Ed): We should dynamically allocate the lexer's array in Allocator_DyanmicContainers. + // TODO(Ed): We need to to attempt to recover from a lex failure? + neverinline -// TokArray lex( Array tokens, Str content ) +// void lex( Array tokens, Str content ) TokArray lex( Str content ) { LexContext c; LexContext* ctx = & c; diff --git a/base/components/parser.cpp b/base/components/parser.cpp index d470021..edd5e08 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -191,7 +191,7 @@ bool _check_parse_args( Str def, char const* func_name ) // Procedure Forwards ( Entire parser internal parser interface ) -internal Code parse_array_decl (); +internal Code parse_array_decl (ParseContext* ctx); internal CodeAttributes parse_attributes (); internal CodeComment parse_comment (); internal Code parse_complicated_definition ( TokType which ); @@ -491,8 +491,15 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines ) return content; } +StrBuilder parser_strip_formatting_2(TokenSlice tokens) +{ + // TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content. + StrBuilder result = struct_zero(StrBuilder); + return result; +} + internal -Code parse_array_decl() +Code parse_array_decl(ParseContext* ctx) { push_scope(); @@ -525,16 +532,20 @@ Code parse_array_decl() return InvalidCode; } + TokenSlice tokens = { & currtok, 1 }; Token untyped_tok = currtok; while ( left && currtok.Type != Tok_BraceSquare_Close ) { eat( currtok.Type ); + ++ tokens.Num; } - untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; + // untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; + untyped_tok.Text = token_range_to_str(untyped_tok, prevtok); Code array_expr = untyped_str( untyped_tok.Text ); + // Code array_expr = untyped_toks( tokens ); // TODO(Ed): Use token slice instead of untyped strings. // [ if ( left == 0 ) @@ -557,7 +568,7 @@ Code parse_array_decl() // Its a multi-dimensional array if ( check( Tok_BraceSquare_Open )) { - Code adjacent_arr_expr = parse_array_decl(); + Code adjacent_arr_expr = parse_array_decl(ctx); // [ ][ ]... array_expr->Next = adjacent_arr_expr; @@ -3291,7 +3302,7 @@ CodeVar parse_variable_after_name( { push_scope(); - Code array_expr = parse_array_decl(); + Code array_expr = parse_array_decl(& _ctx->parser); Code expr = NullCode; Code bitfield_expr = NullCode; @@ -5345,7 +5356,7 @@ CodeTypedef parser_parse_typedef() return InvalidCode; } - array_expr = parse_array_decl(); + array_expr = parse_array_decl(& _ctx->parser); // + } @@ -5591,7 +5602,7 @@ CodeUsing parser_parse_using() type = parser_parse_type(parser_not_from_template, nullptr); // using = - array_expr = parse_array_decl(); + array_expr = parse_array_decl(& _ctx->parser); // + } } diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index fb4fc9a..4308316 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -91,16 +91,31 @@ bool tok_is_end_definition(Token tok) { StrBuilder tok_to_strbuilder(Token tok); -struct TokArray -{ - Array(Token) Arr; - s32 Idx; -}; - struct TokenSlice { Token* Ptr; s32 Num; + +#if GEN_COMPILER_CPP + forceinline operator Token* () const { return Ptr; } + forceinline Token& operator[]( ssize index ) const { return Ptr[index]; } +#endif +}; + +forceinline +Str token_range_to_str(Token start, Token end) +{ + Str result = { + start.Text.Ptr, + (scast(sptr, rcast(uptr, end.Text.Ptr)) + end.Text.Len) - scast(sptr, rcast(uptr, start.Text.Ptr)) + }; + return result; +} + +struct TokArray +{ + Array(Token) Arr; + s32 Idx; }; struct LexContext diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index ec2106c..69abbc7 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -310,7 +310,7 @@ # if GEN_COMPILER_CPP # define struct_zero(type) {} # else -# define struct_zero(type) (type) {0} +# define struct_zero(type) {0} # endif #endif diff --git a/base/dependencies/strings.hpp b/base/dependencies/strings.hpp index 866e944..eaf2e5e 100644 --- a/base/dependencies/strings.hpp +++ b/base/dependencies/strings.hpp @@ -320,7 +320,7 @@ inline StrBuilder strbuilder_fmt_buf(AllocatorInfo allocator, char const* fmt, ...) { local_persist thread_local - PrintF_Buffer buf = struct_init(PrintF_Buffer, {0}); + PrintF_Buffer buf = struct_zero(PrintF_Buffer); va_list va; va_start(va, fmt); From 441a46daaa8d8259f4634c476db6ec76bd876434 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 19 Feb 2025 15:48:56 -0500 Subject: [PATCH 09/19] changing to just pass lib Context instead of ParseContext. If I want to make it more granular I can later. --- base/components/interface.parsing.cpp | 1 + base/components/parser.cpp | 128 ++++++++++++++------------ 2 files changed, 69 insertions(+), 60 deletions(-) diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index 0c10c39..249a0d2 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -349,6 +349,7 @@ CodeVar parse_variable( Str def ) #undef left #undef check #undef push_scope +#undef NullScope #undef def_assign // Here for C Variant diff --git a/base/components/parser.cpp b/base/components/parser.cpp index edd5e08..fb9c4ef 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -185,15 +185,21 @@ bool _check_parse_args( Str def, char const* func_name ) # define push_scope() \ Str null_name = {}; \ ParseStackNode scope = { nullptr, {nullptr, 0}, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ), { nullptr} }; \ - parser_push( & _ctx->parser, & scope ) + parser_push( & _ctx->parser, & scope ) + +#if GEN_COMPILER_CPP +# define NullScope { nullptr, {nullptr, 0}, lex_current( & ctx->parser.Tokens, lex_dont_skip_formatting ), Str{nullptr, 0}, txt( __func__ ), { nullptr} } +#else +# define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( & ctx->parser.Tokens, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} } +#endif #pragma endregion Helper Macros // Procedure Forwards ( Entire parser internal parser interface ) -internal Code parse_array_decl (ParseContext* ctx); -internal CodeAttributes parse_attributes (); -internal CodeComment parse_comment (); +internal Code parse_array_decl (Context* ctx); +internal CodeAttributes parse_attributes (Context* ctx); +internal CodeComment parse_comment (Context* ctx); internal Code parse_complicated_definition ( TokType which ); internal CodeBody parse_class_struct_body ( TokType which, Token name ); internal Code parse_class_struct ( TokType which, bool inplace_def ); @@ -499,9 +505,10 @@ StrBuilder parser_strip_formatting_2(TokenSlice tokens) } internal -Code parse_array_decl(ParseContext* ctx) +Code parse_array_decl(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); if ( check( Tok_Operator ) && currtok.Text.Ptr[0] == '[' && currtok.Text.Ptr[1] == ']' ) { @@ -509,7 +516,7 @@ Code parse_array_decl(ParseContext* ctx) eat( Tok_Operator ); // [] - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return array_expr; } @@ -520,15 +527,15 @@ Code parse_array_decl(ParseContext* ctx) if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type == Tok_BraceSquare_Close ) { - log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -550,15 +557,15 @@ Code parse_array_decl(ParseContext* ctx) if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type != Tok_BraceSquare_Close ) { - log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -574,18 +581,19 @@ Code parse_array_decl(ParseContext* ctx) array_expr->Next = adjacent_arr_expr; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return array_expr; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return NullCode; } internal inline -CodeAttributes parse_attributes() +CodeAttributes parse_attributes(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); Token start = currtok; s32 len = 0; @@ -676,7 +684,7 @@ CodeAttributes parse_attributes() if ( len > 0 ) { Str attribute_txt = { start.Text.Ptr, len }; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); StrBuilder name_stripped = parser_strip_formatting( attribute_txt, parser_strip_formatting_dont_preserve_newlines ); @@ -685,11 +693,10 @@ CodeAttributes parse_attributes() result->Name = cache_str( strbuilder_to_str(name_stripped) ); result->Content = result->Name; // result->Token = - return ( CodeAttributes )result; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return NullCode; } @@ -720,7 +727,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( which ); // - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // if ( check( Tok_Identifier ) ) { @@ -787,7 +794,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) // : , ... { }; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // : , ... { }; } @@ -848,7 +855,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Comment: { - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(_ctx)); break; } case Tok_Access_Public: { @@ -1007,7 +1014,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // } //! Fallthrough intended @@ -1059,7 +1066,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( tok_is_attribute(currtok) ) { // Unfortuantely Unreal has code where there is attirbutes before specifiers - CodeAttributes more_attributes = parse_attributes(); + CodeAttributes more_attributes = parse_attributes(_ctx); if ( attributes ) { @@ -1147,9 +1154,10 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } internal -CodeComment parse_comment() -{ - push_scope(); +CodeComment parse_comment(Context* ctx) +{ + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); CodeComment result = (CodeComment) make_code(); @@ -1158,7 +1166,7 @@ CodeComment parse_comment() // result->Token = currtok_noskip; eat( Tok_Comment ); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } @@ -1456,7 +1464,7 @@ CodeFn parse_function_after_name( // ( ) // Check for trailing specifiers... - CodeAttributes post_rt_attributes = parse_attributes(); + CodeAttributes post_rt_attributes = parse_attributes(_ctx); if (post_rt_attributes) { if (attributes) @@ -1508,7 +1516,7 @@ CodeFn parse_function_after_name( eat( Tok_Statement_End ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // ( ) < = 0 or delete > ; } @@ -1520,7 +1528,7 @@ CodeFn parse_function_after_name( // ( ) < = 0 or delete > ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // ( ) < = 0 or delete > ; } @@ -1674,7 +1682,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Comment: - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(_ctx)); break; case Tok_Decl_Class: @@ -1810,7 +1818,7 @@ CodeBody parse_global_nspace( CodeType which ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // } //! Fallthrough intentional @@ -2555,7 +2563,7 @@ CodeOperator parse_operator_after_ret_type( // operator ( ) ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // operator ( ) ; } @@ -2586,7 +2594,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes // // Thanks Unreal - CodeAttributes post_rt_attributes = parse_attributes(); + CodeAttributes post_rt_attributes = parse_attributes(_ctx); if (post_rt_attributes) { if (attributes) @@ -3302,7 +3310,7 @@ CodeVar parse_variable_after_name( { push_scope(); - Code array_expr = parse_array_decl(& _ctx->parser); + Code array_expr = parse_array_decl(_ctx); Code expr = NullCode; Code bitfield_expr = NullCode; @@ -3412,7 +3420,7 @@ CodeVar parse_variable_after_name( // Check for inline comment : = ; // if ( left && ( currtok_noskip.Type == Tok_Comment ) && currtok_noskip.Line == stmt_end.Line ) { - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // : , ...; // = , ...; // { }, ...; @@ -3596,7 +3604,7 @@ CodeConstructor parser_parse_constructor( CodeSpecifiers specifiers ) // ( ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // ( ); } @@ -3798,7 +3806,7 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) // ~() ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // ~() ; } @@ -3852,7 +3860,7 @@ CodeEnum parser_parse_enum( bool inplace_def ) // enum class } - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // enum if ( check( Tok_Identifier ) ) @@ -3922,7 +3930,7 @@ CodeEnum parser_parse_enum( bool inplace_def ) break; case Tok_Comment: - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(_ctx)); break; case Tok_Preprocess_Define: @@ -4037,7 +4045,7 @@ CodeEnum parser_parse_enum( bool inplace_def ) // enum : { }; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // enum : { }; } @@ -4215,7 +4223,7 @@ CodeFriend parser_parse_friend() // friend ( ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // friend ; // friend ( ); } @@ -4255,7 +4263,7 @@ CodeFn parser_parse_function() } // - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // while ( left && tok_is_specifier(currtok) ) @@ -4356,7 +4364,7 @@ CodeOperator parser_parse_operator() } // - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // while ( left && tok_is_specifier(currtok) ) @@ -4484,7 +4492,7 @@ CodeOpCast parser_parse_operator_cast( CodeSpecifiers specifiers ) // :: ... operator () ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // :: ... operator () ; } @@ -4587,7 +4595,7 @@ CodeTemplate parser_parse_template() Specifier specs_found[ 16 ] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // template< > // Specifiers Parsing @@ -4718,7 +4726,7 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function ) ETypenameTag tag = Tag_None; // Attributes are assumed to be before the type signature - CodeAttributes attributes = parse_attributes(); + CodeAttributes attributes = parse_attributes(_ctx); // // Prefix specifiers @@ -5356,7 +5364,7 @@ CodeTypedef parser_parse_typedef() return InvalidCode; } - array_expr = parse_array_decl(& _ctx->parser); + array_expr = parse_array_decl(_ctx); // + } @@ -5366,7 +5374,7 @@ CodeTypedef parser_parse_typedef() CodeComment inline_cmt = NullCode; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); // typedef ; CodeTypedef @@ -5419,7 +5427,7 @@ CodeUnion parser_parse_union( bool inplace_def ) eat( Tok_Decl_Union ); // union - CodeAttributes attributes = parse_attributes(); + CodeAttributes attributes = parse_attributes(_ctx); // union Str name = { nullptr, 0 }; @@ -5455,7 +5463,7 @@ CodeUnion parser_parse_union( bool inplace_def ) break; case Tok_Comment: - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(_ctx)); break; // TODO(Ed) : Unions can have constructors and destructors @@ -5588,12 +5596,12 @@ CodeUsing parser_parse_using() if ( ! is_namespace ) { - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // using if ( bitfield_is_set( u32, currtok.Flags, TF_Assign )) { - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // using eat( Tok_Operator ); @@ -5602,7 +5610,7 @@ CodeUsing parser_parse_using() type = parser_parse_type(parser_not_from_template, nullptr); // using = - array_expr = parse_array_decl(& _ctx->parser); + array_expr = parse_array_decl(_ctx); // + } } @@ -5613,7 +5621,7 @@ CodeUsing parser_parse_using() CodeComment inline_cmt = NullCode; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) { - inline_cmt = parse_comment(); + inline_cmt = parse_comment(_ctx); } // using = ; @@ -5665,7 +5673,7 @@ CodeVar parser_parse_variable() } // - attributes = parse_attributes(); + attributes = parse_attributes(_ctx); // while ( left && tok_is_specifier(currtok) ) From 790087aa3ca4e9dd9db4b45d41117d787495351b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 16 Mar 2025 23:13:46 -0400 Subject: [PATCH 10/19] progress on modularizing parser paths --- base/components/interface.parsing.cpp | 8 +- base/components/parser.cpp | 444 ++++++++++++++------------ base/dependencies/containers.hpp | 12 +- 3 files changed, 244 insertions(+), 220 deletions(-) diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index 249a0d2..70166f3 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -25,7 +25,7 @@ ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts) _ctx->parser.Tokens = toks; push_scope(); - CodeBody result = parse_global_nspace(CT_Global_Body); + CodeBody result = parse_global_nspace(_ctx, CT_Global_Body); parser_pop(& _ctx->parser); return info; @@ -41,7 +41,7 @@ CodeClass parse_class( Str def ) _ctx->parser.Tokens = toks; push_scope(); - CodeClass result = (CodeClass) parse_class_struct( Tok_Decl_Class, parser_not_inplace_def ); + CodeClass result = (CodeClass) parse_class_struct( _ctx, Tok_Decl_Class, parser_not_inplace_def ); parser_pop(& _ctx->parser); return result; } @@ -209,7 +209,7 @@ CodeBody parse_global_body( Str def ) _ctx->parser.Tokens = toks; push_scope(); - CodeBody result = parse_global_nspace( CT_Global_Body ); + CodeBody result = parse_global_nspace(_ctx, CT_Global_Body ); parser_pop(& _ctx->parser); return result; } @@ -260,7 +260,7 @@ CodeStruct parse_struct( Str def ) _ctx->parser.Tokens = toks; push_scope(); - CodeStruct result = (CodeStruct) parse_class_struct( Tok_Decl_Struct, parser_not_inplace_def ); + CodeStruct result = (CodeStruct) parse_class_struct( _ctx, Tok_Decl_Struct, parser_not_inplace_def ); parser_pop(& _ctx->parser); return result; } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index fb9c4ef..a2b3d68 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -200,19 +200,19 @@ bool _check_parse_args( Str def, char const* func_name ) internal Code parse_array_decl (Context* ctx); internal CodeAttributes parse_attributes (Context* ctx); internal CodeComment parse_comment (Context* ctx); -internal Code parse_complicated_definition ( TokType which ); -internal CodeBody parse_class_struct_body ( TokType which, Token name ); -internal Code parse_class_struct ( TokType which, bool inplace_def ); +internal Code parse_complicated_definition (Context* ctx, TokType which); +internal CodeBody parse_class_struct_body (Context* ctx, TokType which, Token name); +internal Code parse_class_struct (Context* ctx, TokType which, bool inplace_def); internal Code parse_expression (); -internal Code parse_forward_or_definition ( TokType which, bool is_inplace ); -internal CodeFn parse_function_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name ); -internal Code parse_function_body (); -internal CodeBody parse_global_nspace ( CodeType which ); -internal Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ); -internal Token parse_identifier ( bool* possible_member_function ); -internal CodeInclude parse_include (); -internal Code parse_macro_as_definiton ( CodeAttributes attributes, CodeSpecifiers specifiers ); -internal CodeOperator parse_operator_after_ret_type ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type ); +internal Code parse_forward_or_definition (Context* ctx, TokType which, bool is_inplace); +internal CodeFn parse_function_after_name (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name); +internal Code parse_function_body (Context* ctx); +internal CodeBody parse_global_nspace (Context* ctx, CodeType which ); +internal Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers); +internal Token parse_identifier (Context* ctx, bool* possible_member_function); +internal CodeInclude parse_include (Context* ctx); +internal Code parse_macro_as_definiton (Context* ctx, CodeAttributes attributes, CodeSpecifiers specifiers); +internal CodeOperator parse_operator_after_ret_type (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type ); internal Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ); internal CodePragma parse_pragma (); internal CodeParams parse_params ( bool use_template_capture ); @@ -701,10 +701,10 @@ CodeAttributes parse_attributes(Context* ctx) } internal -Code parse_class_struct( TokType which, bool inplace_def ) +Code parse_class_struct(Context* ctx, TokType which, bool inplace_def) { if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) { - log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(_ctx->parser) ); + log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(ctx->parser) ); return InvalidCode; } @@ -727,12 +727,12 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( which ); // - attributes = parse_attributes(_ctx); + attributes = parse_attributes(ctx); // if ( check( Tok_Identifier ) ) { - name = parse_identifier(nullptr); - _ctx->parser.Scope->Name = name.Text; + name = parse_identifier(ctx, nullptr); + ctx->parser.Scope->Name = name.Text; } // @@ -745,10 +745,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) local_persist char interface_arr_mem[ kilobytes(4) ] = {0}; - Array(CodeTypename) interfaces; { - Arena arena = arena_init_from_memory( interface_arr_mem, kilobytes(4) ); - interfaces = array_init_reserve(CodeTypename, arena_allocator_info(& arena), 4 ); - } + Array(CodeTypename) interfaces = {nullptr}; // TODO(Ed) : Make an AST_DerivedType, we'll store any arbitary derived type into there as a linear linked list of them. if ( check( Tok_Assign_Classifer ) ) @@ -762,27 +759,33 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( currtok.Type ); } - Token parent_tok = parse_identifier(nullptr); + Token parent_tok = parse_identifier(ctx, nullptr); parent = def_type( parent_tok.Text ); // : - while ( check(Tok_Comma) ) + if (check(Tok_Comma)) { - eat( Tok_Comma ); - // : , + Arena arena = arena_init_from_memory( interface_arr_mem, kilobytes(4) ); + interfaces = array_init_reserve(CodeTypename, arena_allocator_info(& arena), 4 ); + do + { + eat( Tok_Comma ); + // : , - if ( tok_is_access_specifier(currtok) ) { - eat(currtok.Type); - } - Token interface_tok = parse_identifier(nullptr); + if ( tok_is_access_specifier(currtok) ) { + eat(currtok.Type); + } + Token interface_tok = parse_identifier(ctx, nullptr); - array_append( interfaces, def_type( interface_tok.Text ) ); - // : , ... + array_append( interfaces, def_type( interface_tok.Text ) ); + // : , ... + } + while ( check(Tok_Comma) ); } } if ( check( Tok_BraceCurly_Open ) ) { - body = parse_class_struct_body( which, name ); + body = parse_class_struct_body( ctx, which, name ); } // : , ... { } @@ -794,27 +797,31 @@ Code parse_class_struct( TokType which, bool inplace_def ) // : , ... { }; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(_ctx); + inline_cmt = parse_comment(ctx); // : , ... { }; } + s32 num_interfaces = scast(s32, interfaces ? array_num(interfaces) : 0); + if ( which == Tok_Decl_Class ) - result = cast(Code, def_class( name.Text, def_assign( body, parent, access, attributes, interfaces, scast(s32, array_num(interfaces)), specifiers, mflags ) )); + result = cast(Code, def_class( name.Text, def_assign( body, parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); else - result = cast(Code, def_struct( name.Text, def_assign( body, (CodeTypename)parent, access, attributes, interfaces, scast(s32, array_num(interfaces)), specifiers, mflags ) )); + result = cast(Code, def_struct( name.Text, def_assign( body, (CodeTypename)parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); if ( inline_cmt ) result->InlineCmt = cast(Code, inline_cmt); - array_free(interfaces); + if (interfaces) + array_free(interfaces); return result; } internal neverinline -CodeBody parse_class_struct_body( TokType which, Token name ) +CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); // { @@ -824,7 +831,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( which == Tok_Decl_Class ) result->Type = CT_Class_Body; - else result->Type = CT_Struct_Body; @@ -836,7 +842,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) bool expects_function = false; - // _ctx->parser.Scope->Start = currtok_noskip; + // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -855,7 +861,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Comment: { - member = cast(Code, parse_comment(_ctx)); + member = cast(Code, parse_comment(ctx)); break; } case Tok_Access_Public: { @@ -880,12 +886,12 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Decl_Class: { - member = parse_complicated_definition( Tok_Decl_Class ); + member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; } case Tok_Decl_Enum: { - member = parse_complicated_definition( Tok_Decl_Enum ); + member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; } @@ -900,7 +906,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Decl_Struct: { - member = parse_complicated_definition( Tok_Decl_Struct ); + member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct break; } @@ -915,7 +921,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Decl_Union: { - member = parse_complicated_definition( Tok_Decl_Union ); + member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union break; } @@ -928,7 +934,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { //if ( currtok.Text[0] != '~' ) //{ - // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(_ctx->parser) ); + // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(ctx->parser) ); // return InvalidCode; //} @@ -943,7 +949,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } case Tok_Preprocess_Include: { - member = cast(Code, parse_include()); + member = cast(Code, parse_include(ctx)); // #include break; } @@ -1002,7 +1008,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { if ( ! tok_is_attribute(currtok)) { - log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(ctx->parser)); return InvalidCode; } } @@ -1014,7 +1020,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(_ctx); + attributes = parse_attributes(ctx); // } //! Fallthrough intended @@ -1043,8 +1049,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; default: - log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); - parser_pop(& _ctx->parser); + log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(ctx->parser)) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1066,11 +1072,11 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( tok_is_attribute(currtok) ) { // Unfortuantely Unreal has code where there is attirbutes before specifiers - CodeAttributes more_attributes = parse_attributes(_ctx); + CodeAttributes more_attributes = parse_attributes(ctx); if ( attributes ) { - StrBuilder fused = strbuilder_make_reserve( _ctx->Allocator_Temp, attributes->Content.Len + more_attributes->Content.Len ); + StrBuilder fused = strbuilder_make_reserve( ctx->Allocator_Temp, attributes->Content.Len + more_attributes->Content.Len ); strbuilder_append_fmt( & fused, "%SB %SB", attributes->Content, more_attributes->Content ); Str attrib_name = strbuilder_to_str(fused); @@ -1140,8 +1146,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Failed to parse member\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } body_append(result, member ); @@ -1149,7 +1155,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } @@ -1171,14 +1177,15 @@ CodeComment parse_comment(Context* ctx) } internal -Code parse_complicated_definition( TokType which ) +Code parse_complicated_definition(Context* ctx, TokType which) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); b32 is_inplace = false; b32 is_fn_def = false; - TokArray tokens = _ctx->parser.Tokens; + TokArray tokens = ctx->parser.Tokens; s32 idx = tokens.Idx; s32 level = 0; @@ -1206,16 +1213,16 @@ Code parse_complicated_definition( TokType which ) // Function definition with on return type Code result = parse_operator_function_or_variable(false, NullCode, NullCode); // (...) ... { ... } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } if ( ( idx - 2 ) == tokens.Idx ) { // It's a forward declaration only - Code result = parse_forward_or_definition( which, is_inplace ); + Code result = parse_forward_or_definition(ctx, which, is_inplace ); // ; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } @@ -1239,12 +1246,12 @@ Code parse_complicated_definition( TokType which ) Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // , or Name> ... - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( tok.Type == Tok_Identifier ) @@ -1260,14 +1267,14 @@ Code parse_complicated_definition( TokType which ) ok_to_parse = true; is_inplace = true; - CodeTypename type = cast(CodeTypename, parse_forward_or_definition(which, is_inplace)); + CodeTypename type = cast(CodeTypename, parse_forward_or_definition(ctx, which, is_inplace)); // Should be a name right after the type. - Token name = parse_identifier(nullptr); - _ctx->parser.Scope->Name = name.Text; + Token name = parse_identifier(ctx, nullptr); + ctx->parser.Scope->Name = name.Text; CodeVar result = parse_variable_after_name(ModuleFlag_None, NullCode, NullCode, type, name.Text); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return (Code) result; } else if ( tok.Type == Tok_Identifier && tokens.Arr[ idx - 3 ].Type == which ) @@ -1286,7 +1293,7 @@ Code parse_complicated_definition( TokType which ) // : ; ok_to_parse = true; Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( is_indirection ) @@ -1298,14 +1305,14 @@ Code parse_complicated_definition( TokType which ) if ( ! ok_to_parse ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // , or Name> ... - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( tok.Type >= Tok_Type_Unsigned && tok.Type <= Tok_Type_MS_W64 ) @@ -1317,8 +1324,8 @@ Code parse_complicated_definition( TokType which ) && ( tokens.Arr[idx - 4].Type != which)) ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1326,15 +1333,15 @@ Code parse_complicated_definition( TokType which ) // : ; // : ; Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceCurly_Close ) { // Its a definition - Code result = parse_forward_or_definition( which, is_inplace ); + Code result = parse_forward_or_definition(ctx, which, is_inplace ); // { ... }; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceSquare_Close ) @@ -1342,20 +1349,23 @@ Code parse_complicated_definition( TokType which ) // Its an array definition Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // [ ... ]; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else { - log_failure( "Unsupported or bad member definition after %s declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %s declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } } internal inline -Code parse_assignment_expression() +Code parse_assignment_expression(Context* ctx) { + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + Code expr = { nullptr }; eat( Tok_Operator ); @@ -1365,8 +1375,8 @@ Code parse_assignment_expression() if ( currtok.Type == Tok_Statement_End && currtok.Type != Tok_Comma ) { - log_failure( "Expected expression after assignment operator\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Expected expression after assignment operator\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1390,11 +1400,13 @@ Code parse_assignment_expression() } expr = untyped_str( expr_tok.Text ); // = + + parser_pop(& ctx->parser); return expr; } internal inline -Code parse_forward_or_definition( TokType which, bool is_inplace ) +Code parse_forward_or_definition(Context* ctx, TokType which, bool is_inplace ) { Code result = InvalidCode; @@ -1419,7 +1431,7 @@ Code parse_forward_or_definition( TokType which, bool is_inplace ) default: log_failure( "Error, wrong token type given to parse_complicated_definition " "(only supports class, enum, struct, union) \n%s" - , parser_to_strbuilder(_ctx->parser) ); + , parser_to_strbuilder(ctx->parser) ); return InvalidCode; } @@ -1427,15 +1439,17 @@ Code parse_forward_or_definition( TokType which, bool is_inplace ) // Function parsing is handled in multiple places because its initial signature is shared with variable parsing internal inline -CodeFn parse_function_after_name( - ModuleFlag mflags +CodeFn parse_function_after_name(Context* ctx + , ModuleFlag mflags , CodeAttributes attributes , CodeSpecifiers specifiers , CodeTypename ret_type , Token name ) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + CodeParams params = parse_params(parser_use_parenthesis); // ( ) @@ -1464,12 +1478,12 @@ CodeFn parse_function_after_name( // ( ) // Check for trailing specifiers... - CodeAttributes post_rt_attributes = parse_attributes(_ctx); + CodeAttributes post_rt_attributes = parse_attributes(ctx); if (post_rt_attributes) { if (attributes) { - StrBuilder merged = strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content); + StrBuilder merged = strbuilder_fmt_buf(ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content); attributes->Content = cache_str(strbuilder_to_str(merged)); } else @@ -1483,10 +1497,10 @@ CodeFn parse_function_after_name( CodeComment inline_cmt = NullCode; if ( check( Tok_BraceCurly_Open ) ) { - body = cast(CodeBody, parse_function_body()); + body = cast(CodeBody, parse_function_body(ctx)); if ( cast(Code, body) == Code_Invalid ) { - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return InvalidCode; } // ( ) { } @@ -1516,7 +1530,7 @@ CodeFn parse_function_after_name( eat( Tok_Statement_End ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(_ctx); + inline_cmt = parse_comment(ctx); // ( ) < = 0 or delete > ; } @@ -1528,12 +1542,12 @@ CodeFn parse_function_after_name( // ( ) < = 0 or delete > ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(_ctx); + inline_cmt = parse_comment(ctx); // ( ) < = 0 or delete > ; } StrBuilder - name_stripped = strbuilder_make_str( _ctx->Allocator_Temp, name.Text ); + name_stripped = strbuilder_make_str( ctx->Allocator_Temp, name.Text ); strbuilder_strip_space(name_stripped); CodeFn @@ -1551,8 +1565,8 @@ CodeFn parse_function_after_name( default: { - log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", code_debug_str(body), parser_to_strbuilder(_ctx->parser)); - parser_pop(& _ctx->parser); + log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", code_debug_str(body), parser_to_strbuilder(ctx->parser)); + parser_pop(& ctx->parser); return InvalidCode; } } @@ -1582,14 +1596,15 @@ CodeFn parse_function_after_name( if ( inline_cmt ) result->InlineCmt = inline_cmt; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal -Code parse_function_body() +Code parse_function_body(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); @@ -1624,14 +1639,15 @@ Code parse_function_body() eat( Tok_BraceCurly_Close ); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return cast(Code, result); } internal neverinline -CodeBody parse_global_nspace( CodeType which ) +CodeBody parse_global_nspace(Context* ctx, CodeType which) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); if ( which != CT_Namespace_Body && which != CT_Global_Body && which != CT_Export_Body && which != CT_Extern_Linkage_Body ) return InvalidCode; @@ -1652,7 +1668,7 @@ CodeBody parse_global_nspace( CodeType which ) bool expects_function = false; - // _ctx->parser.Scope->Start = currtok_noskip; + // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -1663,8 +1679,8 @@ CodeBody parse_global_nspace( CodeType which ) { case Tok_Comma: { - log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(_ctx->parser)); - parser_pop( & _ctx->parser); + log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(ctx->parser)); + parser_pop( & ctx->parser); return InvalidCode; } break; @@ -1682,22 +1698,22 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Comment: - member = cast(Code, parse_comment(_ctx)); + member = cast(Code, parse_comment(ctx)); break; case Tok_Decl_Class: - member = parse_complicated_definition( Tok_Decl_Class ); + member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; case Tok_Decl_Enum: - member = parse_complicated_definition( Tok_Decl_Enum ); + member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; case Tok_Decl_Extern_Linkage: if ( which == CT_Extern_Linkage_Body ) - log_failure( "Nested extern linkage\n%s", parser_to_strbuilder(_ctx->parser) ); + log_failure( "Nested extern linkage\n%s", parser_to_strbuilder(ctx->parser) ); member = cast(Code, parser_parse_extern_link()); // extern "..." { ... } @@ -1709,7 +1725,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Decl_Struct: - member = parse_complicated_definition( Tok_Decl_Struct ); + member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct ... break; @@ -1724,7 +1740,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Decl_Union: - member = parse_complicated_definition( Tok_Decl_Union ); + member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union ... break; @@ -1739,7 +1755,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Preprocess_Include: - member = cast(Code, parse_include()); + member = cast(Code, parse_include(ctx)); // #include ... break; @@ -1788,7 +1804,7 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Module_Export: { if ( which == CT_Export_Body ) - log_failure( "Nested export declaration\n%s", parser_to_strbuilder(_ctx->parser) ); + log_failure( "Nested export declaration\n%s", parser_to_strbuilder(ctx->parser) ); member = cast(Code, parser_parse_export_body()); // export { ... } @@ -1806,7 +1822,7 @@ CodeBody parse_global_nspace( CodeType which ) { if ( ! tok_is_attribute(currtok)) { - log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(ctx->parser)); return InvalidCode; } } @@ -1818,7 +1834,7 @@ CodeBody parse_global_nspace( CodeType which ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(_ctx); + attributes = parse_attributes(ctx); // } //! Fallthrough intentional @@ -1849,8 +1865,8 @@ CodeBody parse_global_nspace( CodeType which ) default: Str spec_str = spec_to_str(spec); - log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); - parser_pop(& _ctx->parser); + log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(ctx->parser)) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1883,7 +1899,7 @@ CodeBody parse_global_nspace( CodeType which ) { // This s only in a scope so that Preprocess_Macro_Bare_In_Body works without microsoft extension warnings { - Code constructor_destructor = parse_global_nspace_constructor_destructor( specifiers ); + Code constructor_destructor = parse_global_nspace_constructor_destructor( ctx, specifiers ); // Possible constructor implemented at global file scope. if ( constructor_destructor ) { @@ -1892,16 +1908,16 @@ CodeBody parse_global_nspace( CodeType which ) } bool found_operator_cast_outside_class_implmentation = false; - s32 idx = _ctx->parser.Tokens.Idx; + s32 idx = ctx->parser.Tokens.Idx; - for ( ; idx < array_num(_ctx->parser.Tokens.Arr); idx++ ) + for ( ; idx < array_num(ctx->parser.Tokens.Arr); idx++ ) { - Token tok = _ctx->parser.Tokens.Arr[ idx ]; + Token tok = ctx->parser.Tokens.Arr[ idx ]; if ( tok.Type == Tok_Identifier ) { idx++; - tok = _ctx->parser.Tokens.Arr[ idx ]; + tok = ctx->parser.Tokens.Arr[ idx ]; if ( tok.Type == Tok_Access_StaticSymbol ) continue; @@ -1930,7 +1946,7 @@ CodeBody parse_global_nspace( CodeType which ) if ( member == Code_Invalid ) { log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1942,13 +1958,16 @@ CodeBody parse_global_nspace( CodeType which ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal inline -Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) +Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers) { + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + Code result = { nullptr }; /* @@ -1963,7 +1982,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) TODO(Ed): We could fix this by attempting to parse a type, but we would have to have a way to have it soft fail and rollback. */ - TokArray tokens = _ctx->parser.Tokens; + TokArray tokens = ctx->parser.Tokens; s32 idx = tokens.Idx; Token nav = tokens.Arr[ idx ]; @@ -2007,6 +2026,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) if (tok_right.Type != Tok_Identifier) { + parser_pop(& ctx->parser); // We're not dealing with a constructor if there is no identifier right before the opening of a parameter's scope. return result; } @@ -2023,8 +2043,10 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) tok_left = tokens.Arr[idx]; } - if ( tok_left.Type != Tok_Access_StaticSymbol ) + if ( tok_left.Type != Tok_Access_StaticSymbol ) { + parser_pop(& ctx->parser); return result; + } -- idx; tok_left = tokens.Arr[idx]; @@ -2071,6 +2093,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) } } + parser_pop(& ctx->parser); return result; } @@ -2078,12 +2101,13 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) // This would allow distinction of the qualifier for a symbol :: // This would also allow internal -Token parse_identifier( bool* possible_member_function ) +Token parse_identifier(Context* ctx, bool* possible_member_function) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); Token name = currtok; - _ctx->parser.Scope->Name = name.Text; + ctx->parser.Scope->Name = name.Text; Macro* macro = lookup_macro(currtok.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); @@ -2114,23 +2138,23 @@ Token parse_identifier( bool* possible_member_function ) Token invalid = NullToken; if ( left == 0 ) { - log_failure( "Error, unexpected end of static symbol identifier\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, unexpected end of static symbol identifier\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return invalid; } if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '~' ) { - bool is_destructor = str_are_equal( _ctx->parser.Scope->Prev->ProcName, txt("parser_parse_destructor")); + bool is_destructor = str_are_equal( ctx->parser.Scope->Prev->ProcName, txt("parser_parse_destructor")); if (is_destructor) { name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return name; } - log_failure( "Error, had a ~ operator after %SB but not a destructor\n%s", toktype_to_str( prevtok.Type ), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, had a ~ operator after %SB but not a destructor\n%s", toktype_to_str( prevtok.Type ), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return invalid; } @@ -2141,16 +2165,16 @@ Token parse_identifier( bool* possible_member_function ) else { - log_failure( "Found a member function pointer identifier but the parsing context did not expect it\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Found a member function pointer identifier but the parsing context did not expect it\n%s", parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return invalid; } } if ( currtok.Type != Tok_Identifier ) { - log_failure( "Error, expected static symbol identifier, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, expected static symbol identifier, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(ctx->parser) ); + parser_pop(& ctx->parser); return invalid; } @@ -2163,14 +2187,15 @@ Token parse_identifier( bool* possible_member_function ) } //