diff --git a/.gitignore b/.gitignore index 893d2b0..3aaaa91 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ test/c_library/gen test/cpp_library/gen !scripts/helpers/refactor.exe + +# ai/** diff --git a/base/auxiliary/builder.hpp b/base/auxiliary/builder.hpp index 65d4d5b..f8c18c7 100644 --- a/base/auxiliary/builder.hpp +++ b/base/auxiliary/builder.hpp @@ -3,7 +3,7 @@ # include "helpers/push_ignores.inline.hpp" # include "components/header_start.hpp" # include "components/types.hpp" -# include "components/gen/ecode.hpp" +# include "components/gen/ecodetypes.hpp" # include "components/gen/eoperator.hpp" # include "components/gen/especifier.hpp" # include "components/ast.hpp" diff --git a/base/auxiliary/scanner.hpp b/base/auxiliary/scanner.hpp index 72a57b3..fbeaf11 100644 --- a/base/auxiliary/scanner.hpp +++ b/base/auxiliary/scanner.hpp @@ -3,7 +3,7 @@ # include "helpers/push_ignores.inline.hpp" # include "components/header_start.hpp" # include "components/types.hpp" -# include "components/gen/ecode.hpp" +# include "components/gen/ecodetypes.hpp" # include "components/gen/eoperator.hpp" # include "components/gen/especifier.hpp" # include "components/ast.hpp" diff --git a/base/components/ast.hpp b/base/components/ast.hpp index 2c9f9af..ae2513e 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -1,9 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "types.hpp" -#include "gen/ecode.hpp" -#include "gen/eoperator.hpp" -#include "gen/especifier.hpp" +#include "parser_types.hpp" #endif /* @@ -406,7 +403,8 @@ struct AST Code PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal) }; }; - StrCached Content; // Attributes, Comment, Execution, Include + StrCached Content; // Attributes, Comment, Execution, Include + TokenSlice ContentToks; // TODO(Ed): Use a token slice for content struct { Specifier ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers Code NextSpecs; // Specifiers; If ArrSpecs is full, then NextSpecs is used. @@ -422,7 +420,7 @@ struct AST Code Next; Code Back; }; - Token* Token; // Reference to starting token, only available if it was derived from parsing. + Token* Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice. Code Parent; CodeType Type; // CodeFlag CodeFlags; diff --git a/base/components/code_serialization.cpp b/base/components/code_serialization.cpp index 9ec168b..39dacb5 100644 --- a/base/components/code_serialization.cpp +++ b/base/components/code_serialization.cpp @@ -38,13 +38,13 @@ void body_to_strbuilder_export( CodeBody body, StrBuilder* result ) GEN_ASSERT(result != nullptr); strbuilder_append_fmt( result, "export\n{\n" ); - Code curr = cast(Code, body); + Code curr = body->Front; s32 left = body->NumEntries; while ( left-- ) { code_to_strbuilder_ref(curr, result); // strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) ); - ++curr; + curr = curr->Next; } strbuilder_append_fmt( result, "};\n" ); diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index bbf885b..59b8a77 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -253,7 +253,7 @@ struct CodeSpecifiers Using_Code( CodeSpecifiers ); bool append( Specifier spec ) { return specifiers_append(* this, spec); } bool has( Specifier spec ) { return specifiers_has(* this, spec); } - s32 index_of(Specifier spec) { return specifiers_index_of(* this, spec); } + s32 index_of( Specifier spec ) { return specifiers_index_of(* this, spec); } s32 remove( Specifier to_remove ) { return specifiers_remove(* this, to_remove); } StrBuilder to_strbuilder() { return specifiers_to_strbuilder(* this ); } void to_strbuilder( StrBuilder& result ) { return specifiers_to_strbuilder_ref(* this, & result); } @@ -1074,12 +1074,12 @@ forceinline bool has_entries (CodeParams params ) { forceinline StrBuilder to_strbuilder(CodeParams params ) { return params_to_strbuilder(params); } forceinline void to_strbuilder(CodeParams params, StrBuilder& result ) { return params_to_strbuilder_ref(params, & result); } -forceinline bool append (CodeSpecifiers specifiers, Specifier spec) { return specifiers_append(specifiers, spec); } -forceinline bool has (CodeSpecifiers specifiers, Specifier spec) { return specifiers_has(specifiers, spec); } -forceinline s32 index_of (CodeSpecifiers specifiers, Specifier spec) { return specifiers_index_of(specifiers, spec); } -forceinline s32 remove (CodeSpecifiers specifiers, Specifier to_remove ) { return specifiers_remove(specifiers, to_remove); } -forceinline StrBuilder to_strbuilder(CodeSpecifiers specifiers) { return specifiers_to_strbuilder(specifiers); } -forceinline void to_strbuilder(CodeSpecifiers specifiers, StrBuilder& result) { return specifiers_to_strbuilder_ref(specifiers, & result); } +forceinline bool append (CodeSpecifiers specifiers, Specifier spec) { return specifiers_append(specifiers, spec); } +forceinline bool has (CodeSpecifiers specifiers, Specifier spec) { return specifiers_has(specifiers, spec); } +forceinline s32 index_of (CodeSpecifiers specifiers, Specifier spec) { return specifiers_index_of(specifiers, spec); } +forceinline s32 remove (CodeSpecifiers specifiers, Specifier to_remove ) { return specifiers_remove(specifiers, to_remove); } +forceinline StrBuilder to_strbuilder (CodeSpecifiers specifiers) { return specifiers_to_strbuilder(specifiers); } +forceinline void to_strbuilder (CodeSpecifiers specifiers, StrBuilder& result) { return specifiers_to_strbuilder_ref(specifiers, & result); } forceinline void add_interface (CodeStruct self, CodeTypename interface) { return struct_add_interface(self, interface); } forceinline StrBuilder to_strbuilder (CodeStruct self) { return struct_to_strbuilder(self); } diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index 5bf6bf5..4489516 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "interface.hpp" +#include "constants.hpp" #endif #pragma region Serialization @@ -38,7 +38,7 @@ void body_to_strbuilder_ref( CodeBody body, StrBuilder* result ) { code_to_strbuilder_ref(curr, result); // strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) ); - ++curr; + curr = curr->Next; } } diff --git a/base/components/interface.cpp b/base/components/interface.cpp index b3eaf45..6415e51 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -3,8 +3,8 @@ #include "code_serialization.cpp" #endif -internal void parser_init(); -internal void parser_deinit(); +internal void parser_init(Context* ctx); +internal void parser_deinit(Context* ctx); internal void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, ssize alignment, void* old_memory, ssize old_size, u64 flags ) @@ -71,6 +71,14 @@ void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, return nullptr; } +internal +void fallback_logger(LogEntry entry) +{ + GEN_ASSERT(entry.msg.Len > 0); + GEN_ASSERT(entry.msg.Ptr); + log_fmt("%S: %S", loglevel_to_str(entry.level), entry.msg); +} + internal void define_constants() { @@ -283,6 +291,19 @@ void init(Context* ctx) ctx->InitSize_Fallback_Allocator_Bucket_Size = megabytes(8); } + if (ctx->InitSize_StrCacheTable == 0) + { + ctx->InitSize_StrCacheTable = kilobytes(8); + } + if (ctx->InitSize_MacrosTable == 0) + { + ctx->InitSize_MacrosTable = kilobytes(8); + } + + if (ctx->Logger == nullptr) { + ctx->Logger = & fallback_logger; + } + // Override the current context (user has to put it back if unwanted). _ctx = ctx; @@ -298,7 +319,7 @@ void init(Context* ctx) } // Setup the code pool and code entries arena. { - Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, sizeof(AST) ); + Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, size_of(AST) ); if ( code_pool.PhysicalStart == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the code pool" ); array_append( ctx->CodePools, code_pool ); @@ -311,18 +332,18 @@ void init(Context* ctx) } // Setup the hash tables { - ctx->StrCache = hashtable_init(StrCached, ctx->Allocator_DyanmicContainers); + ctx->StrCache = hashtable_init_reserve(StrCached, ctx->Allocator_DyanmicContainers, ctx->InitSize_StrCacheTable); if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); - ctx->Macros = hashtable_init(Macro, ctx->Allocator_DyanmicContainers); + ctx->Macros = hashtable_init_reserve(Macro, ctx->Allocator_DyanmicContainers, ctx->InitSize_MacrosTable); if (ctx->Macros.Hashes == nullptr || ctx->Macros.Entries == nullptr) { GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" ); } } define_constants(); - parser_init(); + parser_init(ctx); ++ context_counter; } @@ -371,7 +392,7 @@ void deinit(Context* ctx) while ( left--, left ); array_free( ctx->Fallback_AllocatorBuckets); } - parser_deinit(); + parser_deinit(ctx); if (_ctx == ctx) _ctx = nullptr; diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 7578f1c..a46337a 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -15,24 +15,6 @@ \▓▓▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓ \▓▓▓▓▓▓▓\▓▓ \▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ */ -#if 0 -enum LogLevel : u32 -{ - Info, - Warning, - Panic, -}; - -struct LogEntry -{ - Str msg; - u32 line_num; - void* data; -}; - -typedef void LoggerCallback(LogEntry entry); -#endif - // Note(Ed): This is subject to heavily change // with upcoming changes to the library's fallback (default) allocations strategy; // and major changes to lexer/parser context usage. @@ -64,9 +46,16 @@ struct Context u32 InitSize_LexerTokens; u32 SizePer_StringArena; + u32 InitSize_StrCacheTable; + u32 InitSize_MacrosTable; + // TODO(Ed): Symbol Table // Keep track of all resolved symbols (naemspaced identifiers) +// Logging + + LoggerProc* Logger; + // Parser // Used by the lexer to persistently treat all these identifiers as preprocessor defines. @@ -89,9 +78,6 @@ struct Context StringTable StrCache; - // TODO(Ed): This needs to be just handled by a parser context - Array(Token) Lexer_Tokens; - // TODO(Ed): Active parse context vs a parse result need to be separated conceptually ParseContext parser; @@ -104,6 +90,37 @@ struct Context // An implicit context interface will be provided instead as wrapper procedures as convience. GEN_API extern Context* _ctx; +// TODO(Ed): Swap all usage of this with logger_fmt (then rename logger_fmt to log_fmt) +inline +ssize log_fmt(char const* fmt, ...) +{ + ssize res; + va_list va; + + va_start(va, fmt); + res = c_str_fmt_out_va(fmt, va); + va_end(va); + + return res; +} + +inline +void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...) +{ + local_persist thread_local + PrintF_Buffer buf = struct_zero_init(); + + va_list va; + va_start(va, fmt); + ssize res = c_str_fmt_va(buf, GEN_PRINTF_MAXLEN, fmt, va) -1; + va_end(va); + + StrBuilder msg = strbuilder_make_length(ctx->Allocator_Temp, buf, res); + + LogEntry entry = { strbuilder_to_str(msg), level }; + ctx->Logger(entry); +} + // Initialize the library. There first ctx initialized must exist for lifetime of other contextes that come after as its the one that GEN_API void init(Context* ctx); @@ -114,7 +131,7 @@ GEN_API void deinit(Context* ctx); // Retrieves the active context (not usually needed, but here in case...) GEN_API Context* get_context(); -// Clears the allocations, but doesn't free the memoery, then calls init() again. +// Clears the allocations, but doesn't free the memory, then calls init() again. // Ease of use. GEN_API void reset(Context* ctx); @@ -334,37 +351,33 @@ forceinline CodeBody def_union_body ( s32 num, Code* codes ) #pragma region Parsing -#if 0 -struct StackNode +struct ParseStackNode { - StackNode* Prev; - - Token Start; - Token Name; // The name of the AST node (if parsed) - Str FailedProc; // The name of the procedure that failed + ParseStackNode* prev; + + TokenSlice tokens; + Token* start; + Str name; // The name of the AST node (if parsed) + Str proc_name; // The name of the procedure + Code code_rel; // Relevant AST node + // TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling. }; -// Stack nodes are allocated the error's allocator -struct Error +struct ParseInfo { - StrBuilder message; - StackNode* context_stack; + ParseMessage* messages; + LexedInfo lexed; + Code result; }; -struct ParseInfo +struct ParseOpts { - Arena FileMem; - Arena TokMem; - Arena CodeMem; - - FileContents FileContent; - Array Tokens; - Array Errors; - // Errors are allocated to a dedicated general arena. + AllocatorInfo backing_msgs; + AllocatorInfo backing_tokens; + AllocatorInfo backing_ast; }; -CodeBody parse_file( Str path ); -#endif +ParseInfo wip_parse_str( LexedInfo lexed, ParseOpts* opts GEN_PARAM_DEFAULT ); GEN_API CodeClass parse_class ( Str class_def ); GEN_API CodeConstructor parse_constructor ( Str constructor_def ); @@ -395,9 +408,10 @@ GEN_API ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list v //! Do not use directly. Use the token_fmt macro instead. Str token_fmt_impl( ssize, ... ); -GEN_API Code untyped_str( Str content); +GEN_API Code untyped_str ( Str content); GEN_API Code untyped_fmt ( char const* fmt, ... ); GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ); +GEN_API Code untyped_toks ( TokenSlice tokens ); #pragma endregion Untyped text diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index d5f4bef..51755f3 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #include "interface.upfront.cpp" #include "lexer.cpp" #include "parser.cpp" @@ -8,29 +8,70 @@ // Publically Exposed Interface +ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts) +{ + // TODO(Ed): Lift this. + Context* ctx = _ctx; + + if (lexed.tokens.num == 0 && lexed.tokens.ptr == nullptr) { + check_parse_args(lexed.text); + lexed = lex(ctx, lexed.text); + } + ParseInfo info = struct_zero(ParseInfo); + info.lexed = lexed; + + // TODO(Ed): ParseInfo should be set to the parser context. + + ctx->parser = struct_zero(ParseContext); + ctx->parser.tokens = lexed.tokens; + + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + + CodeBody result = parse_global_nspace(ctx,CT_Global_Body); + + parser_pop(& ctx->parser); + return info; +} + CodeClass parse_class( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - push_scope(); - CodeClass result = (CodeClass) parse_class_struct( Tok_Decl_Class, parser_not_inplace_def ); - parser_pop(& _ctx->parser); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + CodeClass result = (CodeClass) parse_class_struct( ctx, Tok_Decl_Class, parser_not_inplace_def ); + parser_pop(& ctx->parser); return result; } -CodeConstructor parse_constructor( Str def ) +CodeConstructor parse_constructor(Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + // TODO(Ed): Constructors can have prefix attributes CodeSpecifiers specifiers = NullCode; @@ -57,8 +98,8 @@ CodeConstructor parse_constructor( Str def ) break; default : - log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -71,247 +112,337 @@ CodeConstructor parse_constructor( Str def ) eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers_arr( NumSpecifiers, specs_found ); // ... } - _ctx->parser.Tokens = toks; - CodeConstructor result = parser_parse_constructor( specifiers ); + CodeConstructor result = parser_parse_constructor(ctx, specifiers); + parser_pop(& ctx->parser); return result; } CodeDefine parse_define( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - push_scope(); - CodeDefine result = parser_parse_define(); - parser_pop(& _ctx->parser); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + CodeDefine result = parser_parse_define(ctx); + parser_pop(& ctx->parser); return result; } CodeDestructor parse_destructor( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; // TODO(Ed): Destructors can have prefix attributes // TODO(Ed): Destructors can have virtual - _ctx->parser.Tokens = toks; - CodeDestructor result = parser_parse_destructor(NullCode); + CodeDestructor result = parser_parse_destructor(ctx, NullCode); return result; } CodeEnum parse_enum( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) - { - parser_pop(& _ctx->parser); + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) { return InvalidCode; } - _ctx->parser.Tokens = toks; - return parser_parse_enum( parser_not_inplace_def); + return parser_parse_enum(ctx, parser_not_inplace_def); } CodeBody parse_export_body( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_export_body(); + return parser_parse_export_body(ctx); } CodeExtern parse_extern_link( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_extern_link(); + return parser_parse_extern_link(ctx); } CodeFriend parse_friend( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_friend(); + return parser_parse_friend(ctx); } CodeFn parse_function( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return (CodeFn) parser_parse_function(); + return (CodeFn) parser_parse_function(ctx); } CodeBody parse_global_body( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - push_scope(); - CodeBody result = parse_global_nspace( CT_Global_Body ); - parser_pop(& _ctx->parser); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + CodeBody result = parse_global_nspace(ctx, CT_Global_Body ); + parser_pop(& ctx->parser); return result; } CodeNS parse_namespace( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_namespace(); + return parser_parse_namespace(ctx); } CodeOperator parse_operator( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return (CodeOperator) parser_parse_operator(); + return (CodeOperator) parser_parse_operator(ctx); } CodeOpCast parse_operator_cast( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_operator_cast(NullCode); + return parser_parse_operator_cast(ctx, NullCode); } CodeStruct parse_struct( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - push_scope(); - CodeStruct result = (CodeStruct) parse_class_struct( Tok_Decl_Struct, parser_not_inplace_def ); - parser_pop(& _ctx->parser); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); + CodeStruct result = (CodeStruct) parse_class_struct( ctx, Tok_Decl_Struct, parser_not_inplace_def ); + parser_pop(& ctx->parser); return result; } CodeTemplate parse_template( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_template(); + return parser_parse_template(ctx); } CodeTypename parse_type( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_type( parser_not_from_template, nullptr); + return parser_parse_type( ctx, parser_not_from_template, nullptr); } CodeTypedef parse_typedef( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_typedef(); + return parser_parse_typedef(ctx); } CodeUnion parse_union( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_union( parser_not_inplace_def); + return parser_parse_union(ctx, parser_not_inplace_def); } CodeUsing parse_using( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_using(); + return parser_parse_using(ctx); } CodeVar parse_variable( Str def ) { + // TODO(Ed): Lift this. + Context* ctx = _ctx; + check_parse_args( def ); - TokArray toks = lex( def ); - if ( toks.Arr == nullptr ) + ctx->parser = struct_zero(ParseContext); + + LexedInfo lexed = lex(ctx, def); + ctx->parser.tokens = lexed.tokens; + if ( ctx->parser.tokens.ptr == nullptr ) return InvalidCode; - _ctx->parser.Tokens = toks; - return parser_parse_variable(); + return parser_parse_variable(ctx); } // Undef helper macros @@ -326,6 +457,7 @@ CodeVar parse_variable( Str def ) #undef left #undef check #undef push_scope +#undef NullScope #undef def_assign // Here for C Variant diff --git a/base/components/interface.untyped.cpp b/base/components/interface.untyped.cpp index 69d74bb..14cfc4e 100644 --- a/base/components/interface.untyped.cpp +++ b/base/components/interface.untyped.cpp @@ -176,3 +176,16 @@ Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ) return result; } + +Code untyped_toks( TokenSlice tokens ) +{ + if ( tokens.num == 0 ) { + log_failure( "untyped_toks: empty token slice" ); + return InvalidCode; + } + Code + result = make_code(); + result->Type = CT_Untyped; + result->ContentToks = tokens; + return result; +} diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 94af5f7..b30864b 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -473,8 +473,10 @@ CodeComment def_comment( Str content ) return (CodeComment) result; } -CodeConstructor def_constructor( Opts_def_constructor p ) +CodeConstructor def_constructor( Opts_def_constructor opt ) { + Opts_def_constructor p = get_optional(opt); + if ( p.params && p.params->Type != CT_Parameters ) { log_failure("gen::def_constructor: params must be of Parameters type - %s", code_debug_str((Code)p.params)); GEN_DEBUG_TRAP(); @@ -510,8 +512,10 @@ CodeConstructor def_constructor( Opts_def_constructor p ) return result; } -CodeClass def_class( Str name, Opts_def_struct p ) +CodeClass def_class( Str name, Opts_def_struct opt ) { + Opts_def_struct p = get_optional(opt); + if ( ! name_check( def_class, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -561,8 +565,10 @@ CodeClass def_class( Str name, Opts_def_struct p ) return result; } -CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) +CodeDefine def_define( Str name, MacroType type, Opts_def_define opt ) { + Opts_def_define p = get_optional(opt); + if ( ! name_check( def_define, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -585,8 +591,10 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) return result; } -CodeDestructor def_destructor( Opts_def_destructor p ) +CodeDestructor def_destructor( Opts_def_destructor opt ) { + Opts_def_destructor p = get_optional(opt); + if ( p.specifiers && p.specifiers->Type != CT_Specifiers ) { log_failure( "gen::def_destructor: specifiers was not a 'Specifiers' type: %s", code_debug_str(p.specifiers) ); GEN_DEBUG_TRAP(); @@ -619,8 +627,10 @@ CodeDestructor def_destructor( Opts_def_destructor p ) return result; } -CodeEnum def_enum( Str name, Opts_def_enum p ) +CodeEnum def_enum( Str name, Opts_def_enum opt ) { + Opts_def_enum p = get_optional(opt); + if ( ! name_check( def_enum, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -742,8 +752,10 @@ CodeFriend def_friend( Code declaration ) return result; } -CodeFn def_function( Str name, Opts_def_function p ) +CodeFn def_function( Str name, Opts_def_function opt ) { + Opts_def_function p = get_optional(opt); + if ( ! name_check( def_function, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -802,8 +814,10 @@ CodeFn def_function( Str name, Opts_def_function p ) return result; } -CodeInclude def_include( Str path, Opts_def_include p ) +CodeInclude def_include( Str path, Opts_def_include opt ) { + Opts_def_include p = get_optional(opt); + if ( path.Len <= 0 || path.Ptr == nullptr ) { log_failure( "gen::def_include: Invalid path provided - %d" ); GEN_DEBUG_TRAP(); @@ -821,8 +835,10 @@ CodeInclude def_include( Str path, Opts_def_include p ) return result; } -CodeModule def_module( Str name, Opts_def_module p ) +CodeModule def_module( Str name, Opts_def_module opt ) { + Opts_def_module p = get_optional(opt); + if ( ! name_check( def_module, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -835,8 +851,10 @@ CodeModule def_module( Str name, Opts_def_module p ) return result; } -CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p ) +CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace opt ) { + Opts_def_namespace p = get_optional(opt); + if ( ! name_check( def_namespace, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -859,8 +877,10 @@ CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p ) return result; } -CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p ) +CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator opt ) { + Opts_def_operator p = get_optional(opt); + if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) { log_failure( "gen::def_operator: PlatformAttributes was provided but its not of attributes type: %s", code_debug_str(p.attributes) ); GEN_DEBUG_TRAP(); @@ -926,8 +946,10 @@ CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p ) return result; } -CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p ) +CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast opt ) { + Opts_def_operator_cast p = get_optional(opt); + if ( ! null_check( def_operator_cast, type )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -959,8 +981,10 @@ CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p ) return result; } -CodeParams def_param( CodeTypename type, Str name, Opts_def_param p ) +CodeParams def_param( CodeTypename type, Str name, Opts_def_param opt ) { + Opts_def_param p = get_optional(opt); + if ( ! name_check( def_param, name ) || ! null_check( def_param, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1034,8 +1058,10 @@ CodeSpecifiers def_specifier( Specifier spec ) return result; } -CodeStruct def_struct( Str name, Opts_def_struct p ) +CodeStruct def_struct( Str name, Opts_def_struct opt ) { + Opts_def_struct p = get_optional(opt); + if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) { log_failure( "gen::def_struct: attributes was not a `PlatformAttributes` type - %s", code_debug_str(cast(Code, p.attributes)) ); GEN_DEBUG_TRAP(); @@ -1076,8 +1102,10 @@ CodeStruct def_struct( Str name, Opts_def_struct p ) return result; } -CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template p ) +CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template opt ) { + Opts_def_template p = get_optional(opt); + if ( ! null_check( def_template, declaration ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1108,8 +1136,10 @@ CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_templat return result; } -CodeTypename def_type( Str name, Opts_def_type p ) +CodeTypename def_type( Str name, Opts_def_type opt ) { + Opts_def_type p = get_optional(opt); + if ( ! name_check( def_type, name )) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1143,8 +1173,10 @@ CodeTypename def_type( Str name, Opts_def_type p ) return result; } -CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p ) +CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef opt ) { + Opts_def_typedef p = get_optional(opt); + if ( ! null_check( def_typedef, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1206,8 +1238,10 @@ CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p ) return result; } -CodeUnion def_union( Str name, CodeBody body, Opts_def_union p ) +CodeUnion def_union( Str name, CodeBody body, Opts_def_union opt ) { + Opts_def_union p = get_optional(opt); + if ( ! null_check( def_union, body ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1233,8 +1267,10 @@ CodeUnion def_union( Str name, CodeBody body, Opts_def_union p ) return result; } -CodeUsing def_using( Str name, CodeTypename type, Opts_def_using p ) +CodeUsing def_using( Str name, CodeTypename type, Opts_def_using opt ) { + Opts_def_using p = get_optional(opt); + if ( ! name_check( def_using, name ) || null_check( def_using, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; @@ -1274,8 +1310,10 @@ CodeUsing def_using_namespace( Str name ) return result; } -CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable p ) +CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable opt ) { + Opts_def_variable p = get_optional(opt); + if ( ! name_check( def_variable, name ) || ! null_check( def_variable, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index f501f70..6c74241 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -1,12 +1,12 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once #include "interface.upfront.cpp" -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #endif -StrBuilder tok_to_strbuilder(Token tok) +StrBuilder tok_to_strbuilder(AllocatorInfo ainfo, Token tok) { - StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); + StrBuilder result = strbuilder_make_reserve( ainfo, kilobytes(4) ); Str type_str = toktype_to_str( tok.Type ); strbuilder_append_fmt( & result, "Line: %d Column: %d, Type: %.*s Content: %.*s" @@ -17,55 +17,55 @@ StrBuilder tok_to_strbuilder(Token tok) return result; } -bool lex__eat( TokArray* self, TokType type ); +bool lex__eat(Context* ctx, ParseContext* self, TokType type ); -Token* lex_current(TokArray* self, bool skip_formatting ) +Token* lex_current(ParseContext* self, bool skip_formatting ) { if ( skip_formatting ) { - while ( self->Arr[self->Idx].Type == Tok_NewLine || self->Arr[self->Idx].Type == Tok_Comment ) - self->Idx++; + while ( self->tokens.ptr[self->token_id].Type == Tok_NewLine || self->tokens.ptr[self->token_id].Type == Tok_Comment ) + self->token_id++; } - return & self->Arr[self->Idx]; + return & self->tokens.ptr[self->token_id]; } -Token* lex_peek(TokArray self, bool skip_formatting) +Token* lex_peek(ParseContext const* self, bool skip_formatting) { - s32 idx = self.Idx; + s32 idx = self->token_id; if ( skip_formatting ) { - while ( self.Arr[idx].Type == Tok_NewLine ) + while ( self->tokens.ptr[idx].Type == Tok_NewLine ) idx++; - return & self.Arr[idx]; + return & self->tokens.ptr[idx]; } - return & self.Arr[idx]; + return & self->tokens.ptr[idx]; } -Token* lex_previous(TokArray self, bool skip_formatting) +Token* lex_previous(ParseContext const* self, bool skip_formatting) { - s32 idx = self.Idx; + s32 idx = self->token_id; if ( skip_formatting ) { - while ( self.Arr[idx].Type == Tok_NewLine ) + while ( self->tokens.ptr[idx].Type == Tok_NewLine ) idx --; - return & self.Arr[idx]; + return & self->tokens.ptr[idx]; } - return & self.Arr[idx - 1]; + return & self->tokens.ptr[idx - 1]; } -Token* lex_next(TokArray self, bool skip_formatting) +Token* lex_next(ParseContext const* self, bool skip_formatting) { - s32 idx = self.Idx; + s32 idx = self->token_id; if ( skip_formatting ) { - while ( self.Arr[idx].Type == Tok_NewLine ) + while ( self->tokens.ptr[idx].Type == Tok_NewLine ) idx++; - return & self.Arr[idx + 1]; + return & self->tokens.ptr[idx + 1]; } - return & self.Arr[idx + 1]; + return & self->tokens.ptr[idx + 1]; } enum @@ -137,7 +137,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) ); // GEN_DEBUG_TRAP(); } - array_append( _ctx->Lexer_Tokens, name ); + array_append(ctx->tokens, name); if ( ctx->left && (* ctx->scanner) == '(' ) { @@ -152,7 +152,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) } Token opening_paren = { { ctx->scanner, 1 }, Tok_Paren_Open, ctx->line, ctx->column, TF_Preprocess }; - array_append( _ctx->Lexer_Tokens, opening_paren ); + array_append(ctx->tokens, opening_paren); move_forward(); Token last_parameter = {}; @@ -168,7 +168,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) move_forward(); move_forward(); - array_append(_ctx->Lexer_Tokens, parameter); + array_append(ctx->tokens, parameter); skip_whitespace(); last_parameter = parameter; @@ -202,7 +202,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) move_forward(); parameter.Text.Len++; } - array_append(_ctx->Lexer_Tokens, parameter); + array_append(ctx->tokens, parameter); skip_whitespace(); last_parameter = parameter; } @@ -229,7 +229,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) return Lex_ReturnNull; } Token comma = { { ctx->scanner, 1 }, Tok_Comma, ctx->line, ctx->column, TF_Preprocess }; - array_append(_ctx->Lexer_Tokens, comma); + array_append(ctx->tokens, comma); move_forward(); } @@ -243,7 +243,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) return Lex_ReturnNull; } Token closing_paren = { { ctx->scanner, 1 }, Tok_Paren_Close, ctx->line, ctx->column, TF_Preprocess }; - array_append(_ctx->Lexer_Tokens, closing_paren); + array_append(ctx->tokens, closing_paren); move_forward(); } else if ( registered_macro && macro_is_functional( * registered_macro) ) { @@ -268,7 +268,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) { char const* hash = ctx->scanner; Token hash_tok = { { hash, 1 }, Tok_Preprocess_Hash, ctx->line, ctx->column, TF_Preprocess }; - array_append( _ctx->Lexer_Tokens, hash_tok ); + array_append(ctx->tokens, hash_tok); move_forward(); skip_whitespace(); @@ -344,14 +344,14 @@ s32 lex_preprocessor_directive( LexContext* ctx ) ctx->token.Text.Len = ctx->token.Text.Len + ctx->token.Text.Ptr - hash; ctx->token.Text.Ptr = hash; - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); return Lex_Continue; // Skip found token, its all handled here. } if ( ctx->token.Type == Tok_Preprocess_Else || ctx->token.Type == Tok_Preprocess_EndIf ) { ctx->token.Flags |= TF_Preprocess_Cond; - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); end_line(); return Lex_Continue; } @@ -360,7 +360,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) ctx->token.Flags |= TF_Preprocess_Cond; } - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); skip_whitespace(); @@ -379,7 +379,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( (* ctx->scanner) != '"' && (* ctx->scanner) != '<' ) { - StrBuilder directive_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%.*s", min( 80, ctx->left + preprocess_content.Text.Len ), ctx->token.Text.Ptr ); + StrBuilder directive_str = strbuilder_fmt_buf( ctx->allocator_temp, "%.*s", min( 80, ctx->left + preprocess_content.Text.Len ), ctx->token.Text.Ptr ); log_failure( "gen::Parser::lex: Expected '\"' or '<' after #include, not '%c' (%d, %d)\n%s" , (* ctx->scanner) @@ -411,7 +411,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) move_forward(); } - array_append( _ctx->Lexer_Tokens, preprocess_content ); + array_append(ctx->tokens, preprocess_content); return Lex_Continue; // Skip found token, its all handled here. } @@ -446,8 +446,8 @@ s32 lex_preprocessor_directive( LexContext* ctx ) } else { - StrBuilder directive_str = strbuilder_make_length( _ctx->Allocator_Temp, ctx->token.Text.Ptr, ctx->token.Text.Len ); - StrBuilder content_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%.*s", min( 400, ctx->left + preprocess_content.Text.Len ), preprocess_content.Text.Ptr ); + StrBuilder directive_str = strbuilder_make_length( ctx->allocator_temp, ctx->token.Text.Ptr, ctx->token.Text.Len ); + StrBuilder content_str = strbuilder_fmt_buf( ctx->allocator_temp, "%.*s", min( 400, ctx->left + preprocess_content.Text.Len ), preprocess_content.Text.Ptr ); log_failure( "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)" " in preprocessor directive '%s' (%d, %d)\n%s" @@ -475,14 +475,14 @@ s32 lex_preprocessor_directive( LexContext* ctx ) preprocess_content.Text.Len++; } - array_append( _ctx->Lexer_Tokens, preprocess_content ); + array_append(ctx->tokens, preprocess_content); return Lex_Continue; // Skip found token, its all handled here. } void lex_found_token( LexContext* ctx ) { if ( ctx->token.Type != Tok_Invalid ) { - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); return; } @@ -508,7 +508,7 @@ void lex_found_token( LexContext* ctx ) } ctx->token.Type = type; - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); return; } if ( ( type <= Tok_Star && type >= Tok_Spec_Alignas) @@ -517,13 +517,13 @@ void lex_found_token( LexContext* ctx ) { ctx->token.Type = type; ctx->token.Flags |= TF_Specifier; - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); return; } if ( type != Tok_Invalid ) { ctx->token.Type = type; - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); return; } @@ -561,50 +561,42 @@ void lex_found_token( LexContext* ctx ) ctx->token.Type = Tok_Identifier; } - array_append( _ctx->Lexer_Tokens, ctx->token ); + array_append(ctx->tokens, ctx->token); } +// TODO(Ed): We should dynamically allocate the lexer's array in Allocator_DyanmicContainers. + // TODO(Ed): We need to to attempt to recover from a lex failure? + neverinline -// TokArray lex( Array tokens, Str content ) -TokArray lex( Str content ) +LexedInfo lex(Context* lib_ctx, Str content) { - LexContext c; LexContext* ctx = & c; - c.content = content; - c.left = content.Len; - c.scanner = content.Ptr; + LexedInfo info = struct_zero(LexedInfo); - char const* word = c.scanner; - s32 word_length = 0; + LexContext c = struct_zero(LexContext); LexContext* ctx = & c; + c.allocator_temp = lib_ctx->Allocator_Temp; + c.content = content; + c.left = content.Len; + c.scanner = content.Ptr; + c.line = 1; + c.column = 1; + c.tokens = array_init_reserve(Token, lib_ctx->Allocator_DyanmicContainers, lib_ctx->InitSize_LexerTokens ); - c.line = 1; - c.column = 1; + // TODO(Ed): Re-implement to new constraints: + // 1. Ability to continue on error + // 2. Return a lexed info. skip_whitespace(); - if ( c.left <= 0 ) - { + if ( c.left <= 0 ) { log_failure( "gen::lex: no tokens found (only whitespace provided)" ); - TokArray null_array = {}; - return null_array; + return info; } - array_clear(_ctx->Lexer_Tokens); - b32 preprocess_args = true; while (c.left ) { - #if 0 - if (Tokens.num()) - { - log_fmt("\nLastTok: %SB", Tokens.back().to_strbuilder()); - } - #endif - - { - Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; - c.token = thanks_c; - } + c.token = struct_init(Token) { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; bool is_define = false; @@ -623,7 +615,7 @@ TokArray lex( Str content ) c.token.Type = Tok_NewLine; c.token.Text.Len++; - array_append( _ctx->Lexer_Tokens, c.token ); + array_append(c.tokens, c.token); continue; } } @@ -662,7 +654,7 @@ TokArray lex( Str content ) c.token.Text.Len++; move_forward(); - array_append( _ctx->Lexer_Tokens, c.token ); + array_append(c.tokens, c.token); } } continue; @@ -670,8 +662,7 @@ TokArray lex( Str content ) case Lex_ReturnNull: { - TokArray tok_array = {}; - return tok_array; + return info; } } } @@ -698,7 +689,7 @@ TokArray lex( Str content ) } else { - StrBuilder context_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%s", c.scanner, min( 100, c.left ) ); + StrBuilder context_str = strbuilder_fmt_buf( lib_ctx->Allocator_Temp, "%s", c.scanner, min( 100, c.left ) ); log_failure( "gen::lex: invalid varadic argument, expected '...' got '..%c' (%d, %d)\n%s", (* ctx->scanner), c.line, c.column, context_str ); } @@ -1119,7 +1110,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; } - array_append( _ctx->Lexer_Tokens, c.token ); + array_append(c.tokens, c.token); continue; } else if ( (* ctx->scanner) == '*' ) @@ -1155,7 +1146,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; } - array_append( _ctx->Lexer_Tokens, c.token ); + array_append(c.tokens, c.token); // end_line(); continue; } @@ -1243,14 +1234,14 @@ TokArray lex( Str content ) } else { - s32 start = max( 0, array_num(_ctx->Lexer_Tokens) - 100 ); + s32 start = max( 0, array_num(c.tokens) - 100 ); log_fmt("\n%d\n", start); - for ( s32 idx = start; idx < array_num(_ctx->Lexer_Tokens); idx++ ) + for ( s32 idx = start; idx < array_num(c.tokens); idx++ ) { log_fmt( "Token %d Type: %s : %.*s\n" , idx - , toktype_to_str( _ctx->Lexer_Tokens[ idx ].Type ).Ptr - , _ctx->Lexer_Tokens[ idx ].Text.Len, _ctx->Lexer_Tokens[ idx ].Text.Ptr + , toktype_to_str( c.tokens[ idx ].Type ).Ptr + , c.tokens[ idx ].Text.Len, c.tokens[ idx ].Text.Ptr ); } @@ -1266,7 +1257,7 @@ TokArray lex( Str content ) FoundToken: { lex_found_token( ctx ); - TokType last_type = array_back(_ctx->Lexer_Tokens)->Type; + TokType last_type = array_back(c.tokens)->Type; if ( last_type == Tok_Preprocess_Macro_Stmt || last_type == Tok_Preprocess_Macro_Expr ) { Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; @@ -1281,21 +1272,22 @@ TokArray lex( Str content ) c.token.Text.Len++; move_forward(); - array_append( _ctx->Lexer_Tokens, c.token ); + array_append(c.tokens, c.token); continue; } } } } - if ( array_num(_ctx->Lexer_Tokens) == 0 ) { + if ( array_num(c.tokens) == 0 ) { log_failure( "Failed to lex any tokens" ); - TokArray tok_array = {}; - return tok_array; + return info; } - - TokArray result = { _ctx->Lexer_Tokens, 0 }; - return result; + + info.messages = c.messages; + info.text = content; + info.tokens = struct_init(TokenSlice) { pcast(Token*, c.tokens), scast(s32, array_num(c.tokens)) }; + return info; } #undef move_forward diff --git a/base/components/parser.cpp b/base/components/parser.cpp index b064e04..254353e 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1,6 +1,6 @@ #ifdef INTELLISENSE_DIRECTIVES #pragma once -#include "gen/etoktype.cpp" +#include "gen/etoktype.hpp" #include "parser_case_macros.cpp" #include "interface.upfront.cpp" #include "lexer.cpp" @@ -11,86 +11,86 @@ constexpr bool lex_dont_skip_formatting = false; constexpr bool lex_skip_formatting = true; -void parser_push( ParseContext* ctx, StackNode* node ) +void parser_push( ParseContext* ctx, ParseStackNode* node ) { - node->Prev = ctx->Scope; - ctx->Scope = node; + node->prev = ctx->scope; + ctx->scope = node; #if 0 && GEN_BUILD_DEBUG - log_fmt("\tEntering _ctx->parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + log_fmt("\tEntering parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif } void parser_pop(ParseContext* ctx) { #if 0 && GEN_BUILD_DEBUG - log_fmt("\tPopping _ctx->parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + log_fmt("\tPopping parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif - ctx->Scope = ctx->Scope->Prev; + ctx->scope = ctx->scope->prev; } -StrBuilder parser_to_strbuilder(ParseContext ctx) +StrBuilder parser_to_strbuilder(ParseContext const* ctx, AllocatorInfo temp) { - StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); + StrBuilder result = strbuilder_make_reserve( temp, kilobytes(4) ); - Token scope_start = * ctx.Scope->Start; - Token last_valid = ctx.Tokens.Idx >= array_num(ctx.Tokens.Arr) ? ctx.Tokens.Arr[array_num(ctx.Tokens.Arr) -1] : (* lex_current(& ctx.Tokens, true)); + Token scope_start = * ctx->scope->start; + Token last_valid = (ctx->token_id >= ctx->tokens.num) ? ctx->tokens.ptr[ctx->tokens.num -1] : (* lex_peek(ctx, true)); sptr length = scope_start.Text.Len; char const* current = scope_start.Text.Ptr + length; - while ( current <= array_back( ctx.Tokens.Arr)->Text.Ptr && (* current) != '\n' && length < 74 ) + while ( current <= ctx->tokens.ptr[ctx->tokens.num - 1].Text.Ptr && (* current) != '\n' && length < 74 ) { current++; length++; } Str scope_str = { scope_start.Text.Ptr, length }; - StrBuilder line = strbuilder_make_str( _ctx->Allocator_Temp, scope_str ); + StrBuilder line = strbuilder_make_str( temp, scope_str ); strbuilder_append_fmt( & result, "\tScope : %s\n", line ); strbuilder_free(& line); sptr dist = (sptr)last_valid.Text.Ptr - (sptr)scope_start.Text.Ptr + 2; sptr length_from_err = dist; - Str err_str = { last_valid.Text.Ptr, length_from_err }; - StrBuilder line_from_err = strbuilder_make_str( _ctx->Allocator_Temp, err_str ); + Str err_str = { last_valid.Text.Ptr, length_from_err }; + StrBuilder line_from_err = strbuilder_make_str( temp, err_str ); if ( length_from_err < 100 ) strbuilder_append_fmt(& result, "\t(%d, %d):%*c\n", last_valid.Line, last_valid.Column, length_from_err, '^' ); else strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column ); - StackNode* curr_scope = ctx.Scope; + ParseStackNode* curr_scope = ctx->scope; s32 level = 0; do { - if ( curr_scope->Name.Ptr ) { - strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Len, curr_scope->Name.Ptr ); + if ( curr_scope->name.Ptr ) { + strbuilder_append_fmt(& result, "\t%d: %S, AST Name: %S\n", level, curr_scope->proc_name, curr_scope->name ); } else { - strbuilder_append_fmt(& result, "\t%d: %s\n", level, curr_scope->ProcName.Ptr ); + strbuilder_append_fmt(& result, "\t%d: %S\n", level, curr_scope->proc_name ); } - curr_scope = curr_scope->Prev; + curr_scope = curr_scope->prev; level++; } while ( curr_scope ); return result; } -bool lex__eat(TokArray* self, TokType type ) +bool lex__eat(Context* ctx, ParseContext* parser, TokType type) { - if ( array_num(self->Arr) - self->Idx <= 0 ) { - log_failure( "No tokens left.\n%s", parser_to_strbuilder(_ctx->parser) ); + if ( parser->tokens.num - parser->token_id <= 0 ) { + log_failure( "No tokens left.\n%SB", parser_to_strbuilder(parser, ctx->Allocator_Temp) ); return false; } - Token at_idx = self->Arr[ self->Idx ]; + Token at_idx = parser->tokens.ptr[ parser->token_id ]; if ( ( at_idx.Type == Tok_NewLine && type != Tok_NewLine ) || ( at_idx.Type == Tok_Comment && type != Tok_Comment ) ) { - self->Idx ++; + parser->token_id ++; } b32 not_accepted = at_idx.Type != type; @@ -103,13 +103,13 @@ bool lex__eat(TokArray* self, TokType type ) } if ( not_accepted ) { - Token tok = * lex_current( self, lex_skip_formatting ); - log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s" - , toktype_to_str(type).Ptr - , at_idx.Text.Len, at_idx.Text.Ptr + Token tok = * lex_current( parser, lex_skip_formatting ); + log_failure( "Parse Error, TokArray::eat, Expected: ' %S ' not ' %S ' (%d, %d)`\n%SB" + , toktype_to_str(type) + , at_idx.Text , tok.Line , tok.Column - , parser_to_strbuilder(_ctx->parser) + , parser_to_strbuilder(parser, ctx->Allocator_Temp) ); GEN_DEBUG_TRAP(); return false; @@ -119,51 +119,48 @@ bool lex__eat(TokArray* self, TokType type ) log_fmt("Ate: %SB\n", self->Arr[Idx].to_strbuilder() ); #endif - self->Idx ++; + parser->token_id ++; return true; } internal -void parser_init() +void parser_init(Context* ctx) { - _ctx->Lexer_Tokens = array_init_reserve(Token, _ctx->Allocator_DyanmicContainers, _ctx->InitSize_LexerTokens ); } internal -void parser_deinit() +void parser_deinit(Context* ctx) { - Array(Token) null_array = { nullptr }; - _ctx->Lexer_Tokens = null_array; } #pragma region Helper Macros -#define check_parse_args( def ) _check_parse_args(def, stringize(_func_) ) -bool _check_parse_args( Str def, char const* func_name ) +#define check_parse_args( def ) _check_parse_args(& ctx->parser, def, stringize(_func_) ) +bool _check_parse_args(ParseContext* parser, Str def, char const* func_name ) { if ( def.Len <= 0 ) { log_failure( c_str_fmt_buf("gen::%s: length must greater than 0", func_name) ); - parser_pop(& _ctx->parser); + parser_pop(parser); return false; } if ( def.Ptr == nullptr ) { log_failure( c_str_fmt_buf("gen::%s: def was null", func_name) ); - parser_pop(& _ctx->parser); + parser_pop(parser); return false; } return true; } -# define currtok_noskip (* lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting )) -# define currtok (* lex_current( & _ctx->parser.Tokens, lex_skip_formatting )) -# define peektok (* lex_peek(_ctx->parser.Tokens, lex_skip_formatting)) -# define prevtok (* lex_previous( _ctx->parser.Tokens, lex_dont_skip_formatting)) -# define nexttok (* lex_next( _ctx->parser.Tokens, lex_skip_formatting )) -# define nexttok_noskip (* lex_next( _ctx->parser.Tokens, lex_dont_skip_formatting)) -# define eat( Type_ ) lex__eat( & _ctx->parser.Tokens, Type_ ) -# define left ( array_num(_ctx->parser.Tokens.Arr) - _ctx->parser.Tokens.Idx ) +# define currtok_noskip (* lex_current( & ctx->parser, lex_dont_skip_formatting )) +# define currtok (* lex_current( & ctx->parser, lex_skip_formatting )) +# define peektok (* lex_peek(& ctx->parser, lex_skip_formatting)) +# define prevtok (* lex_previous( & ctx->parser, lex_dont_skip_formatting)) +# define nexttok (* lex_next( & ctx->parser, lex_skip_formatting )) +# define nexttok_noskip (* lex_next( & ctx->parser, lex_dont_skip_formatting)) +# define eat( Type_ ) lex__eat(ctx, & ctx->parser, Type_ ) +# define left ( ctx->parser.tokens.num - ctx->parser.token_id ) #if GEN_COMPILER_CPP # define def_assign( ... ) { __VA_ARGS__ } @@ -171,7 +168,6 @@ bool _check_parse_args( Str def, char const* func_name ) # define def_assign( ... ) __VA_ARGS__ #endif - #ifdef check #define CHECK_WAS_DEFINED #pragma push_macro("check") @@ -181,60 +177,61 @@ bool _check_parse_args( Str def, char const* func_name ) # define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) # define check( Type_ ) ( left && currtok.Type == Type_ ) -# define push_scope() \ - Str null_name = {}; \ - StackNode scope = { nullptr, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \ - parser_push( & _ctx->parser, & scope ) +#if GEN_COMPILER_CPP +# define NullScope { nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), Str{nullptr, 0}, txt( __func__ ), { nullptr} } +#else +# define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} } +#endif #pragma endregion Helper Macros // Procedure Forwards ( Entire parser internal parser interface ) -internal Code parse_array_decl (); -internal CodeAttributes parse_attributes (); -internal CodeComment parse_comment (); -internal Code parse_complicated_definition ( TokType which ); -internal CodeBody parse_class_struct_body ( TokType which, Token name ); -internal Code parse_class_struct ( TokType which, bool inplace_def ); -internal Code parse_expression (); -internal Code parse_forward_or_definition ( TokType which, bool is_inplace ); -internal CodeFn parse_function_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name ); -internal Code parse_function_body (); -internal CodeBody parse_global_nspace ( CodeType which ); -internal Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ); -internal Token parse_identifier ( bool* possible_member_function ); -internal CodeInclude parse_include (); -internal Code parse_macro_as_definiton ( CodeAttributes attributes, CodeSpecifiers specifiers ); -internal CodeOperator parse_operator_after_ret_type ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type ); -internal Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ); -internal CodePragma parse_pragma (); -internal CodeParams parse_params ( bool use_template_capture ); -internal CodePreprocessCond parse_preprocess_cond (); -internal Code parse_simple_preprocess ( TokType which ); -internal Code parse_static_assert (); -internal void parse_template_args ( Token* token ); -internal CodeVar parse_variable_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename type, Str name ); -internal CodeVar parse_variable_declaration_list (); - -internal CodeClass parser_parse_class ( bool inplace_def ); -internal CodeConstructor parser_parse_constructor ( CodeSpecifiers specifiers ); -internal CodeDefine parser_parse_define (); -internal CodeDestructor parser_parse_destructor ( CodeSpecifiers specifiers ); -internal CodeEnum parser_parse_enum ( bool inplace_def ); -internal CodeBody parser_parse_export_body (); -internal CodeBody parser_parse_extern_link_body(); -internal CodeExtern parser_parse_extern_link (); -internal CodeFriend parser_parse_friend (); -internal CodeFn parser_parse_function (); -internal CodeNS parser_parse_namespace (); -internal CodeOpCast parser_parse_operator_cast ( CodeSpecifiers specifiers ); -internal CodeStruct parser_parse_struct ( bool inplace_def ); -internal CodeVar parser_parse_variable (); -internal CodeTemplate parser_parse_template (); -internal CodeTypename parser_parse_type ( bool from_template, bool* is_function ); -internal CodeTypedef parser_parse_typedef (); -internal CodeUnion parser_parse_union ( bool inplace_def ); -internal CodeUsing parser_parse_using (); +internal Code parse_array_decl (Context* ctx); +internal CodeAttributes parse_attributes (Context* ctx); +internal CodeComment parse_comment (Context* ctx); +internal Code parse_complicated_definition (Context* ctx, TokType which); +internal CodeBody parse_class_struct_body (Context* ctx, TokType which, Token name); +internal Code parse_class_struct (Context* ctx, TokType which, bool inplace_def); +internal Code parse_expression (Context* ctx); +internal Code parse_forward_or_definition (Context* ctx, TokType which, bool is_inplace); +internal CodeFn parse_function_after_name (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name); +internal Code parse_function_body (Context* ctx); +internal CodeBody parse_global_nspace (Context* ctx, CodeType which); +internal Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers); +internal Token parse_identifier (Context* ctx, bool* possible_member_function); +internal CodeInclude parse_include (Context* ctx); +internal Code parse_macro_as_definiton (Context* ctx, CodeAttributes attributes, CodeSpecifiers specifiers); +internal CodeOperator parse_operator_after_ret_type (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type); +internal Code parse_operator_function_or_variable(Context* ctx, bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers); +internal CodePragma parse_pragma (Context* ctx); +internal CodeParams parse_params (Context* ctx, bool use_template_capture); +internal CodePreprocessCond parse_preprocess_cond (Context* ctx); +internal Code parse_simple_preprocess (Context* ctx, TokType which); +internal Code parse_static_assert (Context* ctx); +internal void parse_template_args (Context* ctx, Token* token ); +internal CodeVar parse_variable_after_name (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename type, Str name); +internal CodeVar parse_variable_declaration_list (Context* ctx); + +internal CodeClass parser_parse_class (Context* ctx, bool inplace_def ); +internal CodeConstructor parser_parse_constructor (Context* ctx, CodeSpecifiers specifiers ); +internal CodeDefine parser_parse_define (Context* ctx); +internal CodeDestructor parser_parse_destructor (Context* ctx, CodeSpecifiers specifiers ); +internal CodeEnum parser_parse_enum (Context* ctx, bool inplace_def ); +internal CodeBody parser_parse_export_body (Context* ctx); +internal CodeBody parser_parse_extern_link_body(Context* ctx); +internal CodeExtern parser_parse_extern_link (Context* ctx); +internal CodeFriend parser_parse_friend (Context* ctx); +internal CodeFn parser_parse_function (Context* ctx); +internal CodeNS parser_parse_namespace (Context* ctx); +internal CodeOpCast parser_parse_operator_cast (Context* ctx, CodeSpecifiers specifiers ); +internal CodeStruct parser_parse_struct (Context* ctx, bool inplace_def ); +internal CodeVar parser_parse_variable (Context* ctx); +internal CodeTemplate parser_parse_template (Context* ctx); +internal CodeTypename parser_parse_type (Context* ctx, bool from_template, bool* is_function ); +internal CodeTypedef parser_parse_typedef (Context* ctx); +internal CodeUnion parser_parse_union (Context* ctx, bool inplace_def ); +internal CodeUsing parser_parse_using (Context* ctx); constexpr bool parser_inplace_def = true; constexpr bool parser_not_inplace_def = false; @@ -252,9 +249,9 @@ constexpr bool parser_strip_formatting_dont_preserve_newlines = false; It has edge case failures that prevent it from being used in function bodies. */ internal -StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines ) +StrBuilder parser_strip_formatting(Context* ctx, Str raw_text, bool preserve_newlines ) { - StrBuilder content = strbuilder_make_reserve( _ctx->Allocator_Temp, raw_text.Len ); + StrBuilder content = strbuilder_make_reserve( ctx->Allocator_Temp, raw_text.Len ); if ( raw_text.Len == 0 ) return content; @@ -490,10 +487,18 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines ) return content; } +StrBuilder parser_strip_formatting_2(TokenSlice tokens) +{ + // TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content. + StrBuilder result = struct_zero(StrBuilder); + return result; +} + internal -Code parse_array_decl() +Code parse_array_decl(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); if ( check( Tok_Operator ) && currtok.Text.Ptr[0] == '[' && currtok.Text.Ptr[1] == ']' ) { @@ -501,7 +506,7 @@ Code parse_array_decl() eat( Tok_Operator ); // [] - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return array_expr; } @@ -512,41 +517,45 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type == Tok_BraceSquare_Close ) { - log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, empty array expression in definition\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } + TokenSlice tokens = { & currtok, 1 }; Token untyped_tok = currtok; while ( left && currtok.Type != Tok_BraceSquare_Close ) { eat( currtok.Type ); + ++ tokens.num; } - untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; + // untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; + untyped_tok.Text = token_range_to_str(untyped_tok, prevtok); Code array_expr = untyped_str( untyped_tok.Text ); + // Code array_expr = untyped_toks( tokens ); // TODO(Ed): Use token slice instead of untyped strings. // [ if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Error, unexpected end of array declaration, expected ]\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type != Tok_BraceSquare_Close ) { - log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "%s: Error, expected ] in array declaration, not %S\n%SB", toktype_to_str( currtok.Type ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -556,24 +565,25 @@ Code parse_array_decl() // Its a multi-dimensional array if ( check( Tok_BraceSquare_Open )) { - Code adjacent_arr_expr = parse_array_decl(); + Code adjacent_arr_expr = parse_array_decl(ctx); // [ ][ ]... array_expr->Next = adjacent_arr_expr; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return array_expr; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return NullCode; } internal inline -CodeAttributes parse_attributes() +CodeAttributes parse_attributes(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope); Token start = currtok; s32 len = 0; @@ -664,28 +674,27 @@ CodeAttributes parse_attributes() if ( len > 0 ) { Str attribute_txt = { start.Text.Ptr, len }; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); - StrBuilder name_stripped = parser_strip_formatting( attribute_txt, parser_strip_formatting_dont_preserve_newlines ); + StrBuilder name_stripped = parser_strip_formatting(ctx, attribute_txt, parser_strip_formatting_dont_preserve_newlines ); Code result = make_code(); result->Type = CT_PlatformAttributes; result->Name = cache_str( strbuilder_to_str(name_stripped) ); result->Content = result->Name; // result->Token = - return ( CodeAttributes )result; } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return NullCode; } internal -Code parse_class_struct( TokType which, bool inplace_def ) +Code parse_class_struct(Context* ctx, TokType which, bool inplace_def) { if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) { - log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(_ctx->parser) ); + log_failure( "Error, expected class or struct, not %S\n%SB", toktype_to_str( which ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); return InvalidCode; } @@ -708,12 +717,12 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( which ); // - attributes = parse_attributes(); + attributes = parse_attributes(ctx); // if ( check( Tok_Identifier ) ) { - name = parse_identifier(nullptr); - _ctx->parser.Scope->Name = name.Text; + name = parse_identifier(ctx, nullptr); + ctx->parser.scope->name = name.Text; } // @@ -726,10 +735,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) local_persist char interface_arr_mem[ kilobytes(4) ] = {0}; - Array(CodeTypename) interfaces; { - Arena arena = arena_init_from_memory( interface_arr_mem, kilobytes(4) ); - interfaces = array_init_reserve(CodeTypename, arena_allocator_info(& arena), 4 ); - } + Array(CodeTypename) interfaces = {nullptr}; // TODO(Ed) : Make an AST_DerivedType, we'll store any arbitary derived type into there as a linear linked list of them. if ( check( Tok_Assign_Classifer ) ) @@ -743,27 +749,33 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( currtok.Type ); } - Token parent_tok = parse_identifier(nullptr); + Token parent_tok = parse_identifier(ctx, nullptr); parent = def_type( parent_tok.Text ); // : - while ( check(Tok_Comma) ) + if (check(Tok_Comma)) { - eat( Tok_Comma ); - // : , + Arena arena = arena_init_from_memory( interface_arr_mem, kilobytes(4) ); + interfaces = array_init_reserve(CodeTypename, arena_allocator_info(& arena), 4 ); + do + { + eat( Tok_Comma ); + // : , - if ( tok_is_access_specifier(currtok) ) { - eat(currtok.Type); - } - Token interface_tok = parse_identifier(nullptr); + if ( tok_is_access_specifier(currtok) ) { + eat(currtok.Type); + } + Token interface_tok = parse_identifier(ctx, nullptr); - array_append( interfaces, def_type( interface_tok.Text ) ); - // : , ... + array_append( interfaces, def_type( interface_tok.Text ) ); + // : , ... + } + while ( check(Tok_Comma) ); } } if ( check( Tok_BraceCurly_Open ) ) { - body = parse_class_struct_body( which, name ); + body = parse_class_struct_body( ctx, which, name ); } // : , ... { } @@ -775,27 +787,31 @@ Code parse_class_struct( TokType which, bool inplace_def ) // : , ... { }; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); + inline_cmt = parse_comment(ctx); // : , ... { }; } + s32 num_interfaces = scast(s32, interfaces ? array_num(interfaces) : 0); + if ( which == Tok_Decl_Class ) - result = cast(Code, def_class( name.Text, def_assign( body, parent, access, attributes, interfaces, scast(s32, array_num(interfaces)), specifiers, mflags ) )); + result = cast(Code, def_class( name.Text, def_assign( body, parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); else - result = cast(Code, def_struct( name.Text, def_assign( body, (CodeTypename)parent, access, attributes, interfaces, scast(s32, array_num(interfaces)), specifiers, mflags ) )); + result = cast(Code, def_struct( name.Text, def_assign( body, (CodeTypename)parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); if ( inline_cmt ) result->InlineCmt = cast(Code, inline_cmt); - array_free(interfaces); + if (interfaces) + array_free(interfaces); return result; } internal neverinline -CodeBody parse_class_struct_body( TokType which, Token name ) +CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); // { @@ -805,7 +821,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( which == Tok_Decl_Class ) result->Type = CT_Class_Body; - else result->Type = CT_Struct_Body; @@ -817,7 +832,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) bool expects_function = false; - // _ctx->parser.Scope->Start = currtok_noskip; + // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -826,7 +841,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure - log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(currtok_noskip)); + log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } @@ -836,7 +851,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Comment: { - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(ctx)); break; } case Tok_Access_Public: { @@ -861,47 +876,47 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Decl_Class: { - member = parse_complicated_definition( Tok_Decl_Class ); + member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; } case Tok_Decl_Enum: { - member = parse_complicated_definition( Tok_Decl_Enum ); + member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; } case Tok_Decl_Friend: { - member = cast(Code, parser_parse_friend()); + member = cast(Code, parser_parse_friend(ctx)); // friend break; } case Tok_Decl_Operator: { - member = cast(Code, parser_parse_operator_cast(NullCode)); + member = cast(Code, parser_parse_operator_cast(ctx, NullCode)); // operator () break; } case Tok_Decl_Struct: { - member = parse_complicated_definition( Tok_Decl_Struct ); + member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct break; } case Tok_Decl_Template: { - member = cast(Code, parser_parse_template()); + member = cast(Code, parser_parse_template(ctx)); // template< ... > break; } case Tok_Decl_Typedef: { - member = cast(Code, parser_parse_typedef()); + member = cast(Code, parser_parse_typedef(ctx)); // typedef break; } case Tok_Decl_Union: { - member = parse_complicated_definition( Tok_Decl_Union ); + member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union break; } case Tok_Decl_Using: { - member = cast(Code, parser_parse_using()); + member = cast(Code, parser_parse_using(ctx)); // using break; } @@ -909,22 +924,22 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { //if ( currtok.Text[0] != '~' ) //{ - // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(_ctx->parser) ); + // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(ctx->parser) ); // return InvalidCode; //} - member = cast(Code, parser_parse_destructor(NullCode)); + member = cast(Code, parser_parse_destructor(ctx, NullCode)); // ~() break; } case Tok_Preprocess_Define: { - member = cast(Code, parser_parse_define()); + member = cast(Code, parser_parse_define(ctx)); // #define break; } case Tok_Preprocess_Include: { - member = cast(Code, parse_include()); + member = cast(Code, parse_include(ctx)); // #include break; } @@ -933,7 +948,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) case Tok_Preprocess_IfDef: case Tok_Preprocess_IfNotDef: case Tok_Preprocess_ElIf: - member = cast(Code, parse_preprocess_cond()); + member = cast(Code, parse_preprocess_cond(ctx)); // # break; @@ -951,7 +966,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } case Tok_Preprocess_Macro_Stmt: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); + member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Stmt )); break; } @@ -962,19 +977,19 @@ CodeBody parse_class_struct_body( TokType which, Token name ) // break; case Tok_Preprocess_Pragma: { - member = cast(Code, parse_pragma()); + member = cast(Code, parse_pragma(ctx)); // #pragma break; } case Tok_Preprocess_Unsupported: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported )); + member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Unsupported)); // # break; } case Tok_StaticAssert: { - member = parse_static_assert(); + member = parse_static_assert(ctx); // static_assert break; } @@ -983,7 +998,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { if ( ! tok_is_attribute(currtok)) { - log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); return InvalidCode; } } @@ -995,7 +1010,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(); + attributes = parse_attributes(ctx); // } //! Fallthrough intended @@ -1024,8 +1039,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; default: - log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); - parser_pop(& _ctx->parser); + log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1047,11 +1062,11 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( tok_is_attribute(currtok) ) { // Unfortuantely Unreal has code where there is attirbutes before specifiers - CodeAttributes more_attributes = parse_attributes(); + CodeAttributes more_attributes = parse_attributes(ctx); if ( attributes ) { - StrBuilder fused = strbuilder_make_reserve( _ctx->Allocator_Temp, attributes->Content.Len + more_attributes->Content.Len ); + StrBuilder fused = strbuilder_make_reserve( ctx->Allocator_Temp, attributes->Content.Len + more_attributes->Content.Len ); strbuilder_append_fmt( & fused, "%SB %SB", attributes->Content, more_attributes->Content ); Str attrib_name = strbuilder_to_str(fused); @@ -1065,14 +1080,14 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '~' ) { - member = cast(Code, parser_parse_destructor( specifiers )); + member = cast(Code, parser_parse_destructor(ctx, specifiers )); // ~() break; } if ( currtok.Type == Tok_Decl_Operator ) { - member = cast(Code, parser_parse_operator_cast( specifiers )); + member = cast(Code, parser_parse_operator_cast(ctx, specifiers )); // operator () break; } @@ -1094,13 +1109,13 @@ CodeBody parse_class_struct_body( TokType which, Token name ) { if ( c_str_compare_len( name.Text.Ptr, currtok.Text.Ptr, name.Text.Len ) == 0 ) { - member = cast(Code, parser_parse_constructor( specifiers )); + member = cast(Code, parser_parse_constructor(ctx, specifiers )); // () break; } } - member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); + member = parse_operator_function_or_variable(ctx, expects_function, attributes, specifiers ); // operator ... // or // ... @@ -1121,8 +1136,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Failed to parse member\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } body_append(result, member ); @@ -1130,14 +1145,15 @@ CodeBody parse_class_struct_body( TokType which, Token name ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal -CodeComment parse_comment() -{ - push_scope(); +CodeComment parse_comment(Context* ctx) +{ + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); CodeComment result = (CodeComment) make_code(); @@ -1146,37 +1162,38 @@ CodeComment parse_comment() // result->Token = currtok_noskip; eat( Tok_Comment ); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal -Code parse_complicated_definition( TokType which ) +Code parse_complicated_definition(Context* ctx, TokType which) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); b32 is_inplace = false; b32 is_fn_def = false; - TokArray tokens = _ctx->parser.Tokens; + TokenSlice tokens = ctx->parser.tokens; - s32 idx = tokens.Idx; + s32 idx = ctx->parser.token_id; s32 level = 0; b32 had_def = false; b32 had_paren = false; - for ( ; idx < array_num(tokens.Arr); idx++ ) + for ( ; idx < tokens.num; idx++ ) { - if ( tokens.Arr[ idx ].Type == Tok_BraceCurly_Open ) + if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Open ) level++; - if ( tokens.Arr[ idx ].Type == Tok_BraceCurly_Close ) { + if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Close ) { level--; had_def = level == 0; } b32 found_fn_def = had_def && had_paren; - if ( level == 0 && (tokens.Arr[ idx ].Type == Tok_Statement_End || found_fn_def) ) + if ( level == 0 && (tokens.ptr[ idx ].Type == Tok_Statement_End || found_fn_def) ) break; } @@ -1184,52 +1201,52 @@ Code parse_complicated_definition( TokType which ) if (is_fn_def) { // Function definition with on return type - Code result = parse_operator_function_or_variable(false, NullCode, NullCode); + Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode); // (...) ... { ... } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } - if ( ( idx - 2 ) == tokens.Idx ) + if ( ( idx - 2 ) == ctx->parser.token_id ) { // It's a forward declaration only - Code result = parse_forward_or_definition( which, is_inplace ); + Code result = parse_forward_or_definition(ctx, which, is_inplace ); // ; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } - Token tok = tokens.Arr[ idx - 1 ]; + Token tok = tokens.ptr[ idx - 1 ]; if ( tok_is_specifier(tok) && spec_is_trailing( str_to_specifier( tok.Text)) ) { // (...) ...; s32 spec_idx = idx - 1; - Token spec = tokens.Arr[spec_idx]; + Token spec = tokens.ptr[spec_idx]; while ( tok_is_specifier(spec) && spec_is_trailing( str_to_specifier( spec.Text)) ) { -- spec_idx; - spec = tokens.Arr[spec_idx]; + spec = tokens.ptr[spec_idx]; } - if ( tokens.Arr[spec_idx].Type == Tok_Paren_Close ) + if ( tokens.ptr[spec_idx].Type == Tok_Paren_Close ) { // Forward declaration with trailing specifiers for a procedure - tok = tokens.Arr[spec_idx]; + tok = tokens.ptr[spec_idx]; - Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); + Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode); // , or Name> ... - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } if ( tok.Type == Tok_Identifier ) { - tok = tokens.Arr[ idx - 2 ]; + tok = tokens.ptr[ idx - 2 ]; bool is_indirection = tok.Type == Tok_Ampersand || tok.Type == Tok_Star; bool ok_to_parse = false; @@ -1240,33 +1257,33 @@ Code parse_complicated_definition( TokType which ) ok_to_parse = true; is_inplace = true; - CodeTypename type = cast(CodeTypename, parse_forward_or_definition(which, is_inplace)); + CodeTypename type = cast(CodeTypename, parse_forward_or_definition(ctx, which, is_inplace)); // Should be a name right after the type. - Token name = parse_identifier(nullptr); - _ctx->parser.Scope->Name = name.Text; + Token name = parse_identifier(ctx, nullptr); + ctx->parser.scope->name = name.Text; - CodeVar result = parse_variable_after_name(ModuleFlag_None, NullCode, NullCode, type, name.Text); - parser_pop(& _ctx->parser); + CodeVar result = parse_variable_after_name(ctx, ModuleFlag_None, NullCode, NullCode, type, name.Text); + parser_pop(& ctx->parser); return (Code) result; } - else if ( tok.Type == Tok_Identifier && tokens.Arr[ idx - 3 ].Type == which ) + else if ( tok.Type == Tok_Identifier && tokens.ptr[ idx - 3 ].Type == which ) { // Its a variable with type ID using namespace. // ; ok_to_parse = true; } else if ( tok.Type == Tok_Assign_Classifer - && ( ( tokens.Arr[idx - 5].Type == which && tokens.Arr[idx - 4].Type == Tok_Decl_Class ) - || ( tokens.Arr[idx - 4].Type == which)) + && ( ( tokens.ptr[idx - 5].Type == which && tokens.ptr[idx - 4].Type == Tok_Decl_Class ) + || ( tokens.ptr[idx - 4].Type == which)) ) { // Its a forward declaration of an enum // : ; // : ; ok_to_parse = true; - Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& _ctx->parser); + Code result = cast(Code, parser_parse_enum(ctx, ! parser_inplace_def)); + parser_pop(& ctx->parser); return result; } else if ( is_indirection ) @@ -1278,64 +1295,67 @@ Code parse_complicated_definition( TokType which ) if ( ! ok_to_parse ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } - Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); + Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode ); // , or Name> ... - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( tok.Type >= Tok_Type_Unsigned && tok.Type <= Tok_Type_MS_W64 ) { - tok = tokens.Arr[ idx - 2 ]; + tok = tokens.ptr[ idx - 2 ]; if ( tok.Type != Tok_Assign_Classifer - || ( ( tokens.Arr[idx - 5].Type != which && tokens.Arr[idx - 4].Type != Tok_Decl_Class ) - && ( tokens.Arr[idx - 4].Type != which)) + || ( ( tokens.ptr[idx - 5].Type != which && tokens.ptr[idx - 4].Type != Tok_Decl_Class ) + && ( tokens.ptr[idx - 4].Type != which)) ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } // Its a forward declaration of an enum class // : ; // : ; - Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& _ctx->parser); + Code result = cast(Code, parser_parse_enum(ctx, ! parser_inplace_def)); + parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceCurly_Close ) { // Its a definition - Code result = parse_forward_or_definition( which, is_inplace ); + Code result = parse_forward_or_definition(ctx, which, is_inplace ); // { ... }; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceSquare_Close ) { // Its an array definition - Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); + Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode ); // [ ... ]; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } else { - log_failure( "Unsupported or bad member definition after %s declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } } internal inline -Code parse_assignment_expression() +Code parse_assignment_expression(Context* ctx) { + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + Code expr = { nullptr }; eat( Tok_Operator ); @@ -1345,8 +1365,8 @@ Code parse_assignment_expression() if ( currtok.Type == Tok_Statement_End && currtok.Type != Tok_Comma ) { - log_failure( "Expected expression after assignment operator\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Expected expression after assignment operator\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1370,36 +1390,38 @@ Code parse_assignment_expression() } expr = untyped_str( expr_tok.Text ); // = + + parser_pop(& ctx->parser); return expr; } internal inline -Code parse_forward_or_definition( TokType which, bool is_inplace ) +Code parse_forward_or_definition(Context* ctx, TokType which, bool is_inplace ) { Code result = InvalidCode; switch ( which ) { case Tok_Decl_Class: - result = cast(Code, parser_parse_class( is_inplace )); + result = cast(Code, parser_parse_class(ctx, is_inplace )); return result; case Tok_Decl_Enum: - result = cast(Code, parser_parse_enum( is_inplace )); + result = cast(Code, parser_parse_enum(ctx, is_inplace )); return result; case Tok_Decl_Struct: - result = cast(Code, parser_parse_struct( is_inplace )); + result = cast(Code, parser_parse_struct(ctx, is_inplace )); return result; case Tok_Decl_Union: - result = cast(Code, parser_parse_union( is_inplace )); + result = cast(Code, parser_parse_union(ctx, is_inplace )); return result; default: log_failure( "Error, wrong token type given to parse_complicated_definition " - "(only supports class, enum, struct, union) \n%s" - , parser_to_strbuilder(_ctx->parser) ); + "(only supports class, enum, struct, union) \n%SB" + , parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); return InvalidCode; } @@ -1407,16 +1429,18 @@ Code parse_forward_or_definition( TokType which, bool is_inplace ) // Function parsing is handled in multiple places because its initial signature is shared with variable parsing internal inline -CodeFn parse_function_after_name( - ModuleFlag mflags +CodeFn parse_function_after_name(Context* ctx + , ModuleFlag mflags , CodeAttributes attributes , CodeSpecifiers specifiers , CodeTypename ret_type , Token name ) { - push_scope(); - CodeParams params = parse_params(parser_use_parenthesis); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + + CodeParams params = parse_params(ctx, parser_use_parenthesis); // ( ) Code suffix_specs = NullCode; @@ -1428,7 +1452,7 @@ CodeFn parse_function_after_name( Macro* macro = lookup_macro( currtok.Text ); if (macro && tok_is_specifier(currtok)) { - suffix_specs = parse_simple_preprocess(Tok_Preprocess_Macro_Expr); + suffix_specs = parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Expr); continue; } if ( specifiers == nullptr ) @@ -1443,14 +1467,30 @@ CodeFn parse_function_after_name( } // ( ) + // Check for trailing specifiers... + CodeAttributes post_rt_attributes = parse_attributes(ctx); + if (post_rt_attributes) + { + if (attributes) + { + StrBuilder merged = strbuilder_fmt_buf(ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content); + attributes->Content = cache_str(strbuilder_to_str(merged)); + } + else + { + attributes = post_rt_attributes; + } + } + // ( ) + CodeBody body = NullCode; CodeComment inline_cmt = NullCode; if ( check( Tok_BraceCurly_Open ) ) { - body = cast(CodeBody, parse_function_body()); + body = cast(CodeBody, parse_function_body(ctx)); if ( cast(Code, body) == Code_Invalid ) { - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return InvalidCode; } // ( ) { } @@ -1480,22 +1520,24 @@ CodeFn parse_function_after_name( eat( Tok_Statement_End ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - // ( ) ; + inline_cmt = parse_comment(ctx); + // ( ) < = 0 or delete > ; } - else + + + if (body == nullptr) { Token stmt_end = currtok; eat( Tok_Statement_End ); - // ( ) ; + // ( ) < = 0 or delete > ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - // ( ) ; + inline_cmt = parse_comment(ctx); + // ( ) < = 0 or delete > ; } StrBuilder - name_stripped = strbuilder_make_str( _ctx->Allocator_Temp, name.Text ); + name_stripped = strbuilder_make_str( ctx->Allocator_Temp, name.Text ); strbuilder_strip_space(name_stripped); CodeFn @@ -1513,8 +1555,8 @@ CodeFn parse_function_after_name( default: { - log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", code_debug_str(body), parser_to_strbuilder(_ctx->parser)); - parser_pop(& _ctx->parser); + log_failure("Body must be either of Function_Body or Untyped type, %S\n%SB", code_debug_str(body), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); + parser_pop(& ctx->parser); return InvalidCode; } } @@ -1544,14 +1586,15 @@ CodeFn parse_function_after_name( if ( inline_cmt ) result->InlineCmt = inline_cmt; - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal -Code parse_function_body() +Code parse_function_body(Context* ctx) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); @@ -1586,14 +1629,15 @@ Code parse_function_body() eat( Tok_BraceCurly_Close ); - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return cast(Code, result); } internal neverinline -CodeBody parse_global_nspace( CodeType which ) +CodeBody parse_global_nspace(Context* ctx, CodeType which) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); if ( which != CT_Namespace_Body && which != CT_Global_Body && which != CT_Export_Body && which != CT_Extern_Linkage_Body ) return InvalidCode; @@ -1614,7 +1658,7 @@ CodeBody parse_global_nspace( CodeType which ) bool expects_function = false; - // _ctx->parser.Scope->Start = currtok_noskip; + // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -1625,15 +1669,15 @@ CodeBody parse_global_nspace( CodeType which ) { case Tok_Comma: { - log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(_ctx->parser)); - parser_pop( & _ctx->parser); + log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); + parser_pop( & ctx->parser); return InvalidCode; } break; case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure - log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(currtok_noskip)); + log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } @@ -1644,64 +1688,64 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Comment: - member = cast(Code, parse_comment()); + member = cast(Code, parse_comment(ctx)); break; case Tok_Decl_Class: - member = parse_complicated_definition( Tok_Decl_Class ); + member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; case Tok_Decl_Enum: - member = parse_complicated_definition( Tok_Decl_Enum ); + member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; case Tok_Decl_Extern_Linkage: if ( which == CT_Extern_Linkage_Body ) - log_failure( "Nested extern linkage\n%s", parser_to_strbuilder(_ctx->parser) ); + log_failure( "Nested extern linkage\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); - member = cast(Code, parser_parse_extern_link()); + member = cast(Code, parser_parse_extern_link(ctx)); // extern "..." { ... } break; case Tok_Decl_Namespace: - member = cast(Code, parser_parse_namespace()); + member = cast(Code, parser_parse_namespace(ctx)); // namespace { ... } break; case Tok_Decl_Struct: - member = parse_complicated_definition( Tok_Decl_Struct ); + member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct ... break; case Tok_Decl_Template: - member = cast(Code, parser_parse_template()); + member = cast(Code, parser_parse_template(ctx)); // template<...> ... break; case Tok_Decl_Typedef: - member = cast(Code, parser_parse_typedef()); + member = cast(Code, parser_parse_typedef(ctx)); // typedef ... break; case Tok_Decl_Union: - member = parse_complicated_definition( Tok_Decl_Union ); + member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union ... break; case Tok_Decl_Using: - member = cast(Code, parser_parse_using()); + member = cast(Code, parser_parse_using(ctx)); // using ... break; case Tok_Preprocess_Define: - member = cast(Code, parser_parse_define()); + member = cast(Code, parser_parse_define(ctx)); // #define ... break; case Tok_Preprocess_Include: - member = cast(Code, parse_include()); + member = cast(Code, parse_include(ctx)); // #include ... break; @@ -1709,7 +1753,7 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Preprocess_IfDef: case Tok_Preprocess_IfNotDef: case Tok_Preprocess_ElIf: - member = cast(Code, parse_preprocess_cond()); + member = cast(Code, parse_preprocess_cond(ctx)); // # ... break; @@ -1726,33 +1770,33 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Preprocess_Macro_Stmt: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); + member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Stmt )); break; } case Tok_Preprocess_Pragma: { - member = cast(Code, parse_pragma()); + member = cast(Code, parse_pragma(ctx)); // #pragma ... } break; case Tok_Preprocess_Unsupported: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported )); + member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Unsupported )); // # ... } break; case Tok_StaticAssert: { - member = cast(Code, parse_static_assert()); + member = cast(Code, parse_static_assert(ctx)); // static_assert( , ... ); } break; case Tok_Module_Export: { if ( which == CT_Export_Body ) - log_failure( "Nested export declaration\n%s", parser_to_strbuilder(_ctx->parser) ); + log_failure( "Nested export declaration\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); - member = cast(Code, parser_parse_export_body()); + member = cast(Code, parser_parse_export_body(ctx)); // export { ... } } break; @@ -1766,9 +1810,9 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Preprocess_Macro_Expr: { - if (tok_is_attribute(currtok)) + if ( ! tok_is_attribute(currtok)) { - log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + log_failure("Unbounded macro expression residing in class/struct body\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); return InvalidCode; } } @@ -1780,7 +1824,7 @@ CodeBody parse_global_nspace( CodeType which ) GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { - attributes = parse_attributes(); + attributes = parse_attributes(ctx); // } //! Fallthrough intentional @@ -1811,8 +1855,8 @@ CodeBody parse_global_nspace( CodeType which ) default: Str spec_str = spec_to_str(spec); - log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); - parser_pop(& _ctx->parser); + log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1845,7 +1889,7 @@ CodeBody parse_global_nspace( CodeType which ) { // This s only in a scope so that Preprocess_Macro_Bare_In_Body works without microsoft extension warnings { - Code constructor_destructor = parse_global_nspace_constructor_destructor( specifiers ); + Code constructor_destructor = parse_global_nspace_constructor_destructor( ctx, specifiers ); // Possible constructor implemented at global file scope. if ( constructor_destructor ) { @@ -1854,16 +1898,16 @@ CodeBody parse_global_nspace( CodeType which ) } bool found_operator_cast_outside_class_implmentation = false; - s32 idx = _ctx->parser.Tokens.Idx; + s32 idx = ctx->parser.token_id; - for ( ; idx < array_num(_ctx->parser.Tokens.Arr); idx++ ) + for ( ; idx < ctx->parser.tokens.num; idx++ ) { - Token tok = _ctx->parser.Tokens.Arr[ idx ]; + Token tok = ctx->parser.tokens.ptr[ idx ]; if ( tok.Type == Tok_Identifier ) { idx++; - tok = _ctx->parser.Tokens.Arr[ idx ]; + tok = ctx->parser.tokens.ptr[ idx ]; if ( tok.Type == Tok_Access_StaticSymbol ) continue; @@ -1878,21 +1922,21 @@ CodeBody parse_global_nspace( CodeType which ) if ( found_operator_cast_outside_class_implmentation ) { - member = cast(Code, parser_parse_operator_cast( specifiers )); + member = cast(Code, parser_parse_operator_cast(ctx, specifiers )); // ::operator () { ... } break; } } - member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); + member = parse_operator_function_or_variable(ctx, expects_function, attributes, specifiers ); // ... } } if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); + log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + parser_pop(& ctx->parser); return InvalidCode; } @@ -1904,13 +1948,16 @@ CodeBody parse_global_nspace( CodeType which ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& _ctx->parser); + parser_pop(& ctx->parser); return result; } internal inline -Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) +Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers) { + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); + Code result = { nullptr }; /* @@ -1925,18 +1972,18 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) TODO(Ed): We could fix this by attempting to parse a type, but we would have to have a way to have it soft fail and rollback. */ - TokArray tokens = _ctx->parser.Tokens; + TokenSlice tokens = ctx->parser.tokens; - s32 idx = tokens.Idx; - Token nav = tokens.Arr[ idx ]; - for ( ; idx < array_num(tokens.Arr); idx++, nav = tokens.Arr[ idx ] ) + s32 idx = ctx->parser.token_id; + Token nav = tokens.ptr[ idx ]; + for ( ; idx < tokens.num; idx++, nav = tokens.ptr[ idx ] ) { if ( nav.Text.Ptr[0] == '<' ) { // Skip templated expressions as they mey have expressions with the () operators s32 capture_level = 0; s32 template_level = 0; - for ( ; idx < array_num(tokens.Arr); idx++, nav = tokens.Arr[idx] ) + for ( ; idx < tokens.num; idx++, nav = tokens.ptr[idx] ) { if (nav.Text.Ptr[ 0 ] == '<') ++ template_level; @@ -1964,17 +2011,18 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) } -- idx; - Token tok_right = tokens.Arr[idx]; + Token tok_right = tokens.ptr[idx]; Token tok_left = NullToken; if (tok_right.Type != Tok_Identifier) { + parser_pop(& ctx->parser); // We're not dealing with a constructor if there is no identifier right before the opening of a parameter's scope. return result; } -- idx; - tok_left = tokens.Arr[idx]; + tok_left = tokens.ptr[idx]; // ... bool possible_destructor = false; @@ -1982,20 +2030,22 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) { possible_destructor = true; -- idx; - tok_left = tokens.Arr[idx]; + tok_left = tokens.ptr[idx]; } - if ( tok_left.Type != Tok_Access_StaticSymbol ) + if ( tok_left.Type != Tok_Access_StaticSymbol ) { + parser_pop(& ctx->parser); return result; + } -- idx; - tok_left = tokens.Arr[idx]; + tok_left = tokens.ptr[idx]; // ... :: // We search toward the left until we find the next valid identifier s32 capture_level = 0; s32 template_level = 0; - while ( idx != tokens.Idx ) + while ( idx != ctx->parser.token_id ) { if (tok_left.Text.Ptr[ 0 ] == '<') ++ template_level; @@ -2015,7 +2065,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) break; -- idx; - tok_left = tokens.Arr[idx]; + tok_left = tokens.ptr[idx]; } bool is_same = c_str_compare_len( tok_right.Text.Ptr, tok_left.Text.Ptr, tok_right.Text.Len ) == 0; @@ -2025,14 +2075,15 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) if (possible_destructor) { // :: ~ ( - result = cast(Code, parser_parse_destructor( specifiers )); + result = cast(Code, parser_parse_destructor(ctx, specifiers )); } else { // :: ( - result = cast(Code, parser_parse_constructor( specifiers )); + result = cast(Code, parser_parse_constructor(ctx, specifiers )); } } + parser_pop(& ctx->parser); return result; } @@ -2040,23 +2091,33 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) // This would allow distinction of the qualifier for a symbol :: // This would also allow internal -Token parse_identifier( bool* possible_member_function ) +Token parse_identifier(Context* ctx, bool* possible_member_function) { - push_scope(); + ParseStackNode scope = NullScope; + parser_push(& ctx->parser, & scope ); Token name = currtok; - _ctx->parser.Scope->Name = name.Text; + ctx->parser.scope->name = name.Text; Macro* macro = lookup_macro(currtok.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); + b32 is_decarator = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Identifier_Decorator ); // Typename can be: '::' // If that is the case first option will be Tok_Access_StaticSymbol below - if (check(Tok_Identifier) || accept_as_identifier) - eat( Tok_Identifier ); + if (check(Tok_Identifier) || accept_as_identifier) + { + if (is_decarator) { + Code name_macro = parse_simple_preprocess(ctx, currtok.Type); + name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; + } + else { + eat(Tok_Identifier); + } + } // - parse_template_args( & name ); + parse_template_args(ctx, & name); //