From 213b2fd0f8059751b0cdd05b904c49109b21d474 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 22 Feb 2024 14:01:23 +0000 Subject: Add `bit_field` as a keyword --- src/tokenizer.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index dd9908be5..3d5348074 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -106,6 +106,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \ TOKEN_KIND(Token_union, "union"), \ TOKEN_KIND(Token_enum, "enum"), \ TOKEN_KIND(Token_bit_set, "bit_set"), \ + TOKEN_KIND(Token_bit_field, "bit_field"), \ TOKEN_KIND(Token_map, "map"), \ TOKEN_KIND(Token_dynamic, "dynamic"), \ TOKEN_KIND(Token_auto_cast, "auto_cast"), \ -- cgit v1.2.3 From 46b9bd8c0e3987080f94ae42921b513a79708ef9 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 12 Apr 2024 13:35:14 +0100 Subject: Improve error messages for `switch` and `for` r-values with a suggestion --- src/check_stmt.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ src/error.cpp | 21 ++++++++++++++++----- src/tokenizer.cpp | 1 + 3 files changed, 60 insertions(+), 5 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index a6def5997..f2b7f8661 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -474,16 +474,59 @@ gb_internal Type *check_assignment_variable(CheckerContext *ctx, Operand *lhs, O } Entity *e = entity_of_node(lhs->expr); + Entity *original_e = e; + + Ast *name = unparen_expr(lhs->expr); + while (name->kind == Ast_SelectorExpr) { + name = name->SelectorExpr.expr; + e = entity_of_node(name); + } + if (e == nullptr) { + e = original_e; + } gbString str = expr_to_string(lhs->expr); if (e != nullptr && e->flags & EntityFlag_Param) { + ERROR_BLOCK(); if (e->flags & EntityFlag_Using) { error(lhs->expr, "Cannot assign to '%s' which is from a 'using' procedure parameter", str); } else { error(lhs->expr, "Cannot assign to '%s' which is a procedure parameter", str); } + error_line("\tSuggestion: Did you mean to pass '%.*s' by pointer?\n", LIT(e->token.string)); + show_error_on_line(e->token.pos, token_pos_end(e->token)); } else { + ERROR_BLOCK(); error(lhs->expr, "Cannot assign to '%s'", str); + + if (e) if (e->flags & EntityFlag_ForValue) { + isize offset = show_error_on_line(e->token.pos, token_pos_end(e->token), "Suggestion:"); + if (offset < 0) { + if (is_type_map(e->type)) { + error_line("\tSuggestion: Did you mean? 'for key, &%.*s in ...'\n", LIT(e->token.string)); + } else { + error_line("\tSuggestion: Did you mean? 'for &%.*s in ...'\n", LIT(e->token.string)); + } + } else { + error_line("\t"); + for (isize i = 0; i < offset-1; i++) { + error_line(" "); + } + error_line("'%.*s' is immutable, declare it as '&%.*s' to make it mutable\n", LIT(e->token.string), LIT(e->token.string)); + } + + } else if (e->flags & EntityFlag_SwitchValue) { + isize offset = show_error_on_line(e->token.pos, token_pos_end(e->token), "Suggestion:"); + if (offset < 0) { + error_line("\tSuggestion: Did you mean? 'switch &%.*s in ...'\n", LIT(e->token.string)); + } else { + error_line("\t"); + for (isize i = 0; i < offset-1; i++) { + error_line(" "); + } + error_line("'%.*s' is immutable, declare it as '&%.*s' to make it mutable\n", LIT(e->token.string), LIT(e->token.string)); + } + } } gb_string_free(str); diff --git a/src/error.cpp b/src/error.cpp index eb167d4c3..8647f60b9 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -84,6 +84,7 @@ gb_internal bool set_file_path_string(i32 index, String const &path) { bool ok = false; GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); if (index >= global_file_path_strings.count) { array_resize(&global_file_path_strings, index+1); @@ -94,6 +95,7 @@ gb_internal bool set_file_path_string(i32 index, String const &path) { ok = true; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return ok; } @@ -102,6 +104,7 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { bool ok = false; GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); if (index >= global_files.count) { array_resize(&global_files, index+1); @@ -111,7 +114,7 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { global_files[index] = file; ok = true; } - + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return ok; } @@ -119,12 +122,14 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { gb_internal String get_file_path_string(i32 index) { GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); String path = {}; if (index < global_file_path_strings.count) { path = global_file_path_strings[index]; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return path; } @@ -132,12 +137,14 @@ gb_internal String get_file_path_string(i32 index) { gb_internal AstFile *thread_safe_get_ast_file_from_id(i32 index) { GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); AstFile *file = nullptr; if (index < global_files.count) { file = global_files[index]; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return file; } @@ -247,10 +254,10 @@ gb_internal void terminal_reset_colours(void) { } -gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { +gb_internal isize show_error_on_line(TokenPos const &pos, TokenPos end, char const *prefix=nullptr) { get_error_value()->end = end; if (!show_error_line()) { - return false; + return -1; } i32 offset = 0; @@ -270,6 +277,10 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { MAX_LINE_LENGTH_PADDED = MAX_LINE_LENGTH-MAX_TAB_WIDTH-ELLIPSIS_PADDING, }; + if (prefix) { + error_out("\t%s\n\n", prefix); + } + error_out("\t"); terminal_set_colours(TerminalStyle_Bold, TerminalColour_White); @@ -328,9 +339,9 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { terminal_reset_colours(); error_out("\n"); - return true; + return offset; } - return false; + return -1; } gb_internal void error_out_empty(void) { diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 3d5348074..fdff9224a 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -193,6 +193,7 @@ gb_internal void init_keyword_hash_table(void) { gb_global Array global_file_path_strings; // index is file id gb_global Array global_files; // index is file id +gb_global BlockingMutex global_files_mutex; gb_internal String get_file_path_string(i32 index); gb_internal struct AstFile *thread_safe_get_ast_file_from_id(i32 index); -- cgit v1.2.3 From 29987c20c058fe269ea0961205d12f51fc3e8326 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 28 Apr 2024 06:59:55 -0400 Subject: Fix invalid rune literal reported twice The tokenizer and the parser were reporting it in different positions. This way, they'll report in the same spot. --- src/tokenizer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index fdff9224a..f7751d840 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -767,9 +767,8 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { } } - // TODO(bill): Better Error Handling if (valid && n != 1) { - tokenizer_err(t, "Invalid rune literal"); + tokenizer_err(t, token->pos, "Invalid rune literal"); } token->string.len = t->curr - token->string.text; goto semicolon_check; -- cgit v1.2.3 From 582154f20d44ebf724301316e8cf27b1c768b7ee Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Fri, 14 Jun 2024 15:44:50 -0400 Subject: Remove unused code from tokenizer --- src/tokenizer.cpp | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index f7751d840..236bc84a8 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -777,7 +777,6 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { case '`': // Raw String Literal case '"': // String Literal { - bool has_carriage_return = false; i32 success; Rune quote = curr_rune; token->kind = Token_String; @@ -807,9 +806,6 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { if (r == quote) { break; } - if (r == '\r') { - has_carriage_return = true; - } } } token->string.len = t->curr - token->string.text; -- cgit v1.2.3 From 862a04376f589b23c34700d8e7c746048741e19f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 28 Jun 2024 09:16:01 +0100 Subject: Improve tokenizing wrong number literals --- src/parser.cpp | 12 +++++++++++- src/tokenizer.cpp | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'src/tokenizer.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 0cd96f5b5..0364e2c2b 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -718,7 +718,17 @@ gb_internal ExactValue exact_value_from_token(AstFile *f, Token const &token) { } ExactValue value = exact_value_from_basic_literal(token.kind, s); if (value.kind == ExactValue_Invalid) { - syntax_error(token, "Invalid token literal"); + switch (token.kind) { + case Token_Integer: + syntax_error(token, "Invalid integer literal"); + break; + case Token_Float: + syntax_error(token, "Invalid float literal"); + break; + default: + syntax_error(token, "Invalid token literal"); + break; + } } return value; } diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index f7751d840..1a37043d7 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -433,6 +433,7 @@ gb_internal gb_inline i32 digit_value(Rune r) { } gb_internal gb_inline void scan_mantissa(Tokenizer *t, i32 base) { + base = 16; // always check for any possible letter while (digit_value(t->curr_rune) < base || t->curr_rune == '_') { advance_to_next_rune(t); } -- cgit v1.2.3 From dde7cb6e7f1c9196286c924fae9e06f6a8f9b43f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 28 Jun 2024 09:21:33 +0100 Subject: Improve error message for #3754 --- src/tokenizer.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 1a37043d7..8458062b6 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -470,6 +470,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 2); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid binary integer"); token->kind = Token_Invalid; } goto end; @@ -477,6 +478,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 8); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid octal integer"); token->kind = Token_Invalid; } goto end; @@ -484,6 +486,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 10); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid explicitly decimal integer"); token->kind = Token_Invalid; } goto end; @@ -491,6 +494,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 12); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid dozenal integer"); token->kind = Token_Invalid; } goto end; @@ -498,6 +502,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 16); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid hexadecimal integer"); token->kind = Token_Invalid; } goto end; @@ -506,6 +511,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci advance_to_next_rune(t); scan_mantissa(t, 16); if (t->curr - prev <= 2) { + tokenizer_err(t, "Invalid hexadecimal float"); token->kind = Token_Invalid; } else { u8 *start = prev+2; -- cgit v1.2.3 From 35651cfc173bc42f1afc19c4e2cec46501808dd0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 28 Jun 2024 09:24:34 +0100 Subject: Improve tokenization for false integer literals but not effect possible float literals --- src/tokenizer.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 8458062b6..2af41b881 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -432,8 +432,10 @@ gb_internal gb_inline i32 digit_value(Rune r) { return 16; // NOTE(bill): Larger than highest possible } -gb_internal gb_inline void scan_mantissa(Tokenizer *t, i32 base) { - base = 16; // always check for any possible letter +gb_internal gb_inline void scan_mantissa(Tokenizer *t, i32 base, bool force_base) { + if (!force_base) { + base = 16; // always check for any possible letter + } while (digit_value(t->curr_rune) < base || t->curr_rune == '_') { advance_to_next_rune(t); } @@ -458,7 +460,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci token->string.len += 1; token->pos.column -= 1; token->kind = Token_Float; - scan_mantissa(t, 10); + scan_mantissa(t, 10, true); goto exponent; } @@ -468,7 +470,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci switch (t->curr_rune) { case 'b': // Binary advance_to_next_rune(t); - scan_mantissa(t, 2); + scan_mantissa(t, 2, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid binary integer"); token->kind = Token_Invalid; @@ -476,7 +478,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci goto end; case 'o': // Octal advance_to_next_rune(t); - scan_mantissa(t, 8); + scan_mantissa(t, 8, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid octal integer"); token->kind = Token_Invalid; @@ -484,7 +486,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci goto end; case 'd': // Decimal advance_to_next_rune(t); - scan_mantissa(t, 10); + scan_mantissa(t, 10, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid explicitly decimal integer"); token->kind = Token_Invalid; @@ -492,7 +494,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci goto end; case 'z': // Dozenal advance_to_next_rune(t); - scan_mantissa(t, 12); + scan_mantissa(t, 12, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid dozenal integer"); token->kind = Token_Invalid; @@ -500,7 +502,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci goto end; case 'x': // Hexadecimal advance_to_next_rune(t); - scan_mantissa(t, 16); + scan_mantissa(t, 16, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid hexadecimal integer"); token->kind = Token_Invalid; @@ -509,7 +511,7 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci case 'h': // Hexadecimal Float token->kind = Token_Float; advance_to_next_rune(t); - scan_mantissa(t, 16); + scan_mantissa(t, 16, false); if (t->curr - prev <= 2) { tokenizer_err(t, "Invalid hexadecimal float"); token->kind = Token_Invalid; @@ -534,12 +536,12 @@ gb_internal void scan_number_to_token(Tokenizer *t, Token *token, bool seen_deci } goto end; default: - scan_mantissa(t, 10); + scan_mantissa(t, 10, true); goto fraction; } } - scan_mantissa(t, 10); + scan_mantissa(t, 10, true); fraction: @@ -551,7 +553,7 @@ fraction: advance_to_next_rune(t); token->kind = Token_Float; - scan_mantissa(t, 10); + scan_mantissa(t, 10, true); } exponent: @@ -561,7 +563,7 @@ exponent: if (t->curr_rune == '-' || t->curr_rune == '+') { advance_to_next_rune(t); } - scan_mantissa(t, 10); + scan_mantissa(t, 10, false); } switch (t->curr_rune) { -- cgit v1.2.3 From dc767da12b0e03a6cd9ff20085565c95c06ef7bc Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Thu, 5 Sep 2024 21:17:40 +0200 Subject: Make tags use #+ syntax instead of //+ syntax so it no longer looks like a comment. Old style still works but is deprecated with a warning. Using unknown tags is now an error instead of a warning. There is a new token for #+ which consumes the whole line (or until it hits a comment). The tags are parsed like before. There are errors to tell you if you use something invalid in the pre-package-line block. --- src/parser.cpp | 155 +++++++++++++++++++++++++++++++++++++----------------- src/tokenizer.cpp | 15 ++++++ 2 files changed, 123 insertions(+), 47 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index e3143dd33..03ed725c5 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -5337,6 +5337,12 @@ gb_internal Ast *parse_stmt(AstFile *f) { s = ast_empty_stmt(f, token); expect_semicolon(f); return s; + + case Token_FileTag: + // This is always an error because all valid file tags will have been processed in `parse_file` already. + // Any remaining file tags must be past the package line and thus invalid. + syntax_error(token, "Lines starting with #+ (file tags) are only allowed before the package line."); + return ast_bad_stmt(f, token, f->curr_token); } // Error correction statements @@ -6091,7 +6097,7 @@ gb_internal String build_tag_get_token(String s, String *out) { } gb_internal bool parse_build_tag(Token token_for_pos, String s) { - String const prefix = str_lit("+build"); + String const prefix = str_lit("build"); GB_ASSERT(string_starts_with(s, prefix)); s = string_trim_whitespace(substring(s, prefix.len, s.len)); @@ -6176,7 +6182,7 @@ gb_internal String vet_tag_get_token(String s, String *out) { gb_internal u64 parse_vet_tag(Token token_for_pos, String s) { - String const prefix = str_lit("+vet"); + String const prefix = str_lit("vet"); GB_ASSERT(string_starts_with(s, prefix)); s = string_trim_whitespace(substring(s, prefix.len, s.len)); @@ -6281,7 +6287,7 @@ gb_internal isize calc_decl_count(Ast *decl) { } gb_internal bool parse_build_project_directory_tag(Token token_for_pos, String s) { - String const prefix = str_lit("+build-project-name"); + String const prefix = str_lit("build-project-name"); GB_ASSERT(string_starts_with(s, prefix)); s = string_trim_whitespace(substring(s, prefix.len, s.len)); if (s.len == 0) { @@ -6325,6 +6331,48 @@ gb_internal bool parse_build_project_directory_tag(Token token_for_pos, String s return any_correct; } +gb_internal bool process_file_tag(const String &lc, const Token &tok, AstFile *f) { + if (string_starts_with(lc, str_lit("build-project-name"))) { + if (!parse_build_project_directory_tag(tok, lc)) { + return false; + } + } else if (string_starts_with(lc, str_lit("build"))) { + if (!parse_build_tag(tok, lc)) { + return false; + } + } else if (string_starts_with(lc, str_lit("vet"))) { + f->vet_flags = parse_vet_tag(tok, lc); + f->vet_flags_set = true; + } else if (string_starts_with(lc, str_lit("ignore"))) { + return false; + } else if (string_starts_with(lc, str_lit("private"))) { + f->flags |= AstFile_IsPrivatePkg; + String command = string_trim_starts_with(lc, str_lit("private ")); + command = string_trim_whitespace(command); + if (lc == "private") { + f->flags |= AstFile_IsPrivatePkg; + } else if (command == "package") { + f->flags |= AstFile_IsPrivatePkg; + } else if (command == "file") { + f->flags |= AstFile_IsPrivateFile; + } + } else if (lc == "lazy") { + if (build_context.ignore_lazy) { + // Ignore + } else if (f->pkg->kind == Package_Init && build_context.command_kind == Command_doc) { + // Ignore + } else { + f->flags |= AstFile_IsLazy; + } + } else if (lc == "no-instrumentation") { + f->flags |= AstFile_NoInstrumentation; + } else { + error(tok, "Unknown tag '%.*s'", LIT(lc)); + } + + return true; +} + gb_internal bool parse_file(Parser *p, AstFile *f) { if (f->tokens.count == 0) { return true; @@ -6337,15 +6385,38 @@ gb_internal bool parse_file(Parser *p, AstFile *f) { String filepath = f->tokenizer.fullpath; String base_dir = dir_from_path(filepath); - if (f->curr_token.kind == Token_Comment) { - consume_comment_groups(f, f->prev_token); + + Array tags = array_make(ast_allocator(f)); + + bool has_first_invalid_pre_package_token = false; + Token first_invalid_pre_package_token; + + while (f->curr_token.kind != Token_package && f->curr_token.kind != Token_EOF) { + if (f->curr_token.kind == Token_Comment) { + consume_comment_groups(f, f->prev_token); + } else if (f->curr_token.kind == Token_FileTag) { + array_add(&tags, f->curr_token); + advance_token(f); + } else { + if (!has_first_invalid_pre_package_token) { + has_first_invalid_pre_package_token = true; + first_invalid_pre_package_token = f->curr_token; + } + + advance_token(f); + } } CommentGroup *docs = f->lead_comment; if (f->curr_token.kind != Token_package) { ERROR_BLOCK(); - syntax_error(f->curr_token, "Expected a package declaration at the beginning of the file"); + + // The while loop above scanned until it found the package token. If we never + // found one, then make this error appear on the first invalid token line. + Token t = has_first_invalid_pre_package_token ? first_invalid_pre_package_token : f->curr_token; + syntax_error(t, "Expected a package declaration at the beginning of the file"); + // IMPORTANT NOTE(bill): this is technically a race condition with the suggestion, but it's ony a suggession // so in practice is should be "fine" if (f->pkg && f->pkg->name != "") { @@ -6354,6 +6425,12 @@ gb_internal bool parse_file(Parser *p, AstFile *f) { return false; } + // There was an OK package declaration. But there some invalid token was hit before the package declaration. + if (has_first_invalid_pre_package_token) { + syntax_error(first_invalid_pre_package_token, "There can only be lines starting with #+ or // before package declaration"); + return false; + } + f->package_token = expect_token(f, Token_package); if (f->package_token.kind != Token_package) { return false; @@ -6379,55 +6456,39 @@ gb_internal bool parse_file(Parser *p, AstFile *f) { } f->package_name = package_name.string; - if (!f->pkg->is_single_file && docs != nullptr && docs->list.count > 0) { - for (Token const &tok : docs->list) { - GB_ASSERT(tok.kind == Token_Comment); - String str = tok.string; - if (string_starts_with(str, str_lit("//"))) { - String lc = string_trim_whitespace(substring(str, 2, str.len)); - if (lc.len > 0 && lc[0] == '+') { - if (string_starts_with(lc, str_lit("+build-project-name"))) { - if (!parse_build_project_directory_tag(tok, lc)) { - return false; - } - } else if (string_starts_with(lc, str_lit("+build"))) { - if (!parse_build_tag(tok, lc)) { + if (!f->pkg->is_single_file) { + if (docs != nullptr && docs->list.count > 0) { + for (Token const &tok : docs->list) { + GB_ASSERT(tok.kind == Token_Comment); + String str = tok.string; + if (string_starts_with(str, str_lit("//"))) { + String lc = string_trim_whitespace(substring(str, 2, str.len)); + if (string_starts_with(lc, str_lit("+"))) { + syntax_warning(tok, "//+ is deprecated: Use #+ instead"); + String lt = substring(lc, 1, lc.len); + if (process_file_tag(lt, tok, f) == false) { return false; } - } else if (string_starts_with(lc, str_lit("+vet"))) { - f->vet_flags = parse_vet_tag(tok, lc); - f->vet_flags_set = true; - } else if (string_starts_with(lc, str_lit("+ignore"))) { - return false; - } else if (string_starts_with(lc, str_lit("+private"))) { - f->flags |= AstFile_IsPrivatePkg; - String command = string_trim_starts_with(lc, str_lit("+private ")); - command = string_trim_whitespace(command); - if (lc == "+private") { - f->flags |= AstFile_IsPrivatePkg; - } else if (command == "package") { - f->flags |= AstFile_IsPrivatePkg; - } else if (command == "file") { - f->flags |= AstFile_IsPrivateFile; - } - } else if (lc == "+lazy") { - if (build_context.ignore_lazy) { - // Ignore - } else if (f->pkg->kind == Package_Init && build_context.command_kind == Command_doc) { - // Ignore - } else { - f->flags |= AstFile_IsLazy; - } - } else if (lc == "+no-instrumentation") { - f->flags |= AstFile_NoInstrumentation; - } else { - warning(tok, "Ignoring unknown tag '%.*s'", LIT(lc)); } } } } + + for (Token const &tok : tags) { + GB_ASSERT(tok.kind == Token_FileTag); + String str = tok.string; + + if (string_starts_with(str, str_lit("#+"))) { + String lt = string_trim_whitespace(substring(str, 2, str.len)); + if (process_file_tag(lt, tok, f) == false) { + return false; + } + } + } } + array_free(&tags); + Ast *pd = ast_package_decl(f, f->package_token, package_name, docs, f->line_comment); expect_semicolon(f); f->pkg_decl = pd; diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 4425bee29..e9bad390e 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -2,6 +2,7 @@ TOKEN_KIND(Token_Invalid, "Invalid"), \ TOKEN_KIND(Token_EOF, "EOF"), \ TOKEN_KIND(Token_Comment, "Comment"), \ + TOKEN_KIND(Token_FileTag, "FileTag"), \ \ TOKEN_KIND(Token__LiteralBegin, ""), \ TOKEN_KIND(Token_Ident, "identifier"), \ @@ -939,6 +940,20 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { if (t->curr_rune == '!') { token->kind = Token_Comment; tokenizer_skip_line(t); + } else if (t->curr_rune == '+') { + token->kind = Token_FileTag; + + // Skip the line or until it ends or until we hit was is probably a comment. + // The parsing of tags happens in `parse_file`. + while (t->curr_rune != GB_RUNE_EOF) { + if (t->curr_rune == '\n') { + break; + } + if (t->curr_rune == '/') { + break; + } + advance_to_next_rune(t); + } } break; case '/': -- cgit v1.2.3 From f9de8fdaba12746b9c458e916ba6bb9d7c5b7aa7 Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Mon, 9 Sep 2024 21:51:29 +0200 Subject: Documentation typo fix in tokenizer. --- src/tokenizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index e9bad390e..53f6135d0 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -943,7 +943,7 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { } else if (t->curr_rune == '+') { token->kind = Token_FileTag; - // Skip the line or until it ends or until we hit was is probably a comment. + // Skip until end of line or until we hit what is probably a comment. // The parsing of tags happens in `parse_file`. while (t->curr_rune != GB_RUNE_EOF) { if (t->curr_rune == '\n') { -- cgit v1.2.3