From 2c5c8192f8fcb40fdef183c25a8d799f23f23439 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 28 Jul 2019 22:58:56 +0100 Subject: Fix parsing for procedure literals expression statements; improve assert performance; other minor fixes --- src/parser.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index a377b773c..d1fd94c1d 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -3759,6 +3759,7 @@ Ast *parse_stmt(AstFile *f) { switch (token.kind) { // Operands case Token_context: // Also allows for `context =` + case Token_proc: case Token_inline: case Token_no_inline: case Token_Ident: -- cgit v1.2.3 From 9c632128245e9a51bb63273d0d4531ba28773629 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 9 Aug 2019 21:59:58 +0100 Subject: Struct field tags --- core/odin/ast/ast.odin | 20 ++++++++++++-------- core/odin/parser/parser.odin | 24 +++++++++++++++++------- core/runtime/core.odin | 5 +++-- examples/demo/demo.odin | 25 +++++++++++++++++++++++++ src/check_type.cpp | 6 ++++-- src/ir.cpp | 37 +++++++++++++++++++++++++++++++++---- src/parser.cpp | 37 +++++++++++++++++++++++++++++-------- src/parser.hpp | 4 +++- src/types.cpp | 1 + 9 files changed, 127 insertions(+), 32 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/ast/ast.odin b/core/odin/ast/ast.odin index c0d2948ee..3655a601d 100644 --- a/core/odin/ast/ast.odin +++ b/core/odin/ast/ast.odin @@ -435,7 +435,9 @@ Field_Flag :: enum { C_Vararg, Auto_Cast, In, + Results, + Tags, Default_Parameters, Typeid_Token, } @@ -443,18 +445,19 @@ Field_Flag :: enum { Field_Flags :: distinct bit_set[Field_Flag]; Field_Flags_Struct :: Field_Flags{ - Field_Flag.Using, + .Using, + .Tags, }; Field_Flags_Record_Poly_Params :: Field_Flags{ - Field_Flag.Typeid_Token, + .Typeid_Token, }; Field_Flags_Signature :: Field_Flags{ - Field_Flag.Ellipsis, - Field_Flag.Using, - Field_Flag.No_Alias, - Field_Flag.C_Vararg, - Field_Flag.Auto_Cast, - Field_Flag.Default_Parameters, + .Ellipsis, + .Using, + .No_Alias, + .C_Vararg, + .Auto_Cast, + .Default_Parameters, }; Field_Flags_Signature_Params :: Field_Flags_Signature | {Field_Flag.Typeid_Token}; @@ -483,6 +486,7 @@ Field :: struct { names: []^Expr, // Could be polymorphic type: ^Expr, default_value: ^Expr, + tag: token.Token, flags: Field_Flags, comment: ^Comment_Group, } diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index 0c011e07e..2d8e42720 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1388,19 +1388,18 @@ check_field_flag_prefixes :: proc(p: ^Parser, name_count: int, allowed_flags, se for flag in ast.Field_Flag { if flag notin allowed_flags && flag in flags { - using ast.Field_Flag; #complete switch flag { - case Using: + case .Using: error(p, p.curr_tok.pos, "'using' is not allowed within this field list"); - case No_Alias: + case .No_Alias: error(p, p.curr_tok.pos, "'#no_alias' is not allowed within this field list"); - case C_Vararg: + case .C_Vararg: error(p, p.curr_tok.pos, "'#c_vararg' is not allowed within this field list"); - case Auto_Cast: + case .Auto_Cast: error(p, p.curr_tok.pos, "'auto_cast' is not allowed within this field list"); - case In: + case .In: error(p, p.curr_tok.pos, "'in' is not allowed within this field list"); - case Ellipsis, Results, Default_Parameters, Typeid_Token: + case .Tags, .Ellipsis, .Results, .Default_Parameters, .Typeid_Token: panic("Impossible prefixes"); } flags &~= {flag}; @@ -1540,6 +1539,7 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel type: ^ast.Expr; default_value: ^ast.Expr; + tag: token.Token; expect_token_after(p, token.Colon, "field list"); if p.curr_tok.kind != token.Eq { @@ -1579,9 +1579,19 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel error(p, p.curr_tok.pos, "extra parameter after ellipsis without a default value"); } + if type != nil && default_value == nil { + if p.curr_tok.kind == token.String { + tag = expect_token(p, token.String); + if .Tags notin allowed_flags { + error(p, tag.pos, "Field tags are only allowed within structures"); + } + } + } + ok := expect_field_separator(p, type); field := new_ast_field(names, type, default_value); + field.tag = tag; field.docs = docs; field.flags = flags; field.comment = p.line_comment; diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 02578da3b..28767cc2d 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -79,8 +79,9 @@ Type_Info_Tuple :: struct { // Only really used for procedures Type_Info_Struct :: struct { types: []^Type_Info, names: []string, - offsets: []uintptr, // offsets may not be used in tuples - usings: []bool, // usings may not be used in tuples + offsets: []uintptr, + usings: []bool, + tags: []string, is_packed: bool, is_raw_union: bool, custom_align: bool, diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index 206fa4a2d..21f889746 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -3,6 +3,7 @@ package main import "core:fmt" import "core:mem" import "core:os" +import "core:runtime" when os.OS == "windows" { import "core:thread" @@ -945,6 +946,29 @@ deferred_procedure_associations :: proc() { } } +struct_field_tags :: proc() { + fmt.println("# struct_field_tags"); + + Foo :: struct { + x: int `tag1`, + y: string `tag2`, + z: bool, // no tag + } + + f: Foo; + ti := runtime.type_info_base(type_info_of(Foo)); + s := ti.variant.(runtime.Type_Info_Struct); + fmt.println("Foo :: struct {"); + for _, i in s.names { + if tag := s.tags[i]; tag != "" { + fmt.printf("\t%s: %T `%s`,\n", s.names[i], s.types[i], tag); + } else { + fmt.printf("\t%s: %T,\n", s.names[i], s.types[i]); + } + } + fmt.println("}"); +} + main :: proc() { when true { general_stuff(); @@ -963,5 +987,6 @@ main :: proc() { bit_set_type(); diverging_procedures(); deferred_procedure_associations(); + struct_field_tags(); } } diff --git a/src/check_type.cpp b/src/check_type.cpp index 36d8cf0f8..a9c1d3fe3 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -110,9 +110,10 @@ bool does_field_type_allow_using(Type *t) { return false; } -void check_struct_fields(CheckerContext *ctx, Ast *node, Array *fields, Array const ¶ms, +void check_struct_fields(CheckerContext *ctx, Ast *node, Array *fields, Array *tags, Array const ¶ms, isize init_field_capacity, Type *struct_type, String context) { *fields = array_make(heap_allocator(), 0, init_field_capacity); + *tags = array_make(heap_allocator(), 0, init_field_capacity); GB_ASSERT(node->kind == Ast_StructType); GB_ASSERT(struct_type->kind == Type_Struct); @@ -171,6 +172,7 @@ void check_struct_fields(CheckerContext *ctx, Ast *node, Array *fields Entity *field = alloc_entity_field(ctx->scope, name_token, type, is_using, field_src_index); add_entity(ctx->checker, ctx->scope, name, field); array_add(fields, field); + array_add(tags, p->tag.string); field_src_index += 1; } @@ -504,7 +506,7 @@ void check_struct_type(CheckerContext *ctx, Type *struct_type, Ast *node, Array< if (!is_polymorphic) { - check_struct_fields(ctx, node, &struct_type->Struct.fields, st->fields, min_field_count, struct_type, context); + check_struct_fields(ctx, node, &struct_type->Struct.fields, &struct_type->Struct.tags, st->fields, min_field_count, struct_type, context); } if (st->align != nullptr) { diff --git a/src/ir.cpp b/src/ir.cpp index ee67c0ea8..2499b848f 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -174,6 +174,7 @@ gbAllocator ir_allocator(void) { #define IR_TYPE_INFO_NAMES_NAME "__$type_info_names_data" #define IR_TYPE_INFO_OFFSETS_NAME "__$type_info_offsets_data" #define IR_TYPE_INFO_USINGS_NAME "__$type_info_usings_data" +#define IR_TYPE_INFO_TAGS_NAME "__$type_info_tags_data" #define IR_INSTR_KINDS \ @@ -5271,12 +5272,14 @@ gb_global irValue *ir_global_type_info_member_types = nullptr; gb_global irValue *ir_global_type_info_member_names = nullptr; gb_global irValue *ir_global_type_info_member_offsets = nullptr; gb_global irValue *ir_global_type_info_member_usings = nullptr; +gb_global irValue *ir_global_type_info_member_tags = nullptr; gb_global i32 ir_global_type_info_data_index = 0; gb_global i32 ir_global_type_info_member_types_index = 0; gb_global i32 ir_global_type_info_member_names_index = 0; gb_global i32 ir_global_type_info_member_offsets_index = 0; gb_global i32 ir_global_type_info_member_usings_index = 0; +gb_global i32 ir_global_type_info_member_tags_index = 0; isize ir_type_info_count(CheckerInfo *info) { return info->minimum_dependency_type_info_set.entries.count+1; @@ -9587,6 +9590,16 @@ void ir_init_module(irModule *m, Checker *c) { map_set(&m->members, hash_string(name), g); ir_global_type_info_member_usings = g; } + + { + String name = str_lit(IR_TYPE_INFO_TAGS_NAME); + Entity *e = alloc_entity_variable(nullptr, make_token_ident(name), + alloc_type_array(t_string, count), false); + irValue *g = ir_value_global(e, nullptr); + ir_module_add_value(m, e, g); + map_set(&m->members, hash_string(name), g); + ir_global_type_info_member_tags = g; + } } } } @@ -9720,6 +9733,11 @@ irValue *ir_type_info_member_usings_offset(irProcedure *proc, isize count) { ir_global_type_info_member_usings_index += cast(i32)count; return offset; } +irValue *ir_type_info_member_tags_offset(irProcedure *proc, isize count) { + irValue *offset = ir_emit_array_epi(proc, ir_global_type_info_member_tags, ir_global_type_info_member_tags_index); + ir_global_type_info_member_tags_index += cast(i32)count; + return offset; +} @@ -10065,9 +10083,9 @@ void ir_setup_type_info_data(irProcedure *proc) { // NOTE(bill): Setup type_info irValue *is_packed = ir_const_bool(t->Struct.is_packed); irValue *is_raw_union = ir_const_bool(t->Struct.is_raw_union); irValue *is_custom_align = ir_const_bool(t->Struct.custom_align != 0); - ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 4), is_packed); - ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 5), is_raw_union); - ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 6), is_custom_align); + ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 5), is_packed); + ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 6), is_raw_union); + ir_emit_store(proc, ir_emit_struct_ep(proc, tag, 7), is_custom_align); } isize count = t->Struct.fields.count; @@ -10076,6 +10094,7 @@ void ir_setup_type_info_data(irProcedure *proc) { // NOTE(bill): Setup type_info irValue *memory_names = ir_type_info_member_names_offset (proc, count); irValue *memory_offsets = ir_type_info_member_offsets_offset(proc, count); irValue *memory_usings = ir_type_info_member_usings_offset (proc, count); + irValue *memory_tags = ir_type_info_member_tags_offset (proc, count); type_set_offsets(t); // NOTE(bill): Just incase the offsets have not been set yet for (isize source_index = 0; source_index < count; source_index++) { @@ -10091,7 +10110,7 @@ void ir_setup_type_info_data(irProcedure *proc) { // NOTE(bill): Setup type_info irValue *index = ir_const_int(source_index); irValue *type_info = ir_emit_ptr_offset(proc, memory_types, index); irValue *offset = ir_emit_ptr_offset(proc, memory_offsets, index); - irValue *is_using = ir_emit_ptr_offset(proc, memory_usings, index); + irValue *is_using = ir_emit_ptr_offset(proc, memory_usings, index); ir_emit_store(proc, type_info, ir_type_info(proc, f->type)); if (f->token.string.len > 0) { @@ -10100,6 +10119,15 @@ void ir_setup_type_info_data(irProcedure *proc) { // NOTE(bill): Setup type_info } ir_emit_store(proc, offset, ir_const_uintptr(foffset)); ir_emit_store(proc, is_using, ir_const_bool((f->flags&EntityFlag_Using) != 0)); + + if (t->Struct.tags.count > 0) { + String tag_string = t->Struct.tags[source_index]; + if (tag_string.len > 0) { + irValue *tag_ptr = ir_emit_ptr_offset(proc, memory_tags, index); + ir_emit_store(proc, tag_ptr, ir_const_string(tag_string)); + } + } + } irValue *cv = ir_const_int(count); @@ -10107,6 +10135,7 @@ void ir_setup_type_info_data(irProcedure *proc) { // NOTE(bill): Setup type_info ir_fill_slice(proc, ir_emit_struct_ep(proc, tag, 1), memory_names, cv); ir_fill_slice(proc, ir_emit_struct_ep(proc, tag, 2), memory_offsets, cv); ir_fill_slice(proc, ir_emit_struct_ep(proc, tag, 3), memory_usings, cv); + ir_fill_slice(proc, ir_emit_struct_ep(proc, tag, 4), memory_tags, cv); } break; } diff --git a/src/parser.cpp b/src/parser.cpp index d1fd94c1d..9aebc78d6 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -805,13 +805,14 @@ Ast *ast_bad_decl(AstFile *f, Token begin, Token end) { return result; } -Ast *ast_field(AstFile *f, Array names, Ast *type, Ast *default_value, u32 flags, - CommentGroup *docs, CommentGroup *comment) { +Ast *ast_field(AstFile *f, Array names, Ast *type, Ast *default_value, u32 flags, Token tag, + CommentGroup *docs, CommentGroup *comment) { Ast *result = alloc_ast_node(f, Ast_Field); result->Field.names = names; result->Field.type = type; result->Field.default_value = default_value; result->Field.flags = flags; + result->Field.tag = tag; result->Field.docs = docs; result->Field.comment = comment; return result; @@ -2740,7 +2741,8 @@ Ast *parse_results(AstFile *f, bool *diverging) { Array empty_names = {}; auto list = array_make(heap_allocator(), 0, 1); Ast *type = parse_type(f); - array_add(&list, ast_field(f, empty_names, type, nullptr, 0, nullptr, nullptr)); + Token tag = {}; + array_add(&list, ast_field(f, empty_names, type, nullptr, 0, tag, nullptr, nullptr)); return ast_field_list(f, begin_token, list); } @@ -3095,6 +3097,7 @@ Ast *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKi Ast *type = nullptr; Ast *default_value = nullptr; + Token tag = {}; expect_token_after(f, Token_Colon, "field list"); if (f->curr_token.kind != Token_Eq) { @@ -3132,16 +3135,25 @@ Ast *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKi syntax_error(f->curr_token, "Extra parameter after ellipsis without a default value"); } + if (type != nullptr && default_value == nullptr) { + if (f->curr_token.kind == Token_String) { + tag = expect_token(f, Token_String); + if ((allowed_flags & FieldFlag_Tags) == 0) { + syntax_error(tag, "Field tags are only allowed within structures"); + } + } + } + parse_expect_field_separator(f, type); - Ast *param = ast_field(f, names, type, default_value, set_flags, docs, f->line_comment); + Ast *param = ast_field(f, names, type, default_value, set_flags, tag, docs, f->line_comment); array_add(¶ms, param); while (f->curr_token.kind != follow && f->curr_token.kind != Token_EOF) { CommentGroup *docs = f->lead_comment; - u32 set_flags = parse_field_prefixes(f); + Token tag = {}; Array names = parse_ident_list(f, allow_poly_names); if (names.count == 0) { syntax_error(f->curr_token, "Empty field declaration"); @@ -3184,9 +3196,18 @@ Ast *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKi syntax_error(f->curr_token, "Extra parameter after ellipsis without a default value"); } + if (type != nullptr && default_value == nullptr) { + if (f->curr_token.kind == Token_String) { + tag = expect_token(f, Token_String); + if ((allowed_flags & FieldFlag_Tags) == 0) { + syntax_error(tag, "Field tags are only allowed within structures"); + } + } + } + bool ok = parse_expect_field_separator(f, param); - Ast *param = ast_field(f, names, type, default_value, set_flags, docs, f->line_comment); + Ast *param = ast_field(f, names, type, default_value, set_flags, tag, docs, f->line_comment); array_add(¶ms, param); if (!ok) { @@ -3210,8 +3231,8 @@ Ast *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKi token.pos = ast_token(type).pos; names[0] = ast_ident(f, token); u32 flags = check_field_prefixes(f, list.count, allowed_flags, list[i].flags); - - Ast *param = ast_field(f, names, list[i].node, nullptr, flags, docs, f->line_comment); + Token tag = {}; + Ast *param = ast_field(f, names, list[i].node, nullptr, flags, tag, docs, f->line_comment); array_add(¶ms, param); } diff --git a/src/parser.hpp b/src/parser.hpp index 9f4f68a7b..3489f1a9b 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -186,11 +186,12 @@ enum FieldFlag { FieldFlag_c_vararg = 1<<3, FieldFlag_auto_cast = 1<<4, + FieldFlag_Tags = 1<<10, FieldFlag_Results = 1<<16, FieldFlag_Signature = FieldFlag_ellipsis|FieldFlag_using|FieldFlag_no_alias|FieldFlag_c_vararg|FieldFlag_auto_cast, - FieldFlag_Struct = FieldFlag_using, + FieldFlag_Struct = FieldFlag_using|FieldFlag_Tags, }; enum StmtAllowFlag { @@ -412,6 +413,7 @@ AST_KIND(_DeclEnd, "", bool) \ Array names; \ Ast * type; \ Ast * default_value; \ + Token tag; \ u32 flags; \ CommentGroup * docs; \ CommentGroup * comment; \ diff --git a/src/types.cpp b/src/types.cpp index e5d2509e0..1197ba974 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -107,6 +107,7 @@ struct BasicType { struct TypeStruct { Array fields; + Array tags; Ast *node; Scope * scope; -- cgit v1.2.3 From 01c10aa9447d135bdda5dc25583a5e1f94cbda6d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 26 Aug 2019 13:54:35 +0100 Subject: `inline for` loops (only for 'in' based for loops) --- examples/demo/demo.odin | 39 +++++++ src/check_stmt.cpp | 272 ++++++++++++++++++++++++++++++++++++++++++++---- src/checker.hpp | 3 + src/exact_value.cpp | 4 + src/ir.cpp | 144 +++++++++++++++++++++++++ src/parser.cpp | 83 ++++++++++++++- src/parser.hpp | 9 ++ 7 files changed, 530 insertions(+), 24 deletions(-) (limited to 'src/parser.cpp') diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index a96056b61..cef814a56 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -1018,6 +1018,44 @@ quaternions :: proc() { fmt.println(c); fmt.println(q); } + { // Memory layout of Quaternions + q := 1 + 2i + 3j + 4k; + a := transmute([4]f64)q; + fmt.println("Quaternion memory layout: xyzw/(ijkr)"); + fmt.println(q); // 1.000+2.000i+3.000j+4.000k + fmt.println(a); // [2.000, 3.000, 4.000, 1.000] + } +} + +inline_for_statement :: proc() { + fmt.println("\n#inline for statements"); + + fmt.println("Ranges"); + inline for x, i in 1..<4 { + fmt.println(x, i); + } + + fmt.println("Strings"); + inline for r, i in "Hello, 世界" { + fmt.println(r, i); + } + + fmt.println("Arrays"); + inline for elem, idx in ([4]int{1, 4, 9, 16}) { + fmt.println(elem, idx); + } + + + Foo_Enum :: enum { + A = 1, + B, + C = 6, + D, + } + fmt.println("Enum types"); + inline for elem, idx in Foo_Enum { + fmt.println(elem, idx); + } } main :: proc() { @@ -1040,5 +1078,6 @@ main :: proc() { deferred_procedure_associations(); reflection(); quaternions(); + inline_for_statement(); } } diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index 2787aa436..88661558d 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -132,6 +132,10 @@ bool check_is_terminating(Ast *node) { } case_end; + case_ast_node(rs, InlineRangeStmt, node); + return false; + case_end; + case_ast_node(rs, RangeStmt, node); return false; case_end; @@ -587,6 +591,236 @@ void add_constant_switch_case(CheckerContext *ctx, Map *seen, Oper multi_map_insert(seen, key, tap); } +void check_inline_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) { + ast_node(irs, InlineRangeStmt, node); + check_open_scope(ctx, node); + + Type *val0 = nullptr; + Type *val1 = nullptr; + Entity *entities[2] = {}; + isize entity_count = 0; + + Ast *expr = unparen_expr(irs->expr); + + ExactValue inline_for_depth = exact_value_i64(0); + + if (is_ast_range(expr)) { + ast_node(ie, BinaryExpr, expr); + Operand x = {Addressing_Invalid}; + Operand y = {Addressing_Invalid}; + + check_expr(ctx, &x, ie->left); + if (x.mode == Addressing_Invalid) { + goto skip_expr; + } + check_expr(ctx, &y, ie->right); + if (y.mode == Addressing_Invalid) { + goto skip_expr; + } + + convert_to_typed(ctx, &x, y.type); + if (x.mode == Addressing_Invalid) { + goto skip_expr; + } + convert_to_typed(ctx, &y, x.type); + if (y.mode == Addressing_Invalid) { + goto skip_expr; + } + + convert_to_typed(ctx, &x, default_type(y.type)); + if (x.mode == Addressing_Invalid) { + goto skip_expr; + } + convert_to_typed(ctx, &y, default_type(x.type)); + if (y.mode == Addressing_Invalid) { + goto skip_expr; + } + + if (!are_types_identical(x.type, y.type)) { + if (x.type != t_invalid && + y.type != t_invalid) { + gbString xt = type_to_string(x.type); + gbString yt = type_to_string(y.type); + gbString expr_str = expr_to_string(x.expr); + error(ie->op, "Mismatched types in interval expression '%s' : '%s' vs '%s'", expr_str, xt, yt); + gb_string_free(expr_str); + gb_string_free(yt); + gb_string_free(xt); + } + goto skip_expr; + } + + Type *type = x.type; + if (!is_type_integer(type) && !is_type_float(type) && !is_type_pointer(type) && !is_type_enum(type)) { + error(ie->op, "Only numerical and pointer types are allowed within interval expressions"); + goto skip_expr; + } + + if (x.mode == Addressing_Constant && + y.mode == Addressing_Constant) { + ExactValue a = x.value; + ExactValue b = y.value; + + GB_ASSERT(are_types_identical(x.type, y.type)); + + TokenKind op = Token_Lt; + switch (ie->op.kind) { + case Token_Ellipsis: op = Token_LtEq; break; + case Token_RangeHalf: op = Token_Lt; break; + default: error(ie->op, "Invalid range operator"); break; + } + bool ok = compare_exact_values(op, a, b); + if (!ok) { + // TODO(bill): Better error message + error(ie->op, "Invalid interval range"); + goto skip_expr; + } + + inline_for_depth = exact_value_sub(b, a); + if (ie->op.kind == Token_Ellipsis) { + inline_for_depth = exact_value_increment_one(inline_for_depth); + } + + } else { + error(ie->op, "Interval expressions must be constant"); + goto skip_expr; + } + + add_type_and_value(&ctx->checker->info, ie->left, x.mode, x.type, x.value); + add_type_and_value(&ctx->checker->info, ie->right, y.mode, y.type, y.value); + val0 = type; + val1 = t_int; + } else { + Operand operand = {Addressing_Invalid}; + check_expr_or_type(ctx, &operand, irs->expr); + + if (operand.mode == Addressing_Type) { + if (!is_type_enum(operand.type)) { + gbString t = type_to_string(operand.type); + error(operand.expr, "Cannot iterate over the type '%s'", t); + gb_string_free(t); + goto skip_expr; + } else { + val0 = operand.type; + val1 = t_int; + add_type_info_type(ctx, operand.type); + + Type *bt = base_type(operand.type); + inline_for_depth = exact_value_i64(bt->Enum.fields.count); + goto skip_expr; + } + } else if (operand.mode != Addressing_Invalid) { + Type *t = base_type(operand.type); + switch (t->kind) { + case Type_Basic: + if (is_type_string(t) && t->Basic.kind != Basic_cstring) { + val0 = t_rune; + val1 = t_int; + inline_for_depth = exact_value_i64(operand.value.value_string.len); + } + break; + case Type_Array: + val0 = t->Array.elem; + val1 = t_int; + inline_for_depth = exact_value_i64(t->Array.count); + break; + } + } + + if (val0 == nullptr) { + gbString s = expr_to_string(operand.expr); + gbString t = type_to_string(operand.type); + error(operand.expr, "Cannot iterate over '%s' of type '%s' in an 'inline for' statement", s, t); + gb_string_free(t); + gb_string_free(s); + } else if (operand.mode != Addressing_Constant) { + error(operand.expr, "An 'inline for' expression must be known at compile time"); + } + } + + skip_expr:; // NOTE(zhiayang): again, declaring a variable immediately after a label... weird. + + Ast * lhs[2] = {irs->val0, irs->val1}; + Type *rhs[2] = {val0, val1}; + + for (isize i = 0; i < 2; i++) { + if (lhs[i] == nullptr) { + continue; + } + Ast * name = lhs[i]; + Type *type = rhs[i]; + + Entity *entity = nullptr; + if (name->kind == Ast_Ident) { + Token token = name->Ident.token; + String str = token.string; + Entity *found = nullptr; + + if (!is_blank_ident(str)) { + found = scope_lookup_current(ctx->scope, str); + } + if (found == nullptr) { + bool is_immutable = true; + entity = alloc_entity_variable(ctx->scope, token, type, is_immutable, EntityState_Resolved); + entity->flags |= EntityFlag_Value; + add_entity_definition(&ctx->checker->info, name, entity); + } else { + TokenPos pos = found->token.pos; + error(token, + "Redeclaration of '%.*s' in this scope\n" + "\tat %.*s(%td:%td)", + LIT(str), LIT(pos.file), pos.line, pos.column); + entity = found; + } + } else { + error(name, "A variable declaration must be an identifier"); + } + + if (entity == nullptr) { + entity = alloc_entity_dummy_variable(builtin_pkg->scope, ast_token(name)); + } + + entities[entity_count++] = entity; + + if (type == nullptr) { + entity->type = t_invalid; + entity->flags |= EntityFlag_Used; + } + } + + for (isize i = 0; i < entity_count; i++) { + add_entity(ctx->checker, ctx->scope, entities[i]->identifier, entities[i]); + } + + + // NOTE(bill): Minimize the amount of nesting of an 'inline for' + i64 prev_inline_for_depth = ctx->inline_for_depth; + defer (ctx->inline_for_depth = prev_inline_for_depth); + { + i64 v = exact_value_to_i64(inline_for_depth); + if (v <= 0) { + // Do nothing + } else { + ctx->inline_for_depth = gb_max(ctx->inline_for_depth, 1) * v; + } + + if (ctx->inline_for_depth >= MAX_INLINE_FOR_DEPTH && prev_inline_for_depth < MAX_INLINE_FOR_DEPTH) { + if (prev_inline_for_depth > 0) { + error(node, "Nested 'inline for' loop cannot be inlined as it exceeds the maximum inline for depth (%lld levels >= %lld maximum levels)", v, MAX_INLINE_FOR_DEPTH); + } else { + error(node, "'inline for' loop cannot be inlined as it exceeds the maximum inline for depth (%lld levels >= %lld maximum levels)", v, MAX_INLINE_FOR_DEPTH); + } + error_line("\tUse a normal 'for' loop instead by removing the 'inline' prefix\n"); + ctx->inline_for_depth = MAX_INLINE_FOR_DEPTH; + } + } + + check_stmt(ctx, irs->body, mod_flags); + + + check_close_scope(ctx); +} + void check_switch_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) { ast_node(ss, SwitchStmt, node); @@ -1298,6 +1532,7 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { check_close_scope(ctx); case_end; + case_ast_node(rs, RangeStmt, node); u32 new_flags = mod_flags | Stmt_BreakAllowed | Stmt_ContinueAllowed; @@ -1320,29 +1555,29 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { check_expr(ctx, &x, ie->left); if (x.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } check_expr(ctx, &y, ie->right); if (y.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } convert_to_typed(ctx, &x, y.type); if (x.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } convert_to_typed(ctx, &y, x.type); if (y.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } convert_to_typed(ctx, &x, default_type(y.type)); if (x.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } convert_to_typed(ctx, &y, default_type(x.type)); if (y.mode == Addressing_Invalid) { - goto skip_expr; + goto skip_expr_range_stmt; } if (!are_types_identical(x.type, y.type)) { @@ -1356,13 +1591,13 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { gb_string_free(yt); gb_string_free(xt); } - goto skip_expr; + goto skip_expr_range_stmt; } Type *type = x.type; if (!is_type_integer(type) && !is_type_float(type) && !is_type_pointer(type) && !is_type_enum(type)) { error(ie->op, "Only numerical and pointer types are allowed within interval expressions"); - goto skip_expr; + goto skip_expr_range_stmt; } if (x.mode == Addressing_Constant && @@ -1382,18 +1617,10 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { if (!ok) { // TODO(bill): Better error message error(ie->op, "Invalid interval range"); - goto skip_expr; + goto skip_expr_range_stmt; } } - if (x.mode != Addressing_Constant) { - x.value = empty_exact_value; - } - if (y.mode != Addressing_Constant) { - y.value = empty_exact_value; - } - - add_type_and_value(&ctx->checker->info, ie->left, x.mode, x.type, x.value); add_type_and_value(&ctx->checker->info, ie->right, y.mode, y.type, y.value); val0 = type; @@ -1407,12 +1634,12 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { gbString t = type_to_string(operand.type); error(operand.expr, "Cannot iterate over the type '%s'", t); gb_string_free(t); - goto skip_expr; + goto skip_expr_range_stmt; } else { val0 = operand.type; val1 = t_int; add_type_info_type(ctx, operand.type); - goto skip_expr; + goto skip_expr_range_stmt; } } else if (operand.mode != Addressing_Invalid) { bool is_ptr = is_type_pointer(operand.type); @@ -1457,7 +1684,8 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { } } - skip_expr:; // NOTE(zhiayang): again, declaring a variable immediately after a label... weird. + skip_expr_range_stmt:; // NOTE(zhiayang): again, declaring a variable immediately after a label... weird. + Ast * lhs[2] = {rs->val0, rs->val1}; Type *rhs[2] = {val0, val1}; @@ -1515,6 +1743,10 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { check_close_scope(ctx); case_end; + case_ast_node(irs, InlineRangeStmt, node); + check_inline_range_stmt(ctx, node, mod_flags); + case_end; + case_ast_node(ss, SwitchStmt, node); check_switch_stmt(ctx, node, mod_flags); case_end; diff --git a/src/checker.hpp b/src/checker.hpp index 4e4a8cfe1..a37a87b87 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -276,6 +276,9 @@ struct CheckerContext { CheckerPolyPath *poly_path; isize poly_level; // TODO(bill): Actually handle correctly +#define MAX_INLINE_FOR_DEPTH 1024ll + i64 inline_for_depth; + bool in_enum_type; bool collect_delayed_decls; bool allow_polymorphic_types; diff --git a/src/exact_value.cpp b/src/exact_value.cpp index 54d4ef6f2..fcc6f1973 100644 --- a/src/exact_value.cpp +++ b/src/exact_value.cpp @@ -813,6 +813,10 @@ gb_inline ExactValue exact_value_shift(TokenKind op, ExactValue const &x, ExactV return exact_binary_operator_value(op, x, y); } +gb_inline ExactValue exact_value_increment_one(ExactValue const &x) { + return exact_binary_operator_value(Token_Add, x, exact_value_i64(1)); +} + i32 cmp_f64(f64 a, f64 b) { return (a > b) - (a < b); diff --git a/src/ir.cpp b/src/ir.cpp index 2a4bb028b..7dc5988e3 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -9061,6 +9061,150 @@ void ir_build_stmt_internal(irProcedure *proc, Ast *node) { ir_start_block(proc, done); case_end; + case_ast_node(rs, InlineRangeStmt, node); + ir_emit_comment(proc, str_lit("InlineRangeStmt")); + ir_open_scope(proc); // Open scope here + + Type *val0_type = nullptr; + Type *val1_type = nullptr; + if (rs->val0 != nullptr && !is_blank_ident(rs->val0)) { + val0_type = type_of_expr(rs->val0); + } + if (rs->val1 != nullptr && !is_blank_ident(rs->val1)) { + val1_type = type_of_expr(rs->val1); + } + + if (val0_type != nullptr) { + ir_add_local_for_identifier(proc, rs->val0, true); + } + if (val1_type != nullptr) { + ir_add_local_for_identifier(proc, rs->val1, true); + } + + irValue *val = nullptr; + irValue *key = nullptr; + irBlock *loop = nullptr; + irBlock *done = nullptr; + Ast *expr = unparen_expr(rs->expr); + + TypeAndValue tav = type_and_value_of_expr(expr); + + if (is_ast_range(expr)) { + + irAddr val0_addr = {}; + irAddr val1_addr = {}; + if (val0_type) val0_addr = ir_build_addr(proc, rs->val0); + if (val1_type) val1_addr = ir_build_addr(proc, rs->val1); + + TokenKind op = expr->BinaryExpr.op.kind; + Ast *start_expr = expr->BinaryExpr.left; + Ast *end_expr = expr->BinaryExpr.right; + GB_ASSERT(start_expr->tav.mode == Addressing_Constant); + GB_ASSERT(end_expr->tav.mode == Addressing_Constant); + + ExactValue start = start_expr->tav.value; + ExactValue end = end_expr->tav.value; + if (op == Token_Ellipsis) { // .. [start, end] + ExactValue index = exact_value_i64(0); + for (ExactValue val = start; + compare_exact_values(Token_LtEq, val, end); + val = exact_value_increment_one(val), index = exact_value_increment_one(index)) { + + if (val0_type) ir_addr_store(proc, val0_addr, ir_value_constant(val0_type, val)); + if (val1_type) ir_addr_store(proc, val1_addr, ir_value_constant(val1_type, index)); + + ir_build_stmt(proc, rs->body); + } + } else if (op == Token_RangeHalf) { // ..< [start, end) + ExactValue index = exact_value_i64(0); + for (ExactValue val = start; + compare_exact_values(Token_Lt, val, end); + val = exact_value_increment_one(val), index = exact_value_increment_one(index)) { + + if (val0_type) ir_addr_store(proc, val0_addr, ir_value_constant(val0_type, val)); + if (val1_type) ir_addr_store(proc, val1_addr, ir_value_constant(val1_type, index)); + + ir_build_stmt(proc, rs->body); + } + } + + + } else if (tav.mode == Addressing_Type) { + GB_ASSERT(is_type_enum(type_deref(tav.type))); + Type *et = type_deref(tav.type); + Type *bet = base_type(et); + + irAddr val0_addr = {}; + irAddr val1_addr = {}; + if (val0_type) val0_addr = ir_build_addr(proc, rs->val0); + if (val1_type) val1_addr = ir_build_addr(proc, rs->val1); + + for_array(i, bet->Enum.fields) { + Entity *field = bet->Enum.fields[i]; + GB_ASSERT(field->kind == Entity_Constant); + if (val0_type) ir_addr_store(proc, val0_addr, ir_value_constant(val0_type, field->Constant.value)); + if (val1_type) ir_addr_store(proc, val1_addr, ir_value_constant(val1_type, exact_value_i64(i))); + + ir_build_stmt(proc, rs->body); + } + } else { + irAddr val0_addr = {}; + irAddr val1_addr = {}; + if (val0_type) val0_addr = ir_build_addr(proc, rs->val0); + if (val1_type) val1_addr = ir_build_addr(proc, rs->val1); + + GB_ASSERT(expr->tav.mode == Addressing_Constant); + + Type *t = base_type(expr->tav.type); + + + switch (t->kind) { + case Type_Basic: + GB_ASSERT(is_type_string(t)); + { + ExactValue value = expr->tav.value; + GB_ASSERT(value.kind == ExactValue_String); + String str = value.value_string; + Rune codepoint = 0; + isize offset = 0; + do { + isize width = gb_utf8_decode(str.text+offset, str.len-offset, &codepoint); + if (val0_type) ir_addr_store(proc, val0_addr, ir_value_constant(val0_type, exact_value_i64(codepoint))); + if (val1_type) ir_addr_store(proc, val1_addr, ir_value_constant(val1_type, exact_value_i64(offset))); + ir_build_stmt(proc, rs->body); + + offset += width; + } while (offset < str.len); + } + break; + case Type_Array: + if (t->Array.count > 0) { + irValue *val = ir_build_expr(proc, expr); + irValue *val_addr = ir_address_from_load_or_generate_local(proc, val); + + for (i64 i = 0; i < t->Array.count; i++) { + if (val0_type) { + // NOTE(bill): Due to weird legacy issues in LLVM, this needs to be an i32 + irValue *elem = ir_emit_array_epi(proc, val_addr, cast(i32)i); + ir_addr_store(proc, val0_addr, ir_emit_load(proc, elem)); + } + if (val1_type) ir_addr_store(proc, val1_addr, ir_value_constant(val1_type, exact_value_i64(i))); + + ir_build_stmt(proc, rs->body); + } + + } + break; + default: + GB_PANIC("Invalid inline for type"); + break; + } + } + + + ir_close_scope(proc, irDeferExit_Default, nullptr); + case_end; + case_ast_node(ss, SwitchStmt, node); ir_emit_comment(proc, str_lit("SwitchStmt")); if (ss->init != nullptr) { diff --git a/src/parser.cpp b/src/parser.cpp index 9aebc78d6..8c32f35ff 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -51,6 +51,7 @@ Token ast_token(Ast *node) { case Ast_ReturnStmt: return node->ReturnStmt.token; case Ast_ForStmt: return node->ForStmt.token; case Ast_RangeStmt: return node->RangeStmt.token; + case Ast_InlineRangeStmt: return node->InlineRangeStmt.inline_token; case Ast_CaseClause: return node->CaseClause.token; case Ast_SwitchStmt: return node->SwitchStmt.token; case Ast_TypeSwitchStmt: return node->TypeSwitchStmt.token; @@ -257,6 +258,12 @@ Ast *clone_ast(Ast *node) { n->RangeStmt.expr = clone_ast(n->RangeStmt.expr); n->RangeStmt.body = clone_ast(n->RangeStmt.body); break; + case Ast_InlineRangeStmt: + n->InlineRangeStmt.val0 = clone_ast(n->InlineRangeStmt.val0); + n->InlineRangeStmt.val1 = clone_ast(n->InlineRangeStmt.val1); + n->InlineRangeStmt.expr = clone_ast(n->InlineRangeStmt.expr); + n->InlineRangeStmt.body = clone_ast(n->InlineRangeStmt.body); + break; case Ast_CaseClause: n->CaseClause.list = clone_ast_array(n->CaseClause.list); n->CaseClause.stmts = clone_ast_array(n->CaseClause.stmts); @@ -748,6 +755,18 @@ Ast *ast_range_stmt(AstFile *f, Token token, Ast *val0, Ast *val1, Token in_toke return result; } +Ast *ast_inline_range_stmt(AstFile *f, Token inline_token, Token for_token, Ast *val0, Ast *val1, Token in_token, Ast *expr, Ast *body) { + Ast *result = alloc_ast_node(f, Ast_InlineRangeStmt); + result->InlineRangeStmt.inline_token = inline_token; + result->InlineRangeStmt.for_token = for_token; + result->InlineRangeStmt.val0 = val0; + result->InlineRangeStmt.val1 = val1; + result->InlineRangeStmt.in_token = in_token; + result->InlineRangeStmt.expr = expr; + result->InlineRangeStmt.body = body; + return result; +} + Ast *ast_switch_stmt(AstFile *f, Token token, Ast *init, Ast *tag, Ast *body) { Ast *result = alloc_ast_node(f, Ast_SwitchStmt); result->SwitchStmt.token = token; @@ -1119,6 +1138,17 @@ Token advance_token(AstFile *f) { return prev; } +bool peek_token_kind(AstFile *f, TokenKind kind) { + for (isize i = f->curr_token_index+1; i < f->tokens.count; i++) { + Token tok = f->tokens[i]; + if (kind != Token_Comment && tok.kind == Token_Comment) { + continue; + } + return tok.kind == kind; + } + return false; +} + Token expect_token(AstFile *f, TokenKind kind) { Token prev = f->curr_token; if (prev.kind != kind) { @@ -2092,7 +2122,7 @@ Ast *parse_operand(AstFile *f, bool lhs) { bool prev_allow_range = f->allow_range; f->allow_range = true; - elem = parse_expr(f, false); + elem = parse_expr(f, true); f->allow_range = prev_allow_range; if (allow_token(f, Token_Semicolon)) { underlying = parse_type(f); @@ -2650,7 +2680,7 @@ Ast *parse_simple_stmt(AstFile *f, u32 flags) { allow_token(f, Token_in); bool prev_allow_range = f->allow_range; f->allow_range = true; - Ast *expr = parse_expr(f, false); + Ast *expr = parse_expr(f, true); f->allow_range = prev_allow_range; auto rhs = array_make(heap_allocator(), 0, 1); @@ -3779,10 +3809,55 @@ Ast *parse_stmt(AstFile *f) { Token token = f->curr_token; switch (token.kind) { // Operands - case Token_context: // Also allows for `context =` - case Token_proc: case Token_inline: + if (peek_token_kind(f, Token_for)) { + Token inline_token = expect_token(f, Token_inline); + Token for_token = expect_token(f, Token_for); + Ast *val0 = nullptr; + Ast *val1 = nullptr; + Token in_token = {}; + Ast *expr = nullptr; + Ast *body = nullptr; + + bool bad_stmt = false; + + if (f->curr_token.kind != Token_in) { + Array idents = parse_ident_list(f, false); + switch (idents.count) { + case 1: + val0 = idents[0]; + break; + case 2: + val0 = idents[0]; + val1 = idents[1]; + break; + default: + syntax_error(for_token, "Expected either 1 or 2 identifiers"); + bad_stmt = true; + break; + } + } + in_token = expect_token(f, Token_in); + + bool prev_allow_range = f->allow_range; + f->allow_range = true; + expr = parse_expr(f, true); + f->allow_range = prev_allow_range; + + if (allow_token(f, Token_do)) { + body = convert_stmt_to_body(f, parse_stmt(f)); + } else { + body = parse_block_stmt(f, false); + } + if (bad_stmt) { + return ast_bad_stmt(f, inline_token, f->curr_token); + } + return ast_inline_range_stmt(f, inline_token, for_token, val0, val1, in_token, expr, body); + } + /* fallthrough */ case Token_no_inline: + case Token_context: // Also allows for `context =` + case Token_proc: case Token_Ident: case Token_Integer: case Token_Float: diff --git a/src/parser.hpp b/src/parser.hpp index 3489f1a9b..32398592e 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -326,6 +326,15 @@ AST_KIND(_ComplexStmtBegin, "", bool) \ Ast *expr; \ Ast *body; \ }) \ + AST_KIND(InlineRangeStmt, "inline range statement", struct { \ + Token inline_token; \ + Token for_token; \ + Ast *val0; \ + Ast *val1; \ + Token in_token; \ + Ast *expr; \ + Ast *body; \ + }) \ AST_KIND(CaseClause, "case clause", struct { \ Token token; \ Array list; \ -- cgit v1.2.3 From c44d25d14f2a8f800170daae4a1c9d08858978b6 Mon Sep 17 00:00:00 2001 From: thebirk Date: Mon, 26 Aug 2019 16:47:41 +0200 Subject: Fixed parser creating a new thread for each file. --- src/parser.cpp | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 10 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 9aebc78d6..0a2a3c1fc 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4679,7 +4679,47 @@ skip: return ParseFile_None; } +#if 1 +struct ParserWorkerThreadData { + Parser *parser; + + gbSemaphore resume_work; //NOTE(thebirk): Use to signal that the worker thead has a new work to do + ParseFileError err; + + gbMutex lock; //NOTE(thebirk): All variables below are locked by this mutex + bool error_available; + bool should_exit; +}; + +GB_THREAD_PROC(parse_worker_file_proc) { + GB_ASSERT(thread != nullptr); + + ParserWorkerThreadData* data = cast(ParserWorkerThreadData*) thread->user_data; + + for(;;) { + gb_semaphore_wait(&data->resume_work); + + gb_mutex_lock(&data->lock); + if (data->should_exit) { + gb_mutex_unlock(&data->lock); + return isize(0); + } + + Parser *p = data->parser; + isize index = thread->user_index; + gb_mutex_lock(&p->file_add_mutex); + auto file_to_process = p->files_to_process[index]; + gb_mutex_unlock(&p->file_add_mutex); + data->err = process_imported_file(p, file_to_process); + + data->error_available = true; + gb_mutex_unlock(&data->lock); + } + + //GB_PANIC("A worker thread should not be able to reach the end!!!"); +} +#else GB_THREAD_PROC(parse_worker_file_proc) { if (thread == nullptr) return 0; auto *p = cast(Parser *)thread->user_data; @@ -4690,6 +4730,7 @@ GB_THREAD_PROC(parse_worker_file_proc) { ParseFileError err = process_imported_file(p, file_to_process); return cast(isize)err; } +#endif ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); @@ -4729,14 +4770,36 @@ ParseFileError parse_packages(Parser *p, String init_filename) { curr_import_index++; } + auto worker_threads_data = array_make(heap_allocator(), thread_count); + defer (array_free(&worker_threads_data)); + + for_array(i, worker_threads_data) { + ParserWorkerThreadData *data = &worker_threads_data[i]; + gb_mutex_init(&data->lock); + gb_semaphore_init(&data->resume_work); + data->parser = p; + data->err = ParseFile_None; + data->should_exit = false; + data->error_available = false; + } + defer(for_array(i, worker_threads_data) { + ParserWorkerThreadData *data = &worker_threads_data[i]; + gb_mutex_destroy(&data->lock); + gb_semaphore_destroy(&data->resume_work); + }); + auto worker_threads = array_make(heap_allocator(), thread_count); defer (array_free(&worker_threads)); for_array(i, worker_threads) { gbThread *t = &worker_threads[i]; gb_thread_init(t); + char buffer[64]; + gb_snprintf(buffer, 64, "Parser Worker #%lld", i); + gb_thread_set_name(t, buffer); + gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]); } - defer (for_array(i, worker_threads) { + defer(for_array(i, worker_threads) { gb_thread_destroy(&worker_threads[i]); }); @@ -4744,27 +4807,51 @@ ParseFileError parse_packages(Parser *p, String init_filename) { for (;;) { bool are_any_alive = false; + for_array(i, worker_threads) { gbThread *t = &worker_threads[i]; - if (gb_thread_is_running(t)) { - are_any_alive = true; - } else if (curr_import_index < p->files_to_process.count) { - auto curr_err = cast(ParseFileError)t->return_value; - if (curr_err != ParseFile_None) { - array_add(&errors, curr_err); - } else { + ParserWorkerThreadData *data = &worker_threads_data[i]; + + if (gb_mutex_try_lock(&data->lock)) { + if (data->error_available) { + auto curr_err = data->err; + if (curr_err != ParseFile_None) { + array_add(&errors, curr_err); + } + + data->error_available = false; + } + + if (curr_import_index < p->files_to_process.count) { t->user_index = curr_import_index; curr_import_index++; - gb_thread_start(t, parse_worker_file_proc, p); are_any_alive = true; + + gb_semaphore_release(&data->resume_work); } + + gb_mutex_unlock(&data->lock); + } else { + //NOTE(thebirk): If we cant lock a thread it must be working + are_any_alive = true; } } - if (!are_any_alive && curr_import_index >= p->files_to_process.count) { + + //NOTE(thebirk): Everything collapses without this, but it really shouldn't! + gb_yield(); + + if ((!are_any_alive) && (curr_import_index >= p->files_to_process.count)) { break; } } + //NOTE(thebirk): Signal all workers to exit + for_array(i, worker_threads_data) { + ParserWorkerThreadData* data = &worker_threads_data[i]; + data->should_exit = true; + gb_semaphore_release(&data->resume_work); + } + if (errors.count > 0) { return errors[errors.count-1]; } -- cgit v1.2.3 From 97dfcffa761acdf22a12412db5610cc4beb2c4d6 Mon Sep 17 00:00:00 2001 From: thebirk Date: Mon, 26 Aug 2019 19:09:52 +0200 Subject: Fixed error where the parser would end early. --- src/parser.cpp | 34 +++++++++++++++++++++++++++------- src/parser.hpp | 1 + 2 files changed, 28 insertions(+), 7 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 0a2a3c1fc..d7e7f63b7 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4062,6 +4062,7 @@ bool init_parser(Parser *p) { array_init(&p->files_to_process, heap_allocator()); gb_mutex_init(&p->file_add_mutex); gb_mutex_init(&p->file_decl_mutex); + gb_semaphore_init(&p->worker_finished_semaphore); return true; } @@ -4087,6 +4088,7 @@ void destroy_parser(Parser *p) { map_destroy(&p->package_map); gb_mutex_destroy(&p->file_add_mutex); gb_mutex_destroy(&p->file_decl_mutex); + gb_semaphore_destroy(&p->worker_finished_semaphore); } @@ -4714,6 +4716,7 @@ GB_THREAD_PROC(parse_worker_file_proc) { data->err = process_imported_file(p, file_to_process); data->error_available = true; + gb_semaphore_release(&p->worker_finished_semaphore); gb_mutex_unlock(&data->lock); } @@ -4760,6 +4763,8 @@ ParseFileError parse_packages(Parser *p, String init_filename) { isize thread_count = gb_max(build_context.thread_count, 1); if (thread_count > 1) { isize volatile curr_import_index = 0; + +#if 0 isize initial_file_count = p->files_to_process.count; // NOTE(bill): Make sure that these are in parsed in this order for (isize i = 0; i < initial_file_count; i++) { @@ -4769,6 +4774,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } curr_import_index++; } +#endif auto worker_threads_data = array_make(heap_allocator(), thread_count); defer (array_free(&worker_threads_data)); @@ -4795,7 +4801,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { gbThread *t = &worker_threads[i]; gb_thread_init(t); char buffer[64]; - gb_snprintf(buffer, 64, "Parser Worker #%lld", i); + gb_snprintf(buffer, 64, "Parser Worker #%ll", i); gb_thread_set_name(t, buffer); gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]); } @@ -4806,7 +4812,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { auto errors = array_make(heap_allocator(), 0, 16); for (;;) { - bool are_any_alive = false; + int num_alive = 0; for_array(i, worker_threads) { gbThread *t = &worker_threads[i]; @@ -4825,7 +4831,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { if (curr_import_index < p->files_to_process.count) { t->user_index = curr_import_index; curr_import_index++; - are_any_alive = true; + num_alive += 1; gb_semaphore_release(&data->resume_work); } @@ -4833,14 +4839,28 @@ ParseFileError parse_packages(Parser *p, String init_filename) { gb_mutex_unlock(&data->lock); } else { //NOTE(thebirk): If we cant lock a thread it must be working - are_any_alive = true; + num_alive += 1; + + } + } + + while (num_alive > 0) { + isize prev_files_to_process = p->files_to_process.count; + gb_semaphore_wait(&p->worker_finished_semaphore); + num_alive -= 1; + + if (prev_files_to_process < p->files_to_process.count) { + if (num_alive > 0) { + //NOTE(thebirk): Recreate semaphore to avoid overflowing the counter. Only needs to happen when there are more threads alive + gb_semaphore_destroy(&p->worker_finished_semaphore); + gb_semaphore_init(&p->worker_finished_semaphore); + } + break; } } - //NOTE(thebirk): Everything collapses without this, but it really shouldn't! - gb_yield(); - if ((!are_any_alive) && (curr_import_index >= p->files_to_process.count)) { + if ((num_alive == 0) && (curr_import_index >= p->files_to_process.count)) { break; } } diff --git a/src/parser.hpp b/src/parser.hpp index 3489f1a9b..f1bbd8784 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -139,6 +139,7 @@ struct Parser { isize total_line_count; gbMutex file_add_mutex; gbMutex file_decl_mutex; + gbSemaphore worker_finished_semaphore; }; enum ProcInlining { -- cgit v1.2.3 From 4551521b2cc461d89d9d69ea441ae311a8607f94 Mon Sep 17 00:00:00 2001 From: thebirk Date: Mon, 26 Aug 2019 19:51:33 +0200 Subject: Im just trying things at this point, Bill should just squash this PR at merge time ;) --- src/parser.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index d7e7f63b7..97715b10e 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4690,6 +4690,7 @@ struct ParserWorkerThreadData { gbMutex lock; //NOTE(thebirk): All variables below are locked by this mutex bool error_available; + bool is_working; bool should_exit; }; @@ -4716,8 +4717,9 @@ GB_THREAD_PROC(parse_worker_file_proc) { data->err = process_imported_file(p, file_to_process); data->error_available = true; - gb_semaphore_release(&p->worker_finished_semaphore); + data->is_working = false; gb_mutex_unlock(&data->lock); + gb_semaphore_release(&p->worker_finished_semaphore); } //GB_PANIC("A worker thread should not be able to reach the end!!!"); @@ -4818,7 +4820,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { gbThread *t = &worker_threads[i]; ParserWorkerThreadData *data = &worker_threads_data[i]; - if (gb_mutex_try_lock(&data->lock)) { + if (!data->is_working && gb_mutex_try_lock(&data->lock)) { if (data->error_available) { auto curr_err = data->err; if (curr_err != ParseFile_None) { @@ -4834,6 +4836,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { num_alive += 1; gb_semaphore_release(&data->resume_work); + data->is_working = true; } gb_mutex_unlock(&data->lock); @@ -4844,20 +4847,23 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } } + /* while (num_alive > 0) { isize prev_files_to_process = p->files_to_process.count; gb_semaphore_wait(&p->worker_finished_semaphore); num_alive -= 1; - if (prev_files_to_process < p->files_to_process.count) { + if ((prev_files_to_process < p->files_to_process.count)) { if (num_alive > 0) { //NOTE(thebirk): Recreate semaphore to avoid overflowing the counter. Only needs to happen when there are more threads alive - gb_semaphore_destroy(&p->worker_finished_semaphore); - gb_semaphore_init(&p->worker_finished_semaphore); + //gb_semaphore_destroy(&p->worker_finished_semaphore); + //gb_semaphore_init(&p->worker_finished_semaphore); } + printf("Early out!\n"); break; } } + */ if ((num_alive == 0) && (curr_import_index >= p->files_to_process.count)) { -- cgit v1.2.3 From 6a8b3fee38b3bb05ffc599d619e78472ad8b56a0 Mon Sep 17 00:00:00 2001 From: thebirk Date: Mon, 26 Aug 2019 20:23:52 +0200 Subject: Removed gb_thread_set_name because it segfaults on linux. --- src/parser.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 97715b10e..70cc37433 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4789,6 +4789,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { data->err = ParseFile_None; data->should_exit = false; data->error_available = false; + data->is_working = false; } defer(for_array(i, worker_threads_data) { ParserWorkerThreadData *data = &worker_threads_data[i]; @@ -4802,9 +4803,9 @@ ParseFileError parse_packages(Parser *p, String init_filename) { for_array(i, worker_threads) { gbThread *t = &worker_threads[i]; gb_thread_init(t); - char buffer[64]; - gb_snprintf(buffer, 64, "Parser Worker #%ll", i); - gb_thread_set_name(t, buffer); + //char buffer[64]; + //gb_snprintf(buffer, 64, "Parser Worker #%ll", i); + //gb_thread_set_name(t, buffer); gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]); } defer(for_array(i, worker_threads) { -- cgit v1.2.3 From d76249d90b4d2812c2d8c2d25d854a3e9361b510 Mon Sep 17 00:00:00 2001 From: thebirk Date: Thu, 29 Aug 2019 20:27:38 +0200 Subject: Cleaned up parse_packages and the worker proc. --- src/parser.cpp | 50 ++++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 38 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 70cc37433..3fc999a2c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4681,14 +4681,13 @@ skip: return ParseFile_None; } -#if 1 struct ParserWorkerThreadData { Parser *parser; - - gbSemaphore resume_work; //NOTE(thebirk): Use to signal that the worker thead has a new work to do ParseFileError err; - gbMutex lock; //NOTE(thebirk): All variables below are locked by this mutex + gbSemaphore resume_work; + gbMutex lock; + bool error_available; bool is_working; bool should_exit; @@ -4724,18 +4723,6 @@ GB_THREAD_PROC(parse_worker_file_proc) { //GB_PANIC("A worker thread should not be able to reach the end!!!"); } -#else -GB_THREAD_PROC(parse_worker_file_proc) { - if (thread == nullptr) return 0; - auto *p = cast(Parser *)thread->user_data; - isize index = thread->user_index; - gb_mutex_lock(&p->file_add_mutex); - auto file_to_process = p->files_to_process[index]; - gb_mutex_unlock(&p->file_add_mutex); - ParseFileError err = process_imported_file(p, file_to_process); - return cast(isize)err; -} -#endif ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); @@ -4767,6 +4754,8 @@ ParseFileError parse_packages(Parser *p, String init_filename) { isize volatile curr_import_index = 0; #if 0 + //NOTE(thebirk): Leaving this piece of code behind if it turns out we need it, yes I know git exists + isize initial_file_count = p->files_to_process.count; // NOTE(bill): Make sure that these are in parsed in this order for (isize i = 0; i < initial_file_count; i++) { @@ -4803,8 +4792,11 @@ ParseFileError parse_packages(Parser *p, String init_filename) { for_array(i, worker_threads) { gbThread *t = &worker_threads[i]; gb_thread_init(t); - //char buffer[64]; - //gb_snprintf(buffer, 64, "Parser Worker #%ll", i); + //NOTE(thebirk): This crashes on linux. In addition to that the method used on windows does + // not get picked up by a lot of tools look into using SetThreadDescription + // when running on new enough windows 10 builds + //char buffer[32]; + //gb_snprintf(buffer, 32, "Parser Worker #%ll", i); //gb_thread_set_name(t, buffer); gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]); } @@ -4844,32 +4836,14 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } else { //NOTE(thebirk): If we cant lock a thread it must be working num_alive += 1; - - } - } - - /* - while (num_alive > 0) { - isize prev_files_to_process = p->files_to_process.count; - gb_semaphore_wait(&p->worker_finished_semaphore); - num_alive -= 1; - - if ((prev_files_to_process < p->files_to_process.count)) { - if (num_alive > 0) { - //NOTE(thebirk): Recreate semaphore to avoid overflowing the counter. Only needs to happen when there are more threads alive - //gb_semaphore_destroy(&p->worker_finished_semaphore); - //gb_semaphore_init(&p->worker_finished_semaphore); - } - printf("Early out!\n"); - break; } } - */ - if ((num_alive == 0) && (curr_import_index >= p->files_to_process.count)) { break; } + + gb_yield(); } //NOTE(thebirk): Signal all workers to exit -- cgit v1.2.3 From 4dade346033bc3d19879bee1823abf3d9e230281 Mon Sep 17 00:00:00 2001 From: thebirk Date: Thu, 29 Aug 2019 20:34:09 +0200 Subject: Removed unused semaphore on Parser. --- src/parser.cpp | 2 -- src/parser.hpp | 1 - 2 files changed, 3 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 3fc999a2c..f08688b5f 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4062,7 +4062,6 @@ bool init_parser(Parser *p) { array_init(&p->files_to_process, heap_allocator()); gb_mutex_init(&p->file_add_mutex); gb_mutex_init(&p->file_decl_mutex); - gb_semaphore_init(&p->worker_finished_semaphore); return true; } @@ -4088,7 +4087,6 @@ void destroy_parser(Parser *p) { map_destroy(&p->package_map); gb_mutex_destroy(&p->file_add_mutex); gb_mutex_destroy(&p->file_decl_mutex); - gb_semaphore_destroy(&p->worker_finished_semaphore); } diff --git a/src/parser.hpp b/src/parser.hpp index f1bbd8784..3489f1a9b 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -139,7 +139,6 @@ struct Parser { isize total_line_count; gbMutex file_add_mutex; gbMutex file_decl_mutex; - gbSemaphore worker_finished_semaphore; }; enum ProcInlining { -- cgit v1.2.3 From f921a91fc8a1e2674279e79c7357adfb0016b360 Mon Sep 17 00:00:00 2001 From: thebirk Date: Thu, 29 Aug 2019 20:35:12 +0200 Subject: Properly removed the semaphore. --- src/parser.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index f08688b5f..31d21b737 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4716,7 +4716,6 @@ GB_THREAD_PROC(parse_worker_file_proc) { data->error_available = true; data->is_working = false; gb_mutex_unlock(&data->lock); - gb_semaphore_release(&p->worker_finished_semaphore); } //GB_PANIC("A worker thread should not be able to reach the end!!!"); -- cgit v1.2.3 From b311540b1672129e87a7249650a19cf11d2fccef Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 31 Aug 2019 14:48:56 +0100 Subject: Make `require_results` an attribute rather than a suffix tag for procedures --- core/odin/ast/ast.odin | 1 - core/odin/parser/parser.odin | 2 -- src/check_decl.cpp | 7 +++---- src/checker.cpp | 6 ++++++ src/checker.hpp | 1 + src/parser.cpp | 6 +++++- 6 files changed, 15 insertions(+), 8 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/ast/ast.odin b/core/odin/ast/ast.odin index 3655a601d..87893e0e4 100644 --- a/core/odin/ast/ast.odin +++ b/core/odin/ast/ast.odin @@ -5,7 +5,6 @@ import "core:odin/token" Proc_Tag :: enum { Bounds_Check, No_Bounds_Check, - Require_Results, } Proc_Tags :: distinct bit_set[Proc_Tag; u32]; diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index a9961fafd..1f5add69a 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1745,8 +1745,6 @@ parse_proc_tags :: proc(p: ^Parser) -> (tags: ast.Proc_Tags) { ident := expect_token(p, token.Ident); switch ident.text { - case "require_results": - tags |= {.Require_Results}; case "bounds_check": tags |= {.Bounds_Check}; case "no_bounds_check": diff --git a/src/check_decl.cpp b/src/check_decl.cpp index b5b9dc61d..7e019d82b 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -654,7 +654,6 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) { bool is_foreign = e->Procedure.is_foreign; bool is_export = e->Procedure.is_export; - bool is_require_results = (pl->tags & ProcTag_require_results) != 0; if (e->pkg != nullptr && e->token.string == "main") { if (pt->param_count != 0 || @@ -714,10 +713,10 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) { } } - if (pt->result_count == 0 && is_require_results) { - error(pl->type, "'#require_results' is not needed on a procedure with no results"); + if (pt->result_count == 0 && ac.require_results) { + error(pl->type, "'require_results' is not needed on a procedure with no results"); } else { - pt->require_results = is_require_results; + pt->require_results = ac.require_results; } if (ac.link_name.len > 0) { diff --git a/src/checker.cpp b/src/checker.cpp index 9fa6d4555..b00b4bbac 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2187,6 +2187,12 @@ DECL_ATTRIBUTE_PROC(proc_decl_attribute) { error(elem, "Expected a string value for '%.*s'", LIT(name)); } return true; + } else if (name == "require_results") { + if (value != nullptr) { + error(elem, "Expected no value for '%.*s'", LIT(name)); + } + ac->require_results = true; + return true; } return false; } diff --git a/src/checker.hpp b/src/checker.hpp index a37a87b87..58cb01a82 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -97,6 +97,7 @@ struct DeferredProcedure { struct AttributeContext { bool is_export; bool is_static; + bool require_results; String link_name; String link_prefix; isize init_expr_list_count; diff --git a/src/parser.cpp b/src/parser.cpp index 34a42ba3d..e92489020 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1855,7 +1855,7 @@ Ast *parse_operand(AstFile *f, bool lhs) { } if (tags != 0) { - syntax_error(token, "A procedure type cannot have tags"); + syntax_error(token, "A procedure type cannot have suffix tags"); } return type; @@ -2828,6 +2828,10 @@ Ast *parse_proc_type(AstFile *f, Token proc_token) { u64 tags = 0; parse_proc_tags(f, &tags); + if ((tags & ProcTag_require_results) != 0) { + syntax_error(f->curr_token, "#require_results has now been replaced as an attribute @(require_results) on the declaration"); + tags &= ~ProcTag_require_results; + } bool is_generic = false; -- cgit v1.2.3 From b9d3129fb3a4ba7ef49cea69d086a7f705819f2e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 31 Aug 2019 20:13:28 +0100 Subject: `where` clauses for procedure literals --- core/odin/ast/ast.odin | 2 + core/odin/parser/parser.odin | 18 +++++++ core/odin/token/token.odin | 2 + examples/demo/demo.odin | 65 ++++++++++++++++++++++++ src/check_decl.cpp | 33 ++++++++++--- src/check_expr.cpp | 115 +++++++++++++++++++++++++++++++++++++++++-- src/checker.cpp | 8 +++ src/parser.cpp | 50 ++++++++++++++----- src/parser.hpp | 2 + src/tokenizer.cpp | 1 + 10 files changed, 272 insertions(+), 24 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/ast/ast.odin b/core/odin/ast/ast.odin index 87893e0e4..f1ea79584 100644 --- a/core/odin/ast/ast.odin +++ b/core/odin/ast/ast.odin @@ -99,6 +99,8 @@ Proc_Lit :: struct { body: ^Stmt, tags: Proc_Tags, inlining: Proc_Inlining, + where_token: token.Token, + where_clauses: []^Expr, } Comp_Lit :: struct { diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index 1f5add69a..a0d4d639e 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1985,13 +1985,29 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { type := parse_proc_type(p, tok); + where_token: token.Token; + where_clauses: []^ast.Expr; + if (p.curr_tok.kind == token.Where) { + where_token = expect_token(p, token.Where); + prev_level := p.expr_level; + p.expr_level = -1; + where_clauses = parse_rhs_expr_list(p); + p.expr_level = prev_level; + } + if p.allow_type && p.expr_level < 0 { + if where_token.kind != token.Invalid { + error(p, where_token.pos, "'where' clauses are not allowed on procedure types"); + } return type; } body: ^ast.Stmt; if allow_token(p, token.Undef) { // Okay + if where_token.kind != token.Invalid { + error(p, where_token.pos, "'where' clauses are not allowed on procedure literals without a defined body (replaced with ---"); + } } else if p.curr_tok.kind == token.Open_Brace { prev_proc := p.curr_proc; p.curr_proc = type; @@ -2009,6 +2025,8 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { pl := ast.new(ast.Proc_Lit, tok.pos, end_pos(p.prev_tok)); pl.type = type; pl.body = body; + pl.where_token = where_token; + pl.where_clauses = where_clauses; return pl; case token.Dollar: diff --git a/core/odin/token/token.odin b/core/odin/token/token.odin index d41fa6d05..737ff3586 100644 --- a/core/odin/token/token.odin +++ b/core/odin/token/token.odin @@ -118,6 +118,7 @@ using Kind :: enum u32 { Package, Typeid, When, + Where, If, Else, For, @@ -252,6 +253,7 @@ tokens := [Kind.COUNT]string { "package", "typeid", "when", + "where", "if", "else", "for", diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index fe390e5b0..92d1c17bd 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -4,6 +4,7 @@ import "core:fmt" import "core:mem" import "core:os" import "core:reflect" +import "intrinsics" when os.OS == "windows" { import "core:thread" @@ -1094,6 +1095,69 @@ inline_for_statement :: proc() { } } +procedure_where_clauses :: proc() { + fmt.println("\n#procedure 'where' clauses"); + + { // Sanity checks + simple_sanity_check :: proc(x: [2]int) + where len(x) > 1, + type_of(x) == [2]int { + fmt.println(x); + } + } + { // Parametric polymorphism checks + cross_2d :: proc(a, b: $T/[2]$E) -> E + where intrinsics.type_is_numeric(E) { + return a.x*b.y - a.y*b.x; + } + cross_3d :: proc(a, b: $T/[3]$E) -> T + where intrinsics.type_is_numeric(E) { + x := a.y*b.z - a.z*b.y; + y := a.z*b.x - a.x*b.z; + z := a.x*b.y - a.y*b.z; + return T{x, y, z}; + } + + a := [2]int{1, 2}; + b := [2]int{5, -3}; + fmt.println(cross_2d(a, b)); + + x := [3]f32{1, 4, 9}; + y := [3]f32{-5, 0, 3}; + fmt.println(cross_3d(x, y)); + + // Failure case + // i := [2]bool{true, false}; + // j := [2]bool{false, true}; + // fmt.println(cross_2d(i, j)); + + } + + { // Procedure groups usage + foo :: proc(x: [$N]int) -> bool + where N > 2 { + fmt.println(#procedure, "was called with the parameter", x); + return true; + } + + bar :: proc(x: [$N]int) -> bool + where 0 < N, + N <= 2 { + fmt.println(#procedure, "was called with the parameter", x); + return false; + } + + baz :: proc{foo, bar}; + + x := [3]int{1, 2, 3}; + y := [2]int{4, 9}; + ok_x := baz(x); + ok_y := baz(y); + assert(ok_x == true); + assert(ok_y == false); + } +} + main :: proc() { when true { general_stuff(); @@ -1115,5 +1179,6 @@ main :: proc() { reflection(); quaternions(); inline_for_statement(); + procedure_where_clauses(); } } diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 7e019d82b..156c874ce 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -936,7 +936,6 @@ void check_proc_group_decl(CheckerContext *ctx, Entity *pg_entity, DeclInfo *d) ptr_set_destroy(&entity_set); - for_array(j, pge->entities) { Entity *p = pge->entities[j]; if (p->type == t_invalid) { @@ -962,27 +961,40 @@ void check_proc_group_decl(CheckerContext *ctx, Entity *pg_entity, DeclInfo *d) defer (end_error_block()); ProcTypeOverloadKind kind = are_proc_types_overload_safe(p->type, q->type); - switch (kind) { + bool both_have_where_clauses = false; + if (p->decl_info->proc_lit != nullptr && q->decl_info->proc_lit != nullptr) { + GB_ASSERT(p->decl_info->proc_lit->kind == Ast_ProcLit); + GB_ASSERT(q->decl_info->proc_lit->kind == Ast_ProcLit); + auto pl = &p->decl_info->proc_lit->ProcLit; + auto ql = &q->decl_info->proc_lit->ProcLit; + + // Allow collisions if the procedures both have 'where' clauses and are both polymorphic + bool pw = pl->where_token.kind != Token_Invalid && is_type_polymorphic(p->type, true); + bool qw = ql->where_token.kind != Token_Invalid && is_type_polymorphic(q->type, true); + both_have_where_clauses = pw && qw; + } + + if (!both_have_where_clauses) switch (kind) { case ProcOverload_Identical: - error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in this scope", LIT(name)); + error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; // case ProcOverload_CallingConvention: - // error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in this scope", LIT(name)); + // error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); // is_invalid = true; // break; case ProcOverload_ParamVariadic: - error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in this scope", LIT(name)); + error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; case ProcOverload_ResultCount: case ProcOverload_ResultTypes: - error(p->token, "Overloaded procedure '%.*s' as the same parameters but different results in this scope", LIT(name)); + error(p->token, "Overloaded procedure '%.*s' as the same parameters but different results in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; case ProcOverload_Polymorphic: #if 0 - error(p->token, "Overloaded procedure '%.*s' has a polymorphic counterpart in this scope which is not allowed", LIT(name)); + error(p->token, "Overloaded procedure '%.*s' has a polymorphic counterpart in the procedure group '%.*s' which is not allowed", LIT(name), LIT(proc_group_name)); is_invalid = true; #endif break; @@ -1163,6 +1175,13 @@ void check_proc_body(CheckerContext *ctx_, Token token, DeclInfo *decl, Type *ty } } + + bool where_clause_ok = evaluate_where_clauses(ctx, decl, true); + if (!where_clause_ok) { + // NOTE(bill, 2019-08-31): Don't check the body as the where clauses failed + return; + } + check_open_scope(ctx, body); { for_array(i, using_entities) { diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 86cba0f94..4c8cd61a7 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -5470,6 +5470,74 @@ Entity **populate_proc_parameter_list(CheckerContext *c, Type *proc_type, isize return lhs; } + +bool evaluate_where_clauses(CheckerContext *ctx, DeclInfo *decl, bool print_err) { + Ast *proc_lit = decl->proc_lit; + GB_ASSERT(proc_lit != nullptr); + GB_ASSERT(proc_lit->kind == Ast_ProcLit); + + if (proc_lit->ProcLit.where_token.kind != Token_Invalid) { + auto &clauses = proc_lit->ProcLit.where_clauses; + for_array(i, clauses) { + Ast *clause = clauses[i]; + Operand o = {}; + check_expr(ctx, &o, clause); + if (o.mode != Addressing_Constant) { + if (print_err) error(clause, "'where' clauses expect a constant boolean evaluation"); + return false; + } else if (o.value.kind != ExactValue_Bool) { + if (print_err) error(clause, "'where' clauses expect a constant boolean evaluation"); + return false; + } else if (!o.value.value_bool) { + if (print_err) { + gbString str = expr_to_string(clause); + error(clause, "'where' clause evaluated to false:\n\t%s", str); + gb_string_free(str); + + if (decl->scope != nullptr) { + isize print_count = 0; + for_array(j, decl->scope->elements.entries) { + Entity *e = decl->scope->elements.entries[j].value; + switch (e->kind) { + case Entity_TypeName: { + if (print_count == 0) error_line("\n\tWith the following definitions:\n"); + + gbString str = type_to_string(e->type); + error_line("\t\t%.*s :: %s;\n", LIT(e->token.string), str); + gb_string_free(str); + print_count += 1; + break; + } + case Entity_Constant: { + if (print_count == 0) error_line("\n\tWith the following definitions:\n"); + + gbString str = exact_value_to_string(e->Constant.value); + if (is_type_untyped(e->type)) { + error_line("\t\t%.*s :: %s;\n", LIT(e->token.string), str); + } else { + gbString t = type_to_string(e->type); + error_line("\t\t%.*s : %s : %s;\n", LIT(e->token.string), t, str); + gb_string_free(t); + } + gb_string_free(str); + + print_count += 1; + break; + } + } + } + } + + } + return false; + } + } + } + + return true; +} + + CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type *proc_type, Ast *call) { ast_node(ce, CallExpr, call); @@ -5710,11 +5778,26 @@ CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type err = call_checker(&ctx, call, pt, p, operands, CallArgumentMode_NoErrors, &data); - if (err == CallArgumentError_None) { - valids[valid_count].index = i; - valids[valid_count].score = data.score; - valid_count++; + if (err != CallArgumentError_None) { + continue; } + if (data.gen_entity != nullptr) { + Entity *e = data.gen_entity; + DeclInfo *decl = data.gen_entity->decl_info; + ctx.scope = decl->scope; + ctx.decl = decl; + ctx.proc_name = e->token.string; + ctx.curr_proc_decl = decl; + ctx.curr_proc_sig = e->type; + + if (!evaluate_where_clauses(&ctx, decl, false)) { + continue; + } + } + + valids[valid_count].index = i; + valids[valid_count].score = data.score; + valid_count++; } } @@ -5822,7 +5905,29 @@ CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type if (proc->kind == Entity_Variable) { sep = ":="; } - error_line("\t%.*s %s %s at %.*s(%td:%td)\n", LIT(name), sep, pt, LIT(pos.file), pos.line, pos.column); + error_line("\t%.*s %s %s ", LIT(name), sep, pt); + if (proc->decl_info->proc_lit != nullptr) { + GB_ASSERT(proc->decl_info->proc_lit->kind == Ast_ProcLit); + auto *pl = &proc->decl_info->proc_lit->ProcLit; + if (pl->where_token.kind != Token_Invalid) { + error_line("\n\t\twhere "); + for_array(j, pl->where_clauses) { + Ast *clause = pl->where_clauses[j]; + if (j != 0) { + error_line("\t\t "); + } + gbString str = expr_to_string(clause); + error_line("%s", str); + gb_string_free(str); + + if (j != pl->where_clauses.count-1) { + error_line(","); + } + } + error_line("\n\t"); + } + } + error_line("at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column); // error_line("\t%.*s %s %s at %.*s(%td:%td) %lld\n", LIT(name), sep, pt, LIT(pos.file), pos.line, pos.column, valids[i].score); } result_type = t_invalid; diff --git a/src/checker.cpp b/src/checker.cpp index b00b4bbac..8fe71b63c 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -3697,6 +3697,14 @@ void check_proc_info(Checker *c, ProcInfo pi) { return; } + if (pt->is_polymorphic && pt->is_poly_specialized) { + Entity *e = pi.decl->entity; + if ((e->flags & EntityFlag_Used) == 0) { + // NOTE(bill, 2019-08-31): It was never used, don't check + return; + } + } + bool bounds_check = (pi.tags & ProcTag_bounds_check) != 0; bool no_bounds_check = (pi.tags & ProcTag_no_bounds_check) != 0; diff --git a/src/parser.cpp b/src/parser.cpp index e92489020..8490b0e00 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -144,6 +144,7 @@ Ast *clone_ast(Ast *node) { case Ast_ProcLit: n->ProcLit.type = clone_ast(n->ProcLit.type); n->ProcLit.body = clone_ast(n->ProcLit.body); + n->ProcLit.where_clauses = clone_ast_array(n->ProcLit.where_clauses); break; case Ast_CompoundLit: n->CompoundLit.type = clone_ast(n->CompoundLit.type); @@ -612,11 +613,13 @@ Ast *ast_proc_group(AstFile *f, Token token, Token open, Token close, Array const &where_clauses) { Ast *result = alloc_ast_node(f, Ast_ProcLit); result->ProcLit.type = type; result->ProcLit.body = body; result->ProcLit.tags = tags; + result->ProcLit.where_token = where_token; + result->ProcLit.where_clauses = where_clauses; return result; } @@ -1827,15 +1830,41 @@ Ast *parse_operand(AstFile *f, bool lhs) { } Ast *type = parse_proc_type(f, token); + Token where_token = {}; + Array where_clauses = {}; + u64 tags = 0; + + if (f->curr_token.kind == Token_where) { + where_token = expect_token(f, Token_where); + isize prev_level = f->expr_level; + f->expr_level = -1; + where_clauses = parse_rhs_expr_list(f); + f->expr_level = prev_level; + } + + parse_proc_tags(f, &tags); + if ((tags & ProcTag_require_results) != 0) { + syntax_error(f->curr_token, "#require_results has now been replaced as an attribute @(require_results) on the declaration"); + tags &= ~ProcTag_require_results; + } + GB_ASSERT(type->kind == Ast_ProcType); + type->ProcType.tags = tags; if (f->allow_type && f->expr_level < 0) { + if (tags != 0) { + syntax_error(token, "A procedure type cannot have suffix tags"); + } + if (where_token.kind != Token_Invalid) { + syntax_error(where_token, "'where' clauses are not allowed on procedure types"); + } return type; } - u64 tags = type->ProcType.tags; - if (allow_token(f, Token_Undef)) { - return ast_proc_lit(f, type, nullptr, tags); + if (where_token.kind != Token_Invalid) { + syntax_error(where_token, "'where' clauses are not allowed on procedure literals without a defined body (replaced with ---)"); + } + return ast_proc_lit(f, type, nullptr, tags, where_token, where_clauses); } else if (f->curr_token.kind == Token_OpenBrace) { Ast *curr_proc = f->curr_proc; Ast *body = nullptr; @@ -1843,7 +1872,7 @@ Ast *parse_operand(AstFile *f, bool lhs) { body = parse_body(f); f->curr_proc = curr_proc; - return ast_proc_lit(f, type, body, tags); + return ast_proc_lit(f, type, body, tags, where_token, where_clauses); } else if (allow_token(f, Token_do)) { Ast *curr_proc = f->curr_proc; Ast *body = nullptr; @@ -1851,12 +1880,15 @@ Ast *parse_operand(AstFile *f, bool lhs) { body = convert_stmt_to_body(f, parse_stmt(f)); f->curr_proc = curr_proc; - return ast_proc_lit(f, type, body, tags); + return ast_proc_lit(f, type, body, tags, where_token, where_clauses); } if (tags != 0) { syntax_error(token, "A procedure type cannot have suffix tags"); } + if (where_token.kind != Token_Invalid) { + syntax_error(where_token, "'where' clauses are not allowed on procedure types"); + } return type; } @@ -2827,12 +2859,6 @@ Ast *parse_proc_type(AstFile *f, Token proc_token) { results = parse_results(f, &diverging); u64 tags = 0; - parse_proc_tags(f, &tags); - if ((tags & ProcTag_require_results) != 0) { - syntax_error(f->curr_token, "#require_results has now been replaced as an attribute @(require_results) on the declaration"); - tags &= ~ProcTag_require_results; - } - bool is_generic = false; for_array(i, params->FieldList.list) { diff --git a/src/parser.hpp b/src/parser.hpp index 32398592e..26536fe56 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -229,6 +229,8 @@ enum StmtAllowFlag { Ast *body; \ u64 tags; \ ProcInlining inlining; \ + Token where_token; \ + Array where_clauses; \ }) \ AST_KIND(CompoundLit, "compound literal", struct { \ Ast *type; \ diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 618de54b9..d5e04aa1e 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -86,6 +86,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \ TOKEN_KIND(Token_package, "package"), \ TOKEN_KIND(Token_typeid, "typeid"), \ TOKEN_KIND(Token_when, "when"), \ + TOKEN_KIND(Token_where, "where"), \ TOKEN_KIND(Token_if, "if"), \ TOKEN_KIND(Token_else, "else"), \ TOKEN_KIND(Token_for, "for"), \ -- cgit v1.2.3 From 657103c4cff8f63cfa617d8c4371fd29df7b41a2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 1 Sep 2019 20:02:39 +0100 Subject: ThreadPool for the parser --- core/log/file_console_logger.odin | 32 ++--- core/sort/sort.odin | 7 +- src/check_expr.cpp | 2 +- src/common.cpp | 5 +- src/gb/gb.h | 2 +- src/parser.cpp | 243 ++++++++------------------------------ src/parser.hpp | 13 ++ 7 files changed, 86 insertions(+), 218 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/log/file_console_logger.odin b/core/log/file_console_logger.odin index 3778d3e53..6bddf7ffb 100644 --- a/core/log/file_console_logger.odin +++ b/core/log/file_console_logger.odin @@ -14,18 +14,18 @@ Level_Headers := []string{ }; Default_Console_Logger_Opts :: Options{ - Option.Level, - Option.Terminal_Color, - Option.Short_File_Path, - Option.Line, - Option.Procedure, + .Level, + .Terminal_Color, + .Short_File_Path, + .Line, + .Procedure, } | Full_Timestamp_Opts; Default_File_Logger_Opts :: Options{ - Option.Level, - Option.Short_File_Path, - Option.Line, - Option.Procedure, + .Level, + .Short_File_Path, + .Line, + .Procedure, } | Full_Timestamp_Opts; @@ -109,10 +109,10 @@ do_level_header :: proc(opts : Options, level : Level, str : ^strings.Builder) { case Level.Error, Level.Fatal : col = RED; } - if Option.Level in opts { - if Option.Terminal_Color in opts do fmt.sbprint(str, col); + if .Level in opts { + if .Terminal_Color in opts do fmt.sbprint(str, col); fmt.sbprint(str, Level_Headers[level]); - if Option.Terminal_Color in opts do fmt.sbprint(str, RESET); + if .Terminal_Color in opts do fmt.sbprint(str, RESET); } } @@ -120,7 +120,7 @@ do_location_header :: proc(opts : Options, buf : ^strings.Builder, location := # if Location_Header_Opts & opts != nil do fmt.sbprint(buf, "["); else do return; file := location.file_path; - if Option.Short_File_Path in opts { + if .Short_File_Path in opts { when os.OS == "windows" do delimiter := '\\'; else do delimiter := '/'; last := 0; for r, i in location.file_path do if r == delimiter do last = i+1; @@ -129,13 +129,13 @@ do_location_header :: proc(opts : Options, buf : ^strings.Builder, location := # if Location_File_Opts & opts != nil do fmt.sbprint(buf, file); - if Option.Procedure in opts { + if .Procedure in opts { if Location_File_Opts & opts != nil do fmt.sbprint(buf, "."); fmt.sbprintf(buf, "%s()", location.procedure); } - if Option.Line in opts { - if Location_File_Opts & opts != nil || Option.Procedure in opts do fmt.sbprint(buf, ":"); + if .Line in opts { + if Location_File_Opts & opts != nil || .Procedure in opts do fmt.sbprint(buf, ":"); fmt.sbprint(buf, location.line); } diff --git a/core/sort/sort.odin b/core/sort/sort.odin index 3c8366def..86953d7dd 100644 --- a/core/sort/sort.odin +++ b/core/sort/sort.odin @@ -1,6 +1,7 @@ package sort import "core:mem" +import "intrinsics" bubble_sort_proc :: proc(array: $A/[]$T, f: proc(T, T) -> int) { assert(f != nil); @@ -26,7 +27,7 @@ bubble_sort_proc :: proc(array: $A/[]$T, f: proc(T, T) -> int) { } } -bubble_sort :: proc(array: $A/[]$T) { +bubble_sort :: proc(array: $A/[]$T) where intrinsics.type_is_ordered(T) { count := len(array); init_j, last_j := 0, count-1; @@ -73,7 +74,7 @@ quick_sort_proc :: proc(array: $A/[]$T, f: proc(T, T) -> int) { quick_sort_proc(a[i:n], f); } -quick_sort :: proc(array: $A/[]$T) { +quick_sort :: proc(array: $A/[]$T) where intrinsics.type_is_ordered(T) { a := array; n := len(a); if n < 2 do return; @@ -146,7 +147,7 @@ merge_sort_proc :: proc(array: $A/[]$T, f: proc(T, T) -> int) { if M & 1 == 0 do copy(arr2, arr1); } -merge_sort :: proc(array: $A/[]$T) { +merge_sort :: proc(array: $A/[]$T) where intrinsics.type_is_ordered(T) { merge_slices :: proc(arr1, arr2, out: A) { N1, N2 := len(arr1), len(arr2); i, j := 0, 0; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 4c8cd61a7..ca1aed970 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -159,7 +159,7 @@ isize check_is_assignable_to_using_subtype(Type *src, Type *dst, isize level = 0 src = base_type(src); if (!is_type_struct(src)) { - return false; + return 0; } for_array(i, src->Struct.fields) { diff --git a/src/common.cpp b/src/common.cpp index 8085e895c..db6505a36 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -10,13 +10,11 @@ #define GB_IMPLEMENTATION #include "gb/gb.h" - #include #include #include - template gb_inline U bit_cast(V &v) { return reinterpret_cast(v); } @@ -331,7 +329,7 @@ void mul_overflow_u64(u64 x, u64 y, u64 *lo, u64 *hi) { #include "ptr_set.cpp" #include "string_set.cpp" #include "priority_queue.cpp" - +#include "thread_pool.cpp" gb_global String global_module_path = {0}; @@ -873,7 +871,6 @@ ReadDirectoryError read_directory(String path, Array *fi) { info.size = size; info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; array_add(fi, info); - } while (FindNextFileW(find_file, &file_data)); if (fi->count == 0) { diff --git a/src/gb/gb.h b/src/gb/gb.h index adeb554b2..5e74ff2d8 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -4591,7 +4591,7 @@ gb_inline void gb_semaphore_release(gbSemaphore *s) { gb_semaphore_post(s, 1); } gb_inline void gb_semaphore_init (gbSemaphore *s) { s->win32_handle = CreateSemaphoreA(NULL, 0, I32_MAX, NULL); } gb_inline void gb_semaphore_destroy(gbSemaphore *s) { CloseHandle(s->win32_handle); } gb_inline void gb_semaphore_post (gbSemaphore *s, i32 count) { ReleaseSemaphore(s->win32_handle, count, NULL); } - gb_inline void gb_semaphore_wait (gbSemaphore *s) { WaitForSingleObject(s->win32_handle, INFINITE); } + gb_inline void gb_semaphore_wait (gbSemaphore *s) { WaitForSingleObjectEx(s->win32_handle, INFINITE, FALSE); } #elif defined(GB_SYSTEM_OSX) gb_inline void gb_semaphore_init (gbSemaphore *s) { semaphore_create(mach_task_self(), &s->osx_handle, SYNC_POLICY_FIFO, 0); } diff --git a/src/parser.cpp b/src/parser.cpp index 8490b0e00..a210eb661 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4207,16 +4207,30 @@ void parser_add_package(Parser *p, AstPackage *pkg) { error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name)); GB_ASSERT((*found)->files.count > 0); TokenPos pos = (*found)->files[0]->package_token.pos; - gb_printf_err("\tpreviously declared at %.*s(%td:%td)", LIT(pos.file), pos.line, pos.column); + error_line("\tpreviously declared at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column); } else { map_set(&p->package_map, key, pkg); } } } +ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_file); + +WORKER_TASK_PROC(parser_worker_proc) { + ParserWorkerData *wd = cast(ParserWorkerData *)data; + ParseFileError err = process_imported_file(wd->parser, wd->imported_file); + return cast(isize)err; +} + + void parser_add_file_to_process(Parser *p, AstPackage *pkg, FileInfo fi, TokenPos pos) { + // TODO(bill): Use a better allocator ImportedFile f = {pkg, fi, pos, p->files_to_process.count}; + auto wd = gb_alloc_item(heap_allocator(), ParserWorkerData); + wd->parser = p; + wd->imported_file = f; array_add(&p->files_to_process, f); + thread_pool_add_task(&parser_thread_pool, parser_worker_proc, wd); } @@ -4717,9 +4731,9 @@ bool parse_file(Parser *p, AstFile *f) { } -ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) { +ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_file) { AstPackage *pkg = imported_file.pkg; - FileInfo *fi = &imported_file.fi; + FileInfo const *fi = &imported_file.fi; TokenPos pos = imported_file.pos; AstFile *file = gb_alloc_item(heap_allocator(), AstFile); @@ -4737,35 +4751,32 @@ ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) { error(pos, "Initial file is empty - %.*s\n", LIT(p->init_fullpath)); gb_exit(1); } - goto skip; - } + } else { + switch (err) { + case ParseFile_WrongExtension: + error(pos, "Failed to parse file: %.*s; invalid file extension: File must have the extension '.odin'", LIT(fi->name)); + break; + case ParseFile_InvalidFile: + error(pos, "Failed to parse file: %.*s; invalid file or cannot be found", LIT(fi->name)); + break; + case ParseFile_Permission: + error(pos, "Failed to parse file: %.*s; file permissions problem", LIT(fi->name)); + break; + case ParseFile_NotFound: + error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi->name), LIT(fi->fullpath)); + break; + case ParseFile_InvalidToken: + error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi->name)); + break; + case ParseFile_EmptyFile: + error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name)); + break; + } - switch (err) { - case ParseFile_WrongExtension: - error(pos, "Failed to parse file: %.*s; invalid file extension: File must have the extension '.odin'", LIT(fi->name)); - break; - case ParseFile_InvalidFile: - error(pos, "Failed to parse file: %.*s; invalid file or cannot be found", LIT(fi->name)); - break; - case ParseFile_Permission: - error(pos, "Failed to parse file: %.*s; file permissions problem", LIT(fi->name)); - break; - case ParseFile_NotFound: - error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi->name), LIT(fi->fullpath)); - break; - case ParseFile_InvalidToken: - error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi->name)); - break; - case ParseFile_EmptyFile: - error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name)); - break; + return err; } - - return err; } - -skip: if (parse_file(p, file)) { gb_mutex_lock(&p->file_add_mutex); defer (gb_mutex_unlock(&p->file_add_mutex)); @@ -4781,66 +4792,26 @@ skip: p->total_line_count += file->tokenizer.line_count; p->total_token_count += file->tokens.count; } + return ParseFile_None; } -struct ParserWorkerThreadData { - Parser *parser; - ParseFileError err; - - gbSemaphore resume_work; - gbMutex lock; - - bool error_available; - bool is_working; - bool should_exit; -}; - - -GB_THREAD_PROC(parse_worker_file_proc) { - GB_ASSERT(thread != nullptr); - - ParserWorkerThreadData* data = cast(ParserWorkerThreadData*) thread->user_data; - - for(;;) { - gb_semaphore_wait(&data->resume_work); - - gb_mutex_lock(&data->lock); - if (data->should_exit) { - gb_mutex_unlock(&data->lock); - return isize(0); - } - - Parser *p = data->parser; - isize index = thread->user_index; - gb_mutex_lock(&p->file_add_mutex); - auto file_to_process = p->files_to_process[index]; - gb_mutex_unlock(&p->file_add_mutex); - data->err = process_imported_file(p, file_to_process); - - data->error_available = true; - data->is_working = false; - gb_mutex_unlock(&data->lock); - } - - //GB_PANIC("A worker thread should not be able to reach the end!!!"); -} ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); - // char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char const *)&init_filename[0]); - // String init_fullpath = string_trim_whitespace(make_string_c(fullpath_str)); + isize thread_count = gb_max(build_context.thread_count, 1); + thread_pool_init(&parser_thread_pool, heap_allocator(), thread_count, "ParserWork"); + String init_fullpath = path_to_full_path(heap_allocator(), init_filename); if (!path_is_directory(init_fullpath)) { String const ext = str_lit(".odin"); if (!string_ends_with(init_fullpath, ext)) { - gb_printf_err("Expected either a directory or a .odin file, got '%.*s'\n", LIT(init_filename)); + error_line("Expected either a directory or a .odin file, got '%.*s'\n", LIT(init_filename)); return ParseFile_WrongExtension; } } - TokenPos init_pos = {}; if (!build_context.generate_docs) { String s = get_fullpath_core(heap_allocator(), str_lit("runtime")); @@ -4850,131 +4821,17 @@ ParseFileError parse_packages(Parser *p, String init_filename) { try_add_import_path(p, init_fullpath, init_fullpath, init_pos, Package_Init); p->init_fullpath = init_fullpath; -#if 1 - isize thread_count = gb_max(build_context.thread_count, 1); - if (thread_count > 1) { - isize volatile curr_import_index = 0; - -#if 0 - //NOTE(thebirk): Leaving this piece of code behind if it turns out we need it, yes I know git exists - - isize initial_file_count = p->files_to_process.count; - // NOTE(bill): Make sure that these are in parsed in this order - for (isize i = 0; i < initial_file_count; i++) { - ParseFileError err = process_imported_file(p, p->files_to_process[i]); - if (err != ParseFile_None) { - return err; - } - curr_import_index++; - } -#endif - - auto worker_threads_data = array_make(heap_allocator(), thread_count); - defer (array_free(&worker_threads_data)); - - for_array(i, worker_threads_data) { - ParserWorkerThreadData *data = &worker_threads_data[i]; - gb_mutex_init(&data->lock); - gb_semaphore_init(&data->resume_work); - data->parser = p; - data->err = ParseFile_None; - data->should_exit = false; - data->error_available = false; - data->is_working = false; - } - defer(for_array(i, worker_threads_data) { - ParserWorkerThreadData *data = &worker_threads_data[i]; - gb_mutex_destroy(&data->lock); - gb_semaphore_destroy(&data->resume_work); - }); - - auto worker_threads = array_make(heap_allocator(), thread_count); - defer (array_free(&worker_threads)); - - for_array(i, worker_threads) { - gbThread *t = &worker_threads[i]; - gb_thread_init(t); - //NOTE(thebirk): This crashes on linux. In addition to that the method used on windows does - // not get picked up by a lot of tools look into using SetThreadDescription - // when running on new enough windows 10 builds - //char buffer[32]; - //gb_snprintf(buffer, 32, "Parser Worker #%ll", i); - //gb_thread_set_name(t, buffer); - gb_thread_start(t, parse_worker_file_proc, &worker_threads_data[i]); - } - defer(for_array(i, worker_threads) { - gb_thread_destroy(&worker_threads[i]); - }); - - auto errors = array_make(heap_allocator(), 0, 16); - - for (;;) { - int num_alive = 0; - - for_array(i, worker_threads) { - gbThread *t = &worker_threads[i]; - ParserWorkerThreadData *data = &worker_threads_data[i]; - - if (!data->is_working && gb_mutex_try_lock(&data->lock)) { - if (data->error_available) { - auto curr_err = data->err; - if (curr_err != ParseFile_None) { - array_add(&errors, curr_err); - } - - data->error_available = false; - } - - if (curr_import_index < p->files_to_process.count) { - t->user_index = curr_import_index; - curr_import_index++; - num_alive += 1; + thread_pool_start(&parser_thread_pool); + thread_pool_kick_and_wait(&parser_thread_pool); - gb_semaphore_release(&data->resume_work); - data->is_working = true; - } - - gb_mutex_unlock(&data->lock); - } else { - //NOTE(thebirk): If we cant lock a thread it must be working - num_alive += 1; - } - } - - if ((num_alive == 0) && (curr_import_index >= p->files_to_process.count)) { - break; - } - - gb_yield(); - } - - //NOTE(thebirk): Signal all workers to exit - for_array(i, worker_threads_data) { - ParserWorkerThreadData* data = &worker_threads_data[i]; - data->should_exit = true; - gb_semaphore_release(&data->resume_work); - } - - if (errors.count > 0) { - return errors[errors.count-1]; - } - } else { - for_array(i, p->files_to_process) { - ParseFileError err = process_imported_file(p, p->files_to_process[i]); - if (err != ParseFile_None) { - return err; - } - } - } -#else - for_array(i, p->files_to_process) { - ImportedFile f = p->files_to_process[i]; - ParseFileError err = process_imported_file(p, f); + // NOTE(bill): Get the last error and use that + for (isize i = parser_thread_pool.threads.count-1; i >= 0; i--) { + gbThread *t = &parser_thread_pool.threads[i]; + ParseFileError err = cast(ParseFileError)t->return_value; if (err != ParseFile_None) { return err; } } -#endif return ParseFile_None; } diff --git a/src/parser.hpp b/src/parser.hpp index 26536fe56..56d9b74c7 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -141,6 +141,19 @@ struct Parser { gbMutex file_decl_mutex; }; + +gb_global ThreadPool parser_thread_pool = {}; + +struct ParserWorkerData { + Parser *parser; + ImportedFile imported_file; +}; + + + + + + enum ProcInlining { ProcInlining_none = 0, ProcInlining_inline = 1, -- cgit v1.2.3 From 97dece15d73a6c4830f5bf01d82014cb0eeff062 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 1 Sep 2019 22:18:55 +0100 Subject: Minor changes --- src/gb/gb.h | 31 ++++++++++++++++++++++++------- src/parser.cpp | 5 +---- src/parser.hpp | 2 +- 3 files changed, 26 insertions(+), 12 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/gb/gb.h b/src/gb/gb.h index 5e74ff2d8..65b8b2ff6 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -918,7 +918,10 @@ GB_DEF void gb_lfence (void); #if defined(GB_SYSTEM_WINDOWS) -typedef struct gbSemaphore { void *win32_handle; } gbSemaphore; +typedef struct gbSemaphore { + void *win32_handle; + LONG count; +} gbSemaphore; #elif defined(GB_SYSTEM_OSX) typedef struct gbSemaphore { semaphore_t osx_handle; } gbSemaphore; #elif defined(GB_SYSTEM_UNIX) @@ -930,7 +933,7 @@ typedef struct gbSemaphore { sem_t unix_handle; } gbSemaphore; GB_DEF void gb_semaphore_init (gbSemaphore *s); GB_DEF void gb_semaphore_destroy(gbSemaphore *s); GB_DEF void gb_semaphore_post (gbSemaphore *s, i32 count); -GB_DEF void gb_semaphore_release(gbSemaphore *s); // NOTE(bill): gb_semaphore_post(s, 1) +GB_DEF void gb_semaphore_release(gbSemaphore *s); GB_DEF void gb_semaphore_wait (gbSemaphore *s); @@ -4588,10 +4591,24 @@ gb_inline void gb_lfence(void) { gb_inline void gb_semaphore_release(gbSemaphore *s) { gb_semaphore_post(s, 1); } #if defined(GB_SYSTEM_WINDOWS) - gb_inline void gb_semaphore_init (gbSemaphore *s) { s->win32_handle = CreateSemaphoreA(NULL, 0, I32_MAX, NULL); } - gb_inline void gb_semaphore_destroy(gbSemaphore *s) { CloseHandle(s->win32_handle); } - gb_inline void gb_semaphore_post (gbSemaphore *s, i32 count) { ReleaseSemaphore(s->win32_handle, count, NULL); } - gb_inline void gb_semaphore_wait (gbSemaphore *s) { WaitForSingleObjectEx(s->win32_handle, INFINITE, FALSE); } + gb_inline void gb_semaphore_init(gbSemaphore *s) { + s->win32_handle = CreateSemaphoreA(NULL, 0, I32_MAX, NULL); + s->count = 0; + } + gb_inline void gb_semaphore_destroy(gbSemaphore *s) { + CloseHandle(s->win32_handle); + } + gb_inline void gb_semaphore_post(gbSemaphore *s, i32 count) { + _InterlockedIncrement(&s->count); + if (ReleaseSemaphore(s->win32_handle, count, NULL) == FALSE) { + _InterlockedDecrement(&s->count); + } + } + gb_inline void gb_semaphore_wait(gbSemaphore *s) { + if (WaitForSingleObjectEx(s->win32_handle, INFINITE, FALSE) == WAIT_OBJECT_0) { + _InterlockedDecrement(&s->count); + } + } #elif defined(GB_SYSTEM_OSX) gb_inline void gb_semaphore_init (gbSemaphore *s) { semaphore_create(mach_task_self(), &s->osx_handle, SYNC_POLICY_FIFO, 0); } @@ -8975,7 +8992,7 @@ gb_inline void gb_exit(u32 code) { exit(code); } gb_inline void gb_yield(void) { #if defined(GB_SYSTEM_WINDOWS) - Sleep(0); + YieldProcessor(); #else sched_yield(); #endif diff --git a/src/parser.cpp b/src/parser.cpp index a210eb661..e4d21e72a 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4164,7 +4164,6 @@ bool init_parser(Parser *p) { map_init(&p->package_map, heap_allocator()); array_init(&p->packages, heap_allocator()); array_init(&p->package_imports, heap_allocator()); - array_init(&p->files_to_process, heap_allocator()); gb_mutex_init(&p->file_add_mutex); gb_mutex_init(&p->file_decl_mutex); return true; @@ -4187,7 +4186,6 @@ void destroy_parser(Parser *p) { #endif array_free(&p->packages); array_free(&p->package_imports); - array_free(&p->files_to_process); string_set_destroy(&p->imported_files); map_destroy(&p->package_map); gb_mutex_destroy(&p->file_add_mutex); @@ -4225,11 +4223,10 @@ WORKER_TASK_PROC(parser_worker_proc) { void parser_add_file_to_process(Parser *p, AstPackage *pkg, FileInfo fi, TokenPos pos) { // TODO(bill): Use a better allocator - ImportedFile f = {pkg, fi, pos, p->files_to_process.count}; + ImportedFile f = {pkg, fi, pos, p->file_to_process_count++}; auto wd = gb_alloc_item(heap_allocator(), ParserWorkerData); wd->parser = p; wd->imported_file = f; - array_add(&p->files_to_process, f); thread_pool_add_task(&parser_thread_pool, parser_worker_proc, wd); } diff --git a/src/parser.hpp b/src/parser.hpp index 56d9b74c7..3da2551f1 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -134,7 +134,7 @@ struct Parser { Map package_map; // Key: String (package name) Array packages; Array package_imports; - Array files_to_process; + isize file_to_process_count; isize total_token_count; isize total_line_count; gbMutex file_add_mutex; -- cgit v1.2.3 From 1348d8a8cdefcb02be6ad9346ecbf24a4635fe0c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 2 Sep 2019 18:49:23 +0100 Subject: Improve thread pool (volatile hints, etc) --- src/gb/gb.h | 8 ++++---- src/parser.cpp | 2 +- src/thread_pool.cpp | 57 +++++++++++++++++++++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 22 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/gb/gb.h b/src/gb/gb.h index 65b8b2ff6..1b2bc5188 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -978,10 +978,10 @@ typedef struct gbThread { pthread_t posix_handle; #endif - gbThreadProc *proc; - void * user_data; - isize user_index; - isize return_value; + gbThreadProc * proc; + void * user_data; + isize user_index; + isize volatile return_value; gbSemaphore semaphore; isize stack_size; diff --git a/src/parser.cpp b/src/parser.cpp index e4d21e72a..a026e8ecd 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4822,7 +4822,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { thread_pool_kick_and_wait(&parser_thread_pool); // NOTE(bill): Get the last error and use that - for (isize i = parser_thread_pool.threads.count-1; i >= 0; i--) { + for (isize i = parser_thread_pool.thread_count-1; i >= 0; i--) { gbThread *t = &parser_thread_pool.threads[i]; ParseFileError err = cast(ParseFileError)t->return_value; if (err != ParseFile_None) { diff --git a/src/thread_pool.cpp b/src/thread_pool.cpp index 83178ea47..bbe9ccac6 100644 --- a/src/thread_pool.cpp +++ b/src/thread_pool.cpp @@ -16,8 +16,14 @@ struct ThreadPool { gbAtomic32 processing_work_count; bool is_running; - Array tasks; - Array threads; + gbAllocator allocator; + + WorkerTask *tasks; + isize volatile task_count; + isize volatile task_capacity; + + gbThread *threads; + isize thread_count; char worker_prefix[10]; i32 worker_prefix_len; @@ -33,8 +39,12 @@ void thread_pool_kick_and_wait(ThreadPool *pool); GB_THREAD_PROC(worker_thread_internal); void thread_pool_init(ThreadPool *pool, gbAllocator const &a, isize thread_count, char const *worker_prefix) { - pool->tasks = array_make(a, 0, 1024); - pool->threads = array_make(a, thread_count); + pool->allocator = a; + pool->task_count = 0; + pool->task_capacity = 1024; + pool->tasks = gb_alloc_array(a, WorkerTask, pool->task_capacity); + pool->threads = gb_alloc_array(a, gbThread, thread_count); + pool->thread_count = thread_count; gb_mutex_init(&pool->task_mutex); gb_mutex_init(&pool->mutex); gb_semaphore_init(&pool->semaphore); @@ -48,7 +58,7 @@ void thread_pool_init(ThreadPool *pool, gbAllocator const &a, isize thread_count pool->worker_prefix_len = worker_prefix_len; } - for_array(i, pool->threads) { + for (isize i = 0; i < pool->thread_count; i++) { gbThread *t = &pool->threads[i]; gb_thread_init(t); t->user_index = i; @@ -63,7 +73,7 @@ void thread_pool_init(ThreadPool *pool, gbAllocator const &a, isize thread_count } void thread_pool_start(ThreadPool *pool) { - for_array(i, pool->threads) { + for (isize i = 0; i < pool->thread_count; i++) { gbThread *t = &pool->threads[i]; gb_thread_start(t, worker_thread_internal, pool); } @@ -72,11 +82,11 @@ void thread_pool_start(ThreadPool *pool) { void thread_pool_join(ThreadPool *pool) { pool->is_running = false; - for_array(i, pool->threads) { + for (isize i = 0; i < pool->thread_count; i++) { gb_semaphore_release(&pool->semaphore); } - for_array(i, pool->threads) { + for (isize i = 0; i < pool->thread_count; i++) { gbThread *t = &pool->threads[i]; gb_thread_join(t); } @@ -89,18 +99,30 @@ void thread_pool_destroy(ThreadPool *pool) { gb_semaphore_destroy(&pool->semaphore); gb_mutex_destroy(&pool->mutex); gb_mutex_destroy(&pool->task_mutex); - array_free(&pool->threads); - array_free(&pool->tasks); + gb_free(pool->allocator, pool->threads); + pool->thread_count = 0; + gb_free(pool->allocator, pool->tasks); + pool->task_count = 0; + pool->task_capacity = 0; + } void thread_pool_add_task(ThreadPool *pool, WorkerTaskProc *proc, void *data) { gb_mutex_lock(&pool->task_mutex); + if (pool->task_count == pool->task_capacity) { + isize new_cap = 2*pool->task_capacity + 8; + WorkerTask *new_tasks = gb_alloc_array(pool->allocator, WorkerTask, new_cap); + gb_memmove(new_tasks, pool->tasks, pool->task_count*gb_size_of(WorkerTask)); + pool->tasks = new_tasks; + pool->task_capacity = new_cap; + } WorkerTask task = {}; task.do_work = proc; task.data = data; - array_add(&pool->tasks, task); + + pool->tasks[pool->task_count++] = task; gb_mutex_unlock(&pool->task_mutex); @@ -108,19 +130,20 @@ void thread_pool_add_task(ThreadPool *pool, WorkerTaskProc *proc, void *data) { } void thread_pool_kick(ThreadPool *pool) { - if (pool->tasks.count > 0) { - isize count = gb_min(pool->tasks.count, pool->threads.count); + gb_mutex_lock(&pool->task_mutex); + if (pool->task_count > 0) { + isize count = gb_max(pool->task_count, pool->thread_count); for (isize i = 0; i < count; i++) { gb_semaphore_post(&pool->semaphore, 1); } } - + gb_mutex_unlock(&pool->task_mutex); } void thread_pool_kick_and_wait(ThreadPool *pool) { thread_pool_kick(pool); isize return_value = 0; - while (pool->tasks.count > 0 || gb_atomic32_load(&pool->processing_work_count) != 0) { + while (pool->task_count > 0 || gb_atomic32_load(&pool->processing_work_count) != 0) { gb_yield(); } @@ -138,9 +161,9 @@ GB_THREAD_PROC(worker_thread_internal) { bool got_task = false; if (gb_mutex_try_lock(&pool->task_mutex)) { - if (pool->tasks.count > 0) { + if (pool->task_count > 0) { gb_atomic32_fetch_add(&pool->processing_work_count, +1); - task = array_pop(&pool->tasks); + task = pool->tasks[--pool->task_count]; got_task = true; } gb_mutex_unlock(&pool->task_mutex); -- cgit v1.2.3 From 772c8779fa4ed38fcc53c1a7a5c4c93e8a13f05a Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 3 Sep 2019 22:11:21 +0100 Subject: Clean up thread pool code --- src/gb/gb.h | 15 ++----- src/parser.cpp | 11 ++--- src/thread_pool.cpp | 114 ++++++++++++++++++++++++++-------------------------- 3 files changed, 65 insertions(+), 75 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/gb/gb.h b/src/gb/gb.h index 1b2bc5188..60303729f 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -918,10 +918,7 @@ GB_DEF void gb_lfence (void); #if defined(GB_SYSTEM_WINDOWS) -typedef struct gbSemaphore { - void *win32_handle; - LONG count; -} gbSemaphore; +typedef struct gbSemaphore { void *win32_handle;} gbSemaphore; #elif defined(GB_SYSTEM_OSX) typedef struct gbSemaphore { semaphore_t osx_handle; } gbSemaphore; #elif defined(GB_SYSTEM_UNIX) @@ -4593,21 +4590,15 @@ gb_inline void gb_semaphore_release(gbSemaphore *s) { gb_semaphore_post(s, 1); } #if defined(GB_SYSTEM_WINDOWS) gb_inline void gb_semaphore_init(gbSemaphore *s) { s->win32_handle = CreateSemaphoreA(NULL, 0, I32_MAX, NULL); - s->count = 0; } gb_inline void gb_semaphore_destroy(gbSemaphore *s) { CloseHandle(s->win32_handle); } gb_inline void gb_semaphore_post(gbSemaphore *s, i32 count) { - _InterlockedIncrement(&s->count); - if (ReleaseSemaphore(s->win32_handle, count, NULL) == FALSE) { - _InterlockedDecrement(&s->count); - } + ReleaseSemaphore(s->win32_handle, count, NULL); } gb_inline void gb_semaphore_wait(gbSemaphore *s) { - if (WaitForSingleObjectEx(s->win32_handle, INFINITE, FALSE) == WAIT_OBJECT_0) { - _InterlockedDecrement(&s->count); - } + WaitForSingleObjectEx(s->win32_handle, INFINITE, FALSE); } #elif defined(GB_SYSTEM_OSX) diff --git a/src/parser.cpp b/src/parser.cpp index a026e8ecd..b1d9a457f 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4798,7 +4798,8 @@ ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); isize thread_count = gb_max(build_context.thread_count, 1); - thread_pool_init(&parser_thread_pool, heap_allocator(), thread_count, "ParserWork"); + isize worker_count = thread_count-1; // NOTE(bill): The main thread will also be used for work + thread_pool_init(&parser_thread_pool, heap_allocator(), worker_count, "ParserWork"); String init_fullpath = path_to_full_path(heap_allocator(), init_filename); if (!path_is_directory(init_fullpath)) { @@ -4819,12 +4820,12 @@ ParseFileError parse_packages(Parser *p, String init_filename) { p->init_fullpath = init_fullpath; thread_pool_start(&parser_thread_pool); - thread_pool_kick_and_wait(&parser_thread_pool); + thread_pool_wait_to_process(&parser_thread_pool); // NOTE(bill): Get the last error and use that - for (isize i = parser_thread_pool.thread_count-1; i >= 0; i--) { - gbThread *t = &parser_thread_pool.threads[i]; - ParseFileError err = cast(ParseFileError)t->return_value; + for (isize i = parser_thread_pool.task_tail-1; i >= 0; i--) { + WorkerTask *task = &parser_thread_pool.tasks[i]; + ParseFileError err = cast(ParseFileError)task->result; if (err != ParseFile_None) { return err; } diff --git a/src/thread_pool.cpp b/src/thread_pool.cpp index 8a32b7aca..2467ba609 100644 --- a/src/thread_pool.cpp +++ b/src/thread_pool.cpp @@ -6,20 +6,21 @@ typedef WORKER_TASK_PROC(WorkerTaskProc); struct WorkerTask { WorkerTaskProc *do_work; void *data; + isize result; }; struct ThreadPool { - gbMutex task_mutex; gbMutex mutex; - gbSemaphore semaphore; + gbSemaphore sem_available; gbAtomic32 processing_work_count; bool is_running; gbAllocator allocator; WorkerTask *tasks; - isize volatile task_count; + isize volatile task_head; + isize volatile task_tail; isize volatile task_capacity; gbThread *threads; @@ -40,14 +41,14 @@ GB_THREAD_PROC(worker_thread_internal); void thread_pool_init(ThreadPool *pool, gbAllocator const &a, isize thread_count, char const *worker_prefix) { pool->allocator = a; - pool->task_count = 0; + pool->task_head = 0; + pool->task_tail = 0; pool->task_capacity = 1024; pool->tasks = gb_alloc_array(a, WorkerTask, pool->task_capacity); - pool->threads = gb_alloc_array(a, gbThread, thread_count); - pool->thread_count = thread_count; - gb_mutex_init(&pool->task_mutex); + pool->thread_count = gb_max(thread_count, 0); + pool->threads = gb_alloc_array(a, gbThread, pool->thread_count); gb_mutex_init(&pool->mutex); - gb_semaphore_init(&pool->semaphore); + gb_semaphore_init(&pool->sem_available); pool->is_running = true; pool->worker_prefix_len = 0; @@ -63,6 +64,7 @@ void thread_pool_init(ThreadPool *pool, gbAllocator const &a, isize thread_count gb_thread_init(t); t->user_index = i; #if 0 + // TODO(bill): Fix this on Linux as it causes a seg-fault if (pool->worker_prefix_len > 0) { char worker_name[16] = {}; gb_snprintf(worker_name, gb_size_of(worker_name), "%.*s%u", pool->worker_prefix_len, pool->worker_prefix, cast(u16)i); @@ -82,9 +84,9 @@ void thread_pool_start(ThreadPool *pool) { void thread_pool_join(ThreadPool *pool) { pool->is_running = false; - for (isize i = 0; i < pool->thread_count; i++) { - gb_semaphore_release(&pool->semaphore); - } + gb_semaphore_post(&pool->sem_available, cast(i32)pool->thread_count); + + gb_yield(); for (isize i = 0; i < pool->thread_count; i++) { gbThread *t = &pool->threads[i]; @@ -96,25 +98,24 @@ void thread_pool_join(ThreadPool *pool) { void thread_pool_destroy(ThreadPool *pool) { thread_pool_join(pool); - gb_semaphore_destroy(&pool->semaphore); + gb_semaphore_destroy(&pool->sem_available); gb_mutex_destroy(&pool->mutex); - gb_mutex_destroy(&pool->task_mutex); gb_free(pool->allocator, pool->threads); pool->thread_count = 0; gb_free(pool->allocator, pool->tasks); - pool->task_count = 0; + pool->task_head = 0; + pool->task_tail = 0; pool->task_capacity = 0; - } void thread_pool_add_task(ThreadPool *pool, WorkerTaskProc *proc, void *data) { - gb_mutex_lock(&pool->task_mutex); + gb_mutex_lock(&pool->mutex); - if (pool->task_count == pool->task_capacity) { + if (pool->task_tail == pool->task_capacity) { isize new_cap = 2*pool->task_capacity + 8; WorkerTask *new_tasks = gb_alloc_array(pool->allocator, WorkerTask, new_cap); - gb_memmove(new_tasks, pool->tasks, pool->task_count*gb_size_of(WorkerTask)); + gb_memmove(new_tasks, pool->tasks, (pool->task_tail)*gb_size_of(WorkerTask)); pool->tasks = new_tasks; pool->task_capacity = new_cap; } @@ -122,35 +123,42 @@ void thread_pool_add_task(ThreadPool *pool, WorkerTaskProc *proc, void *data) { task.do_work = proc; task.data = data; - pool->tasks[pool->task_count++] = task; - - gb_semaphore_post(&pool->semaphore, 1); - gb_mutex_unlock(&pool->task_mutex); + pool->tasks[pool->task_tail++] = task; + gb_semaphore_post(&pool->sem_available, 1); + gb_mutex_unlock(&pool->mutex); } -void thread_pool_kick(ThreadPool *pool) { - gb_mutex_lock(&pool->task_mutex); - if (pool->task_count > 0) { - isize count = gb_min(pool->task_count, pool->thread_count); - for (isize i = 0; i < count; i++) { - gb_semaphore_post(&pool->semaphore, 1); +bool thread_pool_try_and_pop_task(ThreadPool *pool, WorkerTask *task) { + bool got_task = false; + if (gb_mutex_try_lock(&pool->mutex)) { + if (pool->task_tail > pool->task_head) { + gb_atomic32_fetch_add(&pool->processing_work_count, +1); + *task = pool->tasks[pool->task_head++]; + got_task = true; } + gb_mutex_unlock(&pool->mutex); } - gb_mutex_unlock(&pool->task_mutex); + return got_task; +} +void thread_pool_do_work(ThreadPool *pool, WorkerTask *task) { + task->result = task->do_work(task->data); + gb_atomic32_fetch_add(&pool->processing_work_count, -1); } -void thread_pool_kick_and_wait(ThreadPool *pool) { - thread_pool_kick(pool); - - isize return_value = 0; - while (pool->task_count > 0 || gb_atomic32_load(&pool->processing_work_count) != 0) { - - if (pool->task_count > 0 && gb_atomic32_load(&pool->processing_work_count) == 0) { - gb_mutex_lock(&pool->task_mutex); - for (isize i = 0; i < pool->task_count; i++) { - gb_semaphore_post(&pool->semaphore, 1); - } - gb_mutex_unlock(&pool->task_mutex); + +void thread_pool_wait_to_process(ThreadPool *pool) { + while (pool->task_tail > pool->task_head || gb_atomic32_load(&pool->processing_work_count) != 0) { + WorkerTask task = {}; + if (thread_pool_try_and_pop_task(pool, &task)) { + thread_pool_do_work(pool, &task); + } + + // Safety-kick + if (pool->task_tail > pool->task_head && gb_atomic32_load(&pool->processing_work_count) == 0) { + gb_mutex_lock(&pool->mutex); + gb_semaphore_post(&pool->sem_available, cast(i32)(pool->task_tail-pool->task_head)); + gb_mutex_unlock(&pool->mutex); } + gb_yield(); } @@ -160,27 +168,17 @@ void thread_pool_kick_and_wait(ThreadPool *pool) { GB_THREAD_PROC(worker_thread_internal) { ThreadPool *pool = cast(ThreadPool *)thread->user_data; - thread->return_value = 0; while (pool->is_running) { - gb_semaphore_wait(&pool->semaphore); + gb_semaphore_wait(&pool->sem_available); WorkerTask task = {}; - bool got_task = false; - - if (gb_mutex_try_lock(&pool->task_mutex)) { - if (pool->task_count > 0) { - gb_atomic32_fetch_add(&pool->processing_work_count, +1); - task = pool->tasks[--pool->task_count]; - got_task = true; - } - gb_mutex_unlock(&pool->task_mutex); - } - - if (got_task) { - thread->return_value = task.do_work(task.data); - gb_atomic32_fetch_add(&pool->processing_work_count, -1); + if (thread_pool_try_and_pop_task(pool, &task)) { + thread_pool_do_work(pool, &task); } } - return thread->return_value; + // Cascade + gb_semaphore_release(&pool->sem_available); + + return 0; } -- cgit v1.2.3 From d54255505a1b7b2a460ce7f4a18ea30a92286aa1 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 4 Sep 2019 18:10:02 +0100 Subject: Fix Compiler does not complain about missing semicolon #433 --- src/parser.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index b1d9a457f..123e1fcee 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1370,10 +1370,6 @@ void expect_semicolon(AstFile *f, Ast *s) { return; } - switch (f->curr_token.kind) { - case Token_EOF: - return; - } if (s != nullptr) { if (prev_token.pos.line != f->curr_token.pos.line) { @@ -1386,12 +1382,21 @@ void expect_semicolon(AstFile *f, Ast *s) { case Token_CloseParen: case Token_else: return; + case Token_EOF: + if (is_semicolon_optional_for_node(f, s)) { + return; + } + break; } } String node_string = ast_strings[s->kind]; syntax_error(prev_token, "Expected ';' after %.*s, got %.*s", - LIT(node_string), LIT(token_strings[prev_token.kind])); + LIT(node_string), LIT(token_strings[f->curr_token.kind])); } else { + switch (f->curr_token.kind) { + case Token_EOF: + return; + } syntax_error(prev_token, "Expected ';'"); } fix_advance_to_next_stmt(f); -- cgit v1.2.3 From 4afc78efc682152a9b096781a1598fd1841ca938 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 8 Sep 2019 12:12:41 +0100 Subject: Add `where` clauses to `struct` and `union` --- core/odin/ast/ast.odin | 18 +++++++----- core/odin/parser/parser.odin | 42 +++++++++++++++++++++------ src/check_decl.cpp | 2 +- src/check_expr.cpp | 22 ++++++--------- src/check_type.cpp | 5 +++- src/checker.cpp | 2 +- src/parser.cpp | 67 +++++++++++++++++++++++++++++++++----------- src/parser.hpp | 28 ++++++++++-------- 8 files changed, 125 insertions(+), 61 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/ast/ast.odin b/core/odin/ast/ast.odin index f1ea79584..baad5f802 100644 --- a/core/odin/ast/ast.odin +++ b/core/odin/ast/ast.odin @@ -568,13 +568,15 @@ Dynamic_Array_Type :: struct { Struct_Type :: struct { using node: Expr, - tok_pos: token.Pos, - poly_params: ^Field_List, - align: ^Expr, - is_packed: bool, - is_raw_union: bool, - fields: ^Field_List, - name_count: int, + tok_pos: token.Pos, + poly_params: ^Field_List, + align: ^Expr, + fields: ^Field_List, + name_count: int, + where_token: token.Token, + where_clauses: []^Expr, + is_packed: bool, + is_raw_union: bool, } Union_Type :: struct { @@ -583,6 +585,8 @@ Union_Type :: struct { poly_params: ^Field_List, align: ^Expr, variants: []^Expr, + where_token: token.Token, + where_clauses: []^Expr, } Enum_Type :: struct { diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index a0d4d639e..f652e2c02 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -2174,17 +2174,29 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { error(p, tok.pos, "'#raw_union' cannot also be '#packed"); } + where_token: token.Token; + where_clauses: []^ast.Expr; + if (p.curr_tok.kind == token.Where) { + where_token = expect_token(p, token.Where); + prev_level := p.expr_level; + p.expr_level = -1; + where_clauses = parse_rhs_expr_list(p); + p.expr_level = prev_level; + } + expect_token(p, token.Open_Brace); fields, name_count = parse_field_list(p, token.Close_Brace, ast.Field_Flags_Struct); close := expect_token(p, token.Close_Brace); st := ast.new(ast.Struct_Type, tok.pos, end_pos(close)); - st.poly_params = poly_params; - st.align = align; - st.is_packed = is_packed; - st.is_raw_union = is_raw_union; - st.fields = fields; - st.name_count = name_count; + st.poly_params = poly_params; + st.align = align; + st.is_packed = is_packed; + st.is_raw_union = is_raw_union; + st.fields = fields; + st.name_count = name_count; + st.where_token = where_token; + st.where_clauses = where_clauses; return st; case token.Union: @@ -2217,6 +2229,16 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { } p.expr_level = prev_level; + where_token: token.Token; + where_clauses: []^ast.Expr; + if (p.curr_tok.kind == token.Where) { + where_token = expect_token(p, token.Where); + prev_level := p.expr_level; + p.expr_level = -1; + where_clauses = parse_rhs_expr_list(p); + p.expr_level = prev_level; + } + variants: [dynamic]^ast.Expr; expect_token_after(p, token.Open_Brace, "union"); @@ -2234,9 +2256,11 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { close := expect_token(p, token.Close_Brace); ut := ast.new(ast.Union_Type, tok.pos, end_pos(close)); - ut.poly_params = poly_params; - ut.variants = variants[:]; - ut.align = align; + ut.poly_params = poly_params; + ut.variants = variants[:]; + ut.align = align; + ut.where_token = where_token; + ut.where_clauses = where_clauses; return ut; diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 388b37864..173ba4a90 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -1176,7 +1176,7 @@ void check_proc_body(CheckerContext *ctx_, Token token, DeclInfo *decl, Type *ty } - bool where_clause_ok = evaluate_where_clauses(ctx, decl, true); + bool where_clause_ok = evaluate_where_clauses(ctx, decl->scope, &decl->proc_lit->ProcLit.where_clauses, true); if (!where_clause_ok) { // NOTE(bill, 2019-08-31): Don't check the body as the where clauses failed return; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index ca1aed970..8a5ed9c1c 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -5471,15 +5471,10 @@ Entity **populate_proc_parameter_list(CheckerContext *c, Type *proc_type, isize } -bool evaluate_where_clauses(CheckerContext *ctx, DeclInfo *decl, bool print_err) { - Ast *proc_lit = decl->proc_lit; - GB_ASSERT(proc_lit != nullptr); - GB_ASSERT(proc_lit->kind == Ast_ProcLit); - - if (proc_lit->ProcLit.where_token.kind != Token_Invalid) { - auto &clauses = proc_lit->ProcLit.where_clauses; - for_array(i, clauses) { - Ast *clause = clauses[i]; +bool evaluate_where_clauses(CheckerContext *ctx, Scope *scope, Array *clauses, bool print_err) { + if (clauses != nullptr) { + for_array(i, *clauses) { + Ast *clause = (*clauses)[i]; Operand o = {}; check_expr(ctx, &o, clause); if (o.mode != Addressing_Constant) { @@ -5494,10 +5489,10 @@ bool evaluate_where_clauses(CheckerContext *ctx, DeclInfo *decl, bool print_err) error(clause, "'where' clause evaluated to false:\n\t%s", str); gb_string_free(str); - if (decl->scope != nullptr) { + if (scope != nullptr) { isize print_count = 0; - for_array(j, decl->scope->elements.entries) { - Entity *e = decl->scope->elements.entries[j].value; + for_array(j, scope->elements.entries) { + Entity *e = scope->elements.entries[j].value; switch (e->kind) { case Entity_TypeName: { if (print_count == 0) error_line("\n\tWith the following definitions:\n"); @@ -5790,7 +5785,8 @@ CallArgumentData check_call_arguments(CheckerContext *c, Operand *operand, Type ctx.curr_proc_decl = decl; ctx.curr_proc_sig = e->type; - if (!evaluate_where_clauses(&ctx, decl, false)) { + GB_ASSERT(decl->proc_lit->kind == Ast_ProcLit); + if (!evaluate_where_clauses(&ctx, decl->scope, &decl->proc_lit->ProcLit.where_clauses, false)) { continue; } } diff --git a/src/check_type.cpp b/src/check_type.cpp index 2fa8939c4..e1602365a 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -504,8 +504,8 @@ void check_struct_type(CheckerContext *ctx, Type *struct_type, Ast *node, Array< struct_type->Struct.polymorphic_params = polymorphic_params; struct_type->Struct.is_poly_specialized = is_poly_specialized; - if (!is_polymorphic) { + bool where_clause_ok = evaluate_where_clauses(ctx, ctx->scope, &st->where_clauses, true); check_struct_fields(ctx, node, &struct_type->Struct.fields, &struct_type->Struct.tags, st->fields, min_field_count, struct_type, context); } @@ -688,6 +688,9 @@ void check_union_type(CheckerContext *ctx, Type *union_type, Ast *node, ArrayUnion.is_polymorphic = is_polymorphic; union_type->Union.is_poly_specialized = is_poly_specialized; + bool where_clause_ok = evaluate_where_clauses(ctx, ctx->scope, &ut->where_clauses, true); + + for_array(i, ut->variants) { Ast *node = ut->variants[i]; Type *t = check_type_expr(ctx, node, nullptr); diff --git a/src/checker.cpp b/src/checker.cpp index 8fe71b63c..994e0776d 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -3157,7 +3157,7 @@ void check_add_import_decl(CheckerContext *ctx, Ast *decl) { Entity *e = scope->elements.entries[elem_index].value; if (e->scope == parent_scope) continue; - if (is_entity_exported(e)) { + if (is_entity_exported(e, true)) { Entity *found = scope_lookup_current(parent_scope, name); if (found != nullptr) { // NOTE(bill): diff --git a/src/parser.cpp b/src/parser.cpp index 123e1fcee..204ff3984 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -349,10 +349,12 @@ Ast *clone_ast(Ast *node) { n->StructType.fields = clone_ast_array(n->StructType.fields); n->StructType.polymorphic_params = clone_ast(n->StructType.polymorphic_params); n->StructType.align = clone_ast(n->StructType.align); + n->StructType.where_clauses = clone_ast_array(n->StructType.where_clauses); break; case Ast_UnionType: n->UnionType.variants = clone_ast_array(n->UnionType.variants); n->UnionType.polymorphic_params = clone_ast(n->UnionType.polymorphic_params); + n->UnionType.where_clauses = clone_ast_array(n->UnionType.where_clauses); break; case Ast_EnumType: n->EnumType.base_type = clone_ast(n->EnumType.base_type); @@ -921,7 +923,8 @@ Ast *ast_dynamic_array_type(AstFile *f, Token token, Ast *elem) { Ast *ast_struct_type(AstFile *f, Token token, Array fields, isize field_count, Ast *polymorphic_params, bool is_packed, bool is_raw_union, - Ast *align) { + Ast *align, + Token where_token, Array const &where_clauses) { Ast *result = alloc_ast_node(f, Ast_StructType); result->StructType.token = token; result->StructType.fields = fields; @@ -930,17 +933,22 @@ Ast *ast_struct_type(AstFile *f, Token token, Array fields, isize field_c result->StructType.is_packed = is_packed; result->StructType.is_raw_union = is_raw_union; result->StructType.align = align; + result->StructType.where_token = where_token; + result->StructType.where_clauses = where_clauses; return result; } -Ast *ast_union_type(AstFile *f, Token token, Array variants, Ast *polymorphic_params, Ast *align, bool no_nil) { +Ast *ast_union_type(AstFile *f, Token token, Array variants, Ast *polymorphic_params, Ast *align, bool no_nil, + Token where_token, Array const &where_clauses) { Ast *result = alloc_ast_node(f, Ast_UnionType); result->UnionType.token = token; result->UnionType.variants = variants; result->UnionType.polymorphic_params = polymorphic_params; result->UnionType.align = align; result->UnionType.no_nil = no_nil; + result->UnionType.where_token = where_token; + result->UnionType.where_clauses = where_clauses; return result; } @@ -2020,6 +2028,18 @@ Ast *parse_operand(AstFile *f, bool lhs) { syntax_error(token, "'#raw_union' cannot also be '#packed'"); } + Token where_token = {}; + Array where_clauses = {}; + + if (f->curr_token.kind == Token_where) { + where_token = expect_token(f, Token_where); + isize prev_level = f->expr_level; + f->expr_level = -1; + where_clauses = parse_rhs_expr_list(f); + f->expr_level = prev_level; + } + + Token open = expect_token_after(f, Token_OpenBrace, "struct"); isize name_count = 0; @@ -2032,7 +2052,7 @@ Ast *parse_operand(AstFile *f, bool lhs) { decls = fields->FieldList.list; } - return ast_struct_type(f, token, decls, name_count, polymorphic_params, is_packed, is_raw_union, align); + return ast_struct_type(f, token, decls, name_count, polymorphic_params, is_packed, is_raw_union, align, where_token, where_clauses); } break; case Token_union: { @@ -2073,6 +2093,18 @@ Ast *parse_operand(AstFile *f, bool lhs) { } } + Token where_token = {}; + Array where_clauses = {}; + + if (f->curr_token.kind == Token_where) { + where_token = expect_token(f, Token_where); + isize prev_level = f->expr_level; + f->expr_level = -1; + where_clauses = parse_rhs_expr_list(f); + f->expr_level = prev_level; + } + + Token open = expect_token_after(f, Token_OpenBrace, "union"); while (f->curr_token.kind != Token_CloseBrace && @@ -2088,7 +2120,7 @@ Ast *parse_operand(AstFile *f, bool lhs) { Token close = expect_token(f, Token_CloseBrace); - return ast_union_type(f, token, variants, polymorphic_params, align, no_nil); + return ast_union_type(f, token, variants, polymorphic_params, align, no_nil, where_token, where_clauses); } break; case Token_enum: { @@ -4424,19 +4456,6 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, } - if (is_package_name_reserved(file_str)) { - *path = file_str; - return true; - } - - if (file_mutex) gb_mutex_lock(file_mutex); - defer (if (file_mutex) gb_mutex_unlock(file_mutex)); - - - if (node->kind == Ast_ForeignImportDecl) { - node->ForeignImportDecl.collection_name = collection_name; - } - if (collection_name.len > 0) { if (collection_name == "system") { if (node->kind != Ast_ForeignImportDecl) { @@ -4467,6 +4486,20 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, #endif } + + if (is_package_name_reserved(file_str)) { + *path = file_str; + return true; + } + + if (file_mutex) gb_mutex_lock(file_mutex); + defer (if (file_mutex) gb_mutex_unlock(file_mutex)); + + + if (node->kind == Ast_ForeignImportDecl) { + node->ForeignImportDecl.collection_name = collection_name; + } + if (has_windows_drive) { *path = file_str; } else { diff --git a/src/parser.hpp b/src/parser.hpp index 3da2551f1..419cf9da3 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -491,20 +491,24 @@ AST_KIND(_TypeBegin, "", bool) \ Ast *elem; \ }) \ AST_KIND(StructType, "struct type", struct { \ - Token token; \ - Array fields; \ - isize field_count; \ - Ast *polymorphic_params; \ - Ast *align; \ - bool is_packed; \ - bool is_raw_union; \ + Token token; \ + Array fields; \ + isize field_count; \ + Ast *polymorphic_params; \ + Ast *align; \ + Token where_token; \ + Array where_clauses; \ + bool is_packed; \ + bool is_raw_union; \ }) \ AST_KIND(UnionType, "union type", struct { \ - Token token; \ - Array variants; \ - Ast *polymorphic_params; \ - Ast * align; \ - bool no_nil; \ + Token token; \ + Array variants; \ + Ast *polymorphic_params; \ + Ast * align; \ + bool no_nil; \ + Token where_token; \ + Array where_clauses; \ }) \ AST_KIND(EnumType, "enum type", struct { \ Token token; \ -- cgit v1.2.3 From 6c69e8c043e7dcf9d9965c7b28c7bdae44e0537c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 6 Oct 2019 14:55:25 +0100 Subject: Make `typeid` semantics consistent across variables and constants --- src/check_decl.cpp | 54 ++++++++++++++++++++++++++++++++--------------- src/check_expr.cpp | 61 +++++++++++++++++++++++++++++++++++++++++------------ src/checker.cpp | 16 ++++++-------- src/exact_value.cpp | 18 ++++++++++++++++ src/ir.cpp | 5 +++++ src/parser.cpp | 4 ---- 6 files changed, 113 insertions(+), 45 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 166b6e715..5e8479a7f 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -41,11 +41,20 @@ Type *check_init_variable(CheckerContext *ctx, Entity *e, Operand *operand, Stri } if (operand->mode == Addressing_Type) { - gbString t = type_to_string(operand->type); - error(operand->expr, "Cannot assign a type '%s' to variable '%.*s'", t, LIT(e->token.string)); - gb_string_free(t); - e->type = operand->type; - return nullptr; + if (e->type != nullptr && is_type_typeid(e->type)) { + add_type_info_type(ctx, operand->type); + add_type_and_value(ctx->info, operand->expr, Addressing_Value, e->type, exact_value_typeid(operand->type)); + return e->type; + } else { + gbString t = type_to_string(operand->type); + defer (gb_string_free(t)); + error(operand->expr, "Cannot assign a type '%s' to variable '%.*s'", t, LIT(e->token.string)); + if (e->type == nullptr) { + error_line("\tThe type of the variable '%.*s' cannot be inferred as a type does not have a type\n", LIT(e->token.string)); + } + e->type = operand->type; + return nullptr; + } } @@ -240,7 +249,7 @@ isize total_attribute_count(DeclInfo *decl) { } -void check_type_decl(CheckerContext *ctx, Entity *e, Ast *type_expr, Type *def) { +void check_type_decl(CheckerContext *ctx, Entity *e, Ast *init_expr, Type *def) { GB_ASSERT(e->type == nullptr); DeclInfo *decl = decl_info_of_entity(e); @@ -248,9 +257,8 @@ void check_type_decl(CheckerContext *ctx, Entity *e, Ast *type_expr, Type *def) check_decl_attributes(ctx, decl->attributes, const_decl_attribute, nullptr); } - - bool is_distinct = is_type_distinct(type_expr); - Ast *te = remove_type_alias_clutter(type_expr); + bool is_distinct = is_type_distinct(init_expr); + Ast *te = remove_type_alias_clutter(init_expr); e->type = t_invalid; String name = e->token.string; Type *named = alloc_type_named(name, nullptr, e); @@ -266,7 +274,7 @@ void check_type_decl(CheckerContext *ctx, Entity *e, Ast *type_expr, Type *def) named->Named.base = base_type(bt); if (is_distinct && is_type_typeid(e->type)) { - error(type_expr, "'distinct' cannot be applied to 'typeid'"); + error(init_expr, "'distinct' cannot be applied to 'typeid'"); is_distinct = false; } if (!is_distinct) { @@ -275,6 +283,19 @@ void check_type_decl(CheckerContext *ctx, Entity *e, Ast *type_expr, Type *def) e->TypeName.is_type_alias = true; } + + if (decl->type_expr != nullptr) { + Type *t = check_type(ctx, decl->type_expr); + if (t != nullptr && !is_type_typeid(t)) { + Operand operand = {}; + operand.mode = Addressing_Type; + operand.type = e->type; + operand.expr = init_expr; + check_assignment(ctx, &operand, t, str_lit("constant declaration")); + } + } + + // using decl if (decl->is_using) { // NOTE(bill): Must be an enum declaration @@ -363,15 +384,14 @@ void check_const_decl(CheckerContext *ctx, Entity *e, Ast *type_expr, Ast *init, switch (operand.mode) { case Addressing_Type: { + if (e->type != nullptr && !is_type_typeid(e->type)) { + check_assignment(ctx, &operand, e->type, str_lit("constant declaration")); + } + e->kind = Entity_TypeName; e->type = nullptr; - DeclInfo *d = ctx->decl; - if (d->type_expr != nullptr) { - error(e->token, "A type declaration cannot have an type parameter"); - } - d->type_expr = d->init_expr; - check_type_decl(ctx, e, d->type_expr, named_type); + check_type_decl(ctx, e, ctx->decl->init_expr, named_type); return; } @@ -1070,7 +1090,7 @@ void check_entity_decl(CheckerContext *ctx, Entity *e, DeclInfo *d, Type *named_ check_const_decl(&c, e, d->type_expr, d->init_expr, named_type); break; case Entity_TypeName: { - check_type_decl(&c, e, d->type_expr, named_type); + check_type_decl(&c, e, d->init_expr, named_type); break; } case Entity_Procedure: diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 697abef5f..6a0d8221f 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -442,6 +442,10 @@ i64 check_distance_between_types(CheckerContext *c, Operand *operand, Type *type } if (operand->mode == Addressing_Type) { + if (is_type_typeid(type)) { + add_type_info_type(c, operand->type); + return 4; + } return -1; } @@ -755,7 +759,12 @@ void check_assignment(CheckerContext *c, Operand *operand, Type *type, String co return; } - if (!check_is_assignable_to(c, operand, type)) { + if (check_is_assignable_to(c, operand, type)) { + if (operand->mode == Addressing_Type && is_type_typeid(type)) { + add_type_info_type(c, operand->type); + add_type_and_value(c->info, operand->expr, Addressing_Value, type, exact_value_typeid(operand->type)); + } + } else { gbString expr_str = expr_to_string(operand->expr); gbString op_type_str = type_to_string(operand->type); gbString type_str = type_to_string(type); @@ -1734,6 +1743,23 @@ void check_comparison(CheckerContext *c, Operand *x, Operand *y, TokenKind op) { return; } + if (x->mode == Addressing_Type && is_type_typeid(y->type)) { + add_type_info_type(c, x->type); + add_type_and_value(c->info, x->expr, Addressing_Value, y->type, exact_value_typeid(x->type)); + + x->mode = Addressing_Value; + x->type = t_untyped_bool; + return; + } else if (is_type_typeid(x->type) && y->mode == Addressing_Type) { + add_type_info_type(c, y->type); + add_type_and_value(c->info, y->expr, Addressing_Value, x->type, exact_value_typeid(y->type)); + + x->mode = Addressing_Value; + x->type = t_untyped_bool; + return; + } + + gbString err_str = nullptr; defer (if (err_str != nullptr) { @@ -2324,8 +2350,16 @@ void check_binary_expr(CheckerContext *c, Operand *x, Ast *node, Type *type_hint // If only one is a type, this is an error if (xt ^ yt) { GB_ASSERT(xt != yt); - if (xt) error_operand_not_expression(x); - if (yt) error_operand_not_expression(y); + if (xt) { + if (!is_type_typeid(y->type)) { + error_operand_not_expression(x); + } + } + if (yt) { + if (!is_type_typeid(x->type)) { + error_operand_not_expression(y); + } + } } break; @@ -5254,6 +5288,11 @@ CALL_ARGUMENT_CHECKER(check_call_arguments_internal) { } } score += s; + + if (o.mode == Addressing_Type && is_type_typeid(e->type)) { + add_type_info_type(c, o.type); + add_type_and_value(c->info, o.expr, Addressing_Value, e->type, exact_value_typeid(o.type)); + } } if (variadic) { @@ -5496,6 +5535,11 @@ CALL_ARGUMENT_CHECKER(check_named_call_arguments) { } score += s; } + + if (o->mode == Addressing_Type && is_type_typeid(e->type)) { + add_type_info_type(c, o->type); + add_type_and_value(c->info, o->expr, Addressing_Value, e->type, exact_value_typeid(o->type)); + } } if (data) { @@ -6432,17 +6476,6 @@ ExprKind check_call_expr(CheckerContext *c, Operand *operand, Ast *call) { } } - // NOTE(bill): Should this be here or on the `add_entity_use`? - // if (ce->proc != nullptr) { - // Entity *e = entity_of_node(&c->info, ce->proc); - // if (e != nullptr && e->kind == Entity_Procedure) { - // String msg = e->Procedure.deprecated_message; - // if (msg.len > 0) { - // warning(call, "%.*s is deprecated: %.*s", LIT(e->token.string), LIT(msg)); - // } - // } - // } - CallArgumentData data = check_call_arguments(c, operand, proc_type, call); Type *result_type = data.result_type; gb_zero_item(operand); diff --git a/src/checker.cpp b/src/checker.cpp index 994e0776d..f30273439 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2613,14 +2613,14 @@ void check_collect_value_decl(CheckerContext *c, Ast *decl) { Entity *e = nullptr; d->attributes = vd->attributes; + d->type_expr = vd->type; + d->init_expr = init; if (is_ast_type(init)) { e = alloc_entity_type_name(d->scope, token, nullptr); - if (vd->type != nullptr) { - error(name, "A type declaration cannot have an type parameter"); - } - d->type_expr = init; - d->init_expr = init; + // if (vd->type != nullptr) { + // error(name, "A type declaration cannot have an type parameter"); + // } } else if (init->kind == Ast_ProcLit) { if (c->scope->flags&ScopeFlag_Type) { error(name, "Procedure declarations are not allowed within a struct"); @@ -2647,19 +2647,15 @@ void check_collect_value_decl(CheckerContext *c, Ast *decl) { pl->type->ProcType.calling_convention = cc; } d->proc_lit = init; - d->type_expr = vd->type; + d->init_expr = init; } else if (init->kind == Ast_ProcGroup) { ast_node(pg, ProcGroup, init); e = alloc_entity_proc_group(d->scope, token, nullptr); if (fl != nullptr) { error(name, "Procedure groups are not allowed within a foreign block"); } - d->init_expr = init; - d->type_expr = vd->type; } else { e = alloc_entity_constant(d->scope, token, nullptr, empty_exact_value); - d->type_expr = vd->type; - d->init_expr = init; } e->identifier = name; diff --git a/src/exact_value.cpp b/src/exact_value.cpp index fcc6f1973..95d04d93b 100644 --- a/src/exact_value.cpp +++ b/src/exact_value.cpp @@ -38,6 +38,7 @@ enum ExactValueKind { ExactValue_Pointer, ExactValue_Compound, // TODO(bill): Is this good enough? ExactValue_Procedure, // TODO(bill): Is this good enough? + ExactValue_Typeid, ExactValue_Count, }; @@ -54,6 +55,7 @@ struct ExactValue { Quaternion256 value_quaternion; Ast * value_compound; Ast * value_procedure; + Type * value_typeid; }; }; @@ -82,6 +84,8 @@ HashKey hash_exact_value(ExactValue v) { return hash_pointer(v.value_compound); case ExactValue_Procedure: return hash_pointer(v.value_procedure); + case ExactValue_Typeid: + return hash_pointer(v.value_typeid); } return hashing_proc(&v, gb_size_of(ExactValue)); @@ -154,6 +158,13 @@ ExactValue exact_value_procedure(Ast *node) { } +ExactValue exact_value_typeid(Type *type) { + ExactValue result = {ExactValue_Typeid}; + result.value_typeid = type; + return result; +} + + ExactValue exact_value_integer_from_string(String const &string) { ExactValue result = {ExactValue_Integer}; big_int_from_string(&result.value_integer, string); @@ -889,6 +900,13 @@ bool compare_exact_values(TokenKind op, ExactValue x, ExactValue y) { } break; } + + case ExactValue_Typeid: + switch (op) { + case Token_CmpEq: return are_types_identical(x.value_typeid, y.value_typeid); + case Token_NotEq: return !are_types_identical(x.value_typeid, y.value_typeid); + } + break; } GB_PANIC("Invalid comparison"); diff --git a/src/ir.cpp b/src/ir.cpp index 758267c24..f29d389f4 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -6592,6 +6592,11 @@ irValue *ir_build_expr_internal(irProcedure *proc, Ast *expr) { return ir_emit_conv(proc, x, tv.type); } + if (tv.value.kind == ExactValue_Typeid) { + irValue *v = ir_typeid(proc->module, tv.value.value_typeid); + return ir_emit_conv(proc, v, tv.type); + } + return ir_add_module_constant(proc->module, tv.type, tv.value); } diff --git a/src/parser.cpp b/src/parser.cpp index 204ff3984..fb093ffd9 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1923,10 +1923,6 @@ Ast *parse_operand(AstFile *f, bool lhs) { case Token_typeid: { Token token = expect_token(f, Token_typeid); - // Ast *specialization = nullptr; - // if (allow_token(f, Token_Quo)) { - // specialization = parse_type(f); - // } return ast_typeid_type(f, token, nullptr); } break; -- cgit v1.2.3 From d62503d031855534adddc37f84bd2e92dc2a5467 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 6 Oct 2019 18:14:02 +0100 Subject: Change precedence for `in` and `notin` to match + - | ~ --- core/odin/parser/parser.odin | 10 +++++----- examples/demo/demo.odin | 10 +--------- src/parser.cpp | 11 ++++++----- 3 files changed, 12 insertions(+), 19 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index f652e2c02..7c5c096ba 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1144,17 +1144,17 @@ token_precedence :: proc(p: ^Parser, kind: token.Kind) -> int { token.Lt_Eq, token.Gt_Eq: return 5; case token.In, token.Notin: - if p.expr_level >= 0 || p.allow_in_expr { - return 6; + if p.expr_level < 0 && !p.allow_in_expr { + return 0; } - return 0; + fallthrough; case token.Add, token.Sub, token.Or, token.Xor: - return 7; + return 6; case token.Mul, token.Quo, token.Mod, token.Mod_Mod, token.And, token.And_Not, token.Shl, token.Shr: - return 8; + return 7; } return 0; } diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index 2c8148501..1460bf8be 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -1190,15 +1190,7 @@ where_clauses :: proc() { } main :: proc() { - x := "foobarbaz"; - i : int; - i = strings.last_index(x, "foo"); fmt.println(i); - i = strings.last_index(x, "bar"); fmt.println(i); - i = strings.last_index(x, "baz"); fmt.println(i); - i = strings.last_index(x, "asd"); fmt.println(i); - i = strings.last_index(x, "a"); fmt.println(i); - i = strings.last_index(x, "ba"); fmt.println(i); - when false { + when true { general_stuff(); union_type(); parametric_polymorphism(); diff --git a/src/parser.cpp b/src/parser.cpp index fb093ffd9..29254d7e8 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -2479,17 +2479,18 @@ i32 token_precedence(AstFile *f, TokenKind t) { case Token_LtEq: case Token_GtEq: return 5; + case Token_in: case Token_notin: - if (f->expr_level >= 0 || f->allow_in_expr) { - return 6; + if (f->expr_level < 0 && !f->allow_in_expr) { + return 0; } - return 0; + /*fallthrough*/ case Token_Add: case Token_Sub: case Token_Or: case Token_Xor: - return 7; + return 6; case Token_Mul: case Token_Quo: case Token_Mod: @@ -2498,7 +2499,7 @@ i32 token_precedence(AstFile *f, TokenKind t) { case Token_AndNot: case Token_Shl: case Token_Shr: - return 8; + return 7; } return 0; } -- cgit v1.2.3 From 939459b635b5042b10ab64a28efc573600ac98cb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 6 Oct 2019 19:16:55 +0100 Subject: Change implicit semicolon rules for record types within procedure bodies; Update `package odin/*` --- core/encoding/json/marshal.odin | 7 +- core/odin/ast/ast.odin | 188 ++++----- core/odin/ast/clone.odin | 4 +- core/odin/ast/file.odin | 4 +- core/odin/parser/parser.odin | 760 ++++++++++++++++++------------------- core/odin/token/token.odin | 333 ---------------- core/odin/tokenizer/tokenizer.odin | 151 ++++---- core/runtime/core.odin | 2 +- core/strconv/generic_float.odin | 2 +- src/parser.cpp | 3 +- 10 files changed, 560 insertions(+), 894 deletions(-) delete mode 100644 core/odin/token/token.odin (limited to 'src/parser.cpp') diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 52d75524a..60839b21b 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -284,11 +284,10 @@ marshal_arg :: proc(b: ^strings.Builder, v: any) -> Marshal_Error { t := runtime.type_info_base(ti); switch info in t.variant { case runtime.Type_Info_Integer: - using runtime.Type_Info_Endianness; switch info.endianness { - case Platform: return false; - case Little: return ODIN_ENDIAN != "little"; - case Big: return ODIN_ENDIAN != "big"; + case .Platform: return false; + case .Little: return ODIN_ENDIAN != "little"; + case .Big: return ODIN_ENDIAN != "big"; } } return false; diff --git a/core/odin/ast/ast.odin b/core/odin/ast/ast.odin index baad5f802..309d2c147 100644 --- a/core/odin/ast/ast.odin +++ b/core/odin/ast/ast.odin @@ -1,6 +1,6 @@ package odin_ast -import "core:odin/token" +import "core:odin/tokenizer" Proc_Tag :: enum { Bounds_Check, @@ -33,12 +33,12 @@ Node_State_Flags :: distinct bit_set[Node_State_Flag]; Comment_Group :: struct { - list: []token.Token, + list: []tokenizer.Token, } Node :: struct { - pos: token.Pos, - end: token.Pos, + pos: tokenizer.Pos, + end: tokenizer.Pos, derived: any, state_flags: Node_State_Flags, } @@ -67,29 +67,29 @@ Ident :: struct { Implicit :: struct { using node: Expr, - tok: token.Token, + tok: tokenizer.Token, } Undef :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, } Basic_Lit :: struct { using node: Expr, - tok: token.Token, + tok: tokenizer.Token, } Basic_Directive :: struct { using node: Expr, - tok: token.Token, + tok: tokenizer.Token, name: string, } Ellipsis :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, expr: ^Expr, } @@ -99,44 +99,44 @@ Proc_Lit :: struct { body: ^Stmt, tags: Proc_Tags, inlining: Proc_Inlining, - where_token: token.Token, + where_token: tokenizer.Token, where_clauses: []^Expr, } Comp_Lit :: struct { using node: Expr, type: ^Expr, - open: token.Pos, + open: tokenizer.Pos, elems: []^Expr, - close: token.Pos, + close: tokenizer.Pos, } Tag_Expr :: struct { using node: Expr, - op: token.Token, + op: tokenizer.Token, name: string, expr: ^Expr, } Unary_Expr :: struct { using node: Expr, - op: token.Token, + op: tokenizer.Token, expr: ^Expr, } Binary_Expr :: struct { using node: Expr, left: ^Expr, - op: token.Token, + op: tokenizer.Token, right: ^Expr, } Paren_Expr :: struct { using node: Expr, - open: token.Pos, + open: tokenizer.Pos, expr: ^Expr, - close: token.Pos, + close: tokenizer.Pos, } Selector_Expr :: struct { @@ -153,74 +153,74 @@ Implicit_Selector_Expr :: struct { Index_Expr :: struct { using node: Expr, expr: ^Expr, - open: token.Pos, + open: tokenizer.Pos, index: ^Expr, - close: token.Pos, + close: tokenizer.Pos, } Deref_Expr :: struct { using node: Expr, expr: ^Expr, - op: token.Token, + op: tokenizer.Token, } Slice_Expr :: struct { using node: Expr, expr: ^Expr, - open: token.Pos, + open: tokenizer.Pos, low: ^Expr, - interval: token.Token, + interval: tokenizer.Token, high: ^Expr, - close: token.Pos, + close: tokenizer.Pos, } Call_Expr :: struct { using node: Expr, inlining: Proc_Inlining, expr: ^Expr, - open: token.Pos, + open: tokenizer.Pos, args: []^Expr, - ellipsis: token.Token, - close: token.Pos, + ellipsis: tokenizer.Token, + close: tokenizer.Pos, } Field_Value :: struct { using node: Expr, field: ^Expr, - sep: token.Pos, + sep: tokenizer.Pos, value: ^Expr, } Ternary_Expr :: struct { using node: Expr, cond: ^Expr, - op1: token.Token, + op1: tokenizer.Token, x: ^Expr, - op2: token.Token, + op2: tokenizer.Token, y: ^Expr, } Type_Assertion :: struct { using node: Expr, expr: ^Expr, - dot: token.Pos, - open: token.Pos, + dot: tokenizer.Pos, + open: tokenizer.Pos, type: ^Expr, - close: token.Pos, + close: tokenizer.Pos, } Type_Cast :: struct { using node: Expr, - tok: token.Token, - open: token.Pos, + tok: tokenizer.Token, + open: tokenizer.Pos, type: ^Expr, - close: token.Pos, + close: tokenizer.Pos, expr: ^Expr, } Auto_Cast :: struct { using node: Expr, - op: token.Token, + op: tokenizer.Token, expr: ^Expr, } @@ -235,7 +235,7 @@ Bad_Stmt :: struct { Empty_Stmt :: struct { using node: Stmt, - semicolon: token.Pos, // Position of the following ';' + semicolon: tokenizer.Pos, // Position of the following ';' } Expr_Stmt :: struct { @@ -245,7 +245,7 @@ Expr_Stmt :: struct { Tag_Stmt :: struct { using node: Stmt, - op: token.Token, + op: tokenizer.Token, name: string, stmt: ^Stmt, } @@ -253,7 +253,7 @@ Tag_Stmt :: struct { Assign_Stmt :: struct { using node: Stmt, lhs: []^Expr, - op: token.Token, + op: tokenizer.Token, rhs: []^Expr, } @@ -261,15 +261,15 @@ Assign_Stmt :: struct { Block_Stmt :: struct { using node: Stmt, label: ^Expr, - open: token.Pos, + open: tokenizer.Pos, stmts: []^Stmt, - close: token.Pos, + close: tokenizer.Pos, } If_Stmt :: struct { using node: Stmt, label: ^Expr, - if_pos: token.Pos, + if_pos: tokenizer.Pos, init: ^Stmt, cond: ^Expr, body: ^Stmt, @@ -278,7 +278,7 @@ If_Stmt :: struct { When_Stmt :: struct { using node: Stmt, - when_pos: token.Pos, + when_pos: tokenizer.Pos, cond: ^Expr, body: ^Stmt, else_stmt: ^Stmt, @@ -297,7 +297,7 @@ Defer_Stmt :: struct { For_Stmt :: struct { using node: Stmt, label: ^Expr, - for_pos: token.Pos, + for_pos: tokenizer.Pos, init: ^Stmt, cond: ^Expr, post: ^Stmt, @@ -307,10 +307,10 @@ For_Stmt :: struct { Range_Stmt :: struct { using node: Stmt, label: ^Expr, - for_pos: token.Pos, + for_pos: tokenizer.Pos, val0: ^Expr, val1: ^Expr, - in_pos: token.Pos, + in_pos: tokenizer.Pos, expr: ^Expr, body: ^Stmt, } @@ -318,16 +318,16 @@ Range_Stmt :: struct { Case_Clause :: struct { using node: Stmt, - case_pos: token.Pos, + case_pos: tokenizer.Pos, list: []^Expr, - terminator: token.Token, + terminator: tokenizer.Token, body: []^Stmt, } Switch_Stmt :: struct { using node: Stmt, label: ^Expr, - switch_pos: token.Pos, + switch_pos: tokenizer.Pos, init: ^Stmt, cond: ^Expr, body: ^Stmt, @@ -337,7 +337,7 @@ Switch_Stmt :: struct { Type_Switch_Stmt :: struct { using node: Stmt, label: ^Expr, - switch_pos: token.Pos, + switch_pos: tokenizer.Pos, tag: ^Stmt, expr: ^Expr, body: ^Stmt, @@ -346,7 +346,7 @@ Type_Switch_Stmt :: struct { Branch_Stmt :: struct { using node: Stmt, - tok: token.Token, + tok: tokenizer.Token, label: ^Ident, } @@ -377,7 +377,7 @@ Value_Decl :: struct { Package_Decl :: struct { using node: Decl, docs: ^Comment_Group, - token: token.Token, + token: tokenizer.Token, name: string, comment: ^Comment_Group, } @@ -386,9 +386,9 @@ Import_Decl :: struct { using node: Decl, docs: ^Comment_Group, is_using: bool, - import_tok: token.Token, - name: token.Token, - relpath: token.Token, + import_tok: tokenizer.Token, + name: tokenizer.Token, + relpath: tokenizer.Token, fullpath: string, comment: ^Comment_Group, } @@ -397,7 +397,7 @@ Foreign_Block_Decl :: struct { using node: Decl, docs: ^Comment_Group, attributes: [dynamic]^Attribute, // dynamic as parsing will add to them lazily - tok: token.Token, + tok: tokenizer.Token, foreign_library: ^Expr, body: ^Stmt, } @@ -405,8 +405,8 @@ Foreign_Block_Decl :: struct { Foreign_Import_Decl :: struct { using node: Decl, docs: ^Comment_Group, - foreign_tok: token.Token, - import_tok: token.Token, + foreign_tok: tokenizer.Token, + import_tok: tokenizer.Token, name: ^Ident, collection_name: string, fullpaths: []string, @@ -467,18 +467,18 @@ Field_Flags_Signature_Results :: Field_Flags_Signature; Proc_Group :: struct { using node: Expr, - tok: token.Token, - open: token.Pos, + tok: tokenizer.Token, + open: tokenizer.Pos, args: []^Expr, - close: token.Pos, + close: tokenizer.Pos, } Attribute :: struct { using node: Node, - tok: token.Kind, - open: token.Pos, + tok: tokenizer.Token_Kind, + open: tokenizer.Pos, elems: []^Expr, - close: token.Pos, + close: tokenizer.Pos, } Field :: struct { @@ -487,57 +487,57 @@ Field :: struct { names: []^Expr, // Could be polymorphic type: ^Expr, default_value: ^Expr, - tag: token.Token, + tag: tokenizer.Token, flags: Field_Flags, comment: ^Comment_Group, } Field_List :: struct { using node: Node, - open: token.Pos, + open: tokenizer.Pos, list: []^Field, - close: token.Pos, + close: tokenizer.Pos, } // Types Typeid_Type :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, specialization: ^Expr, } Helper_Type :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, type: ^Expr, } Distinct_Type :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, type: ^Expr, } Opaque_Type :: struct { using node: Expr, - tok: token.Kind, + tok: tokenizer.Token_Kind, type: ^Expr, } Poly_Type :: struct { using node: Expr, - dollar: token.Pos, + dollar: tokenizer.Pos, type: ^Ident, specialization: ^Expr, } Proc_Type :: struct { using node: Expr, - tok: token.Token, + tok: tokenizer.Token, calling_convention: Proc_Calling_Convention, params: ^Field_List, - arrow: token.Pos, + arrow: tokenizer.Pos, results: ^Field_List, tags: Proc_Tags, generic: bool, @@ -546,34 +546,34 @@ Proc_Type :: struct { Pointer_Type :: struct { using node: Expr, - pointer: token.Pos, + pointer: tokenizer.Pos, elem: ^Expr, } Array_Type :: struct { using node: Expr, - open: token.Pos, + open: tokenizer.Pos, len: ^Expr, // Ellipsis node for [?]T arrray types, nil for slice types - close: token.Pos, + close: tokenizer.Pos, elem: ^Expr, } Dynamic_Array_Type :: struct { using node: Expr, - open: token.Pos, - dynamic_pos: token.Pos, - close: token.Pos, + open: tokenizer.Pos, + dynamic_pos: tokenizer.Pos, + close: tokenizer.Pos, elem: ^Expr, } Struct_Type :: struct { using node: Expr, - tok_pos: token.Pos, + tok_pos: tokenizer.Pos, poly_params: ^Field_List, align: ^Expr, fields: ^Field_List, name_count: int, - where_token: token.Token, + where_token: tokenizer.Token, where_clauses: []^Expr, is_packed: bool, is_raw_union: bool, @@ -581,46 +581,46 @@ Struct_Type :: struct { Union_Type :: struct { using node: Expr, - tok_pos: token.Pos, + tok_pos: tokenizer.Pos, poly_params: ^Field_List, align: ^Expr, variants: []^Expr, - where_token: token.Token, + where_token: tokenizer.Token, where_clauses: []^Expr, } Enum_Type :: struct { using node: Expr, - tok_pos: token.Pos, + tok_pos: tokenizer.Pos, base_type: ^Expr, - open: token.Pos, + open: tokenizer.Pos, fields: []^Expr, - close: token.Pos, + close: tokenizer.Pos, is_using: bool, } Bit_Field_Type :: struct { using node: Expr, - tok_pos: token.Pos, + tok_pos: tokenizer.Pos, align: ^Expr, - open: token.Pos, + open: tokenizer.Pos, fields: []^Field_Value, // Field_Value with ':' rather than '=' - close: token.Pos, + close: tokenizer.Pos, } Bit_Set_Type :: struct { using node: Expr, - tok_pos: token.Pos, - open: token.Pos, + tok_pos: tokenizer.Pos, + open: tokenizer.Pos, elem: ^Expr, underlying: ^Expr, - close: token.Pos, + close: tokenizer.Pos, } Map_Type :: struct { using node: Expr, - tok_pos: token.Pos, + tok_pos: tokenizer.Pos, key: ^Expr, value: ^Expr, } diff --git a/core/odin/ast/clone.odin b/core/odin/ast/clone.odin index ded47a931..ddb3068f1 100644 --- a/core/odin/ast/clone.odin +++ b/core/odin/ast/clone.odin @@ -2,9 +2,9 @@ package odin_ast import "core:mem" import "core:fmt" -import "core:odin/token" +import "core:odin/tokenizer" -new :: proc($T: typeid, pos, end: token.Pos) -> ^T { +new :: proc($T: typeid, pos, end: tokenizer.Pos) -> ^T { n := mem.new(T); n.pos = pos; n.end = end; diff --git a/core/odin/ast/file.odin b/core/odin/ast/file.odin index 9d7550f45..dafb10e14 100644 --- a/core/odin/ast/file.odin +++ b/core/odin/ast/file.odin @@ -1,6 +1,6 @@ package odin_ast -import "core:odin/token" +import "core:odin/tokenizer" Package_Kind :: enum { Normal, @@ -26,7 +26,7 @@ File :: struct { src: []byte, pkg_decl: ^Package_Decl, - pkg_token: token.Token, + pkg_token: tokenizer.Token, pkg_name: string, decls: [dynamic]^Stmt, diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index cc3768030..213041924 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1,13 +1,12 @@ package odin_parser import "core:odin/ast" -import "core:odin/token" import "core:odin/tokenizer" import "core:fmt" -Warning_Handler :: #type proc(pos: token.Pos, fmt: string, args: ..any); -Error_Handler :: #type proc(pos: token.Pos, fmt: string, args: ..any); +Warning_Handler :: #type proc(pos: tokenizer.Pos, fmt: string, args: ..any); +Error_Handler :: #type proc(pos: tokenizer.Pos, fmt: string, args: ..any); Parser :: struct { file: ^ast.File, @@ -16,8 +15,8 @@ Parser :: struct { warn: Warning_Handler, err: Error_Handler, - prev_tok: token.Token, - curr_tok: token.Token, + prev_tok: tokenizer.Token, + curr_tok: tokenizer.Token, // >= 0: In Expression // < 0: In Control Clause @@ -50,25 +49,25 @@ Import_Decl_Kind :: enum { -default_warning_handler :: proc(pos: token.Pos, msg: string, args: ..any) { +default_warning_handler :: proc(pos: tokenizer.Pos, msg: string, args: ..any) { fmt.eprintf("%s(%d:%d): Warning: ", pos.file, pos.line, pos.column); fmt.eprintf(msg, ..args); fmt.eprintf("\n"); } -default_error_handler :: proc(pos: token.Pos, msg: string, args: ..any) { +default_error_handler :: proc(pos: tokenizer.Pos, msg: string, args: ..any) { fmt.eprintf("%s(%d:%d): ", pos.file, pos.line, pos.column); fmt.eprintf(msg, ..args); fmt.eprintf("\n"); } -warn :: proc(p: ^Parser, pos: token.Pos, msg: string, args: ..any) { +warn :: proc(p: ^Parser, pos: tokenizer.Pos, msg: string, args: ..any) { if p.warn != nil { p.warn(pos, msg, ..args); } p.file.syntax_warning_count += 1; } -error :: proc(p: ^Parser, pos: token.Pos, msg: string, args: ..any) { +error :: proc(p: ^Parser, pos: tokenizer.Pos, msg: string, args: ..any) { if p.err != nil { p.err(pos, msg, ..args); } @@ -77,11 +76,11 @@ error :: proc(p: ^Parser, pos: token.Pos, msg: string, args: ..any) { } -end_pos :: proc(tok: token.Token) -> token.Pos { +end_pos :: proc(tok: tokenizer.Token) -> tokenizer.Pos { pos := tok.pos; pos.offset += len(tok.text); - if tok.kind == token.Comment { + if tok.kind == .Comment { if tok.text[:2] != "/*" { pos.column += len(tok.text); } else { @@ -133,13 +132,13 @@ parse_file :: proc(p: ^Parser, file: ^ast.File) -> bool { docs := p.lead_comment; - p.file.pkg_token = expect_token(p, token.Package); - if p.file.pkg_token.kind != token.Package { + p.file.pkg_token = expect_token(p, .Package); + if p.file.pkg_token.kind != .Package { return false; } - pkg_name := expect_token_after(p, token.Ident, "package"); - if pkg_name.kind == token.Ident { + pkg_name := expect_token_after(p, .Ident, "package"); + if pkg_name.kind == .Ident { if is_blank_ident(pkg_name) { error(p, pkg_name.pos, "invalid package name '_'"); } @@ -161,7 +160,7 @@ parse_file :: proc(p: ^Parser, file: ^ast.File) -> bool { p.file.decls = make([dynamic]^ast.Stmt); - for p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .EOF { stmt := parse_stmt(p); if stmt != nil { if _, ok := stmt.derived.(ast.Empty_Stmt); !ok { @@ -181,16 +180,16 @@ parse_file :: proc(p: ^Parser, file: ^ast.File) -> bool { next_token0 :: proc(p: ^Parser) -> bool { p.curr_tok = tokenizer.scan(&p.tok); - if p.curr_tok.kind == token.EOF { + if p.curr_tok.kind == .EOF { // error(p, p.curr_tok.pos, "token is EOF"); return false; } return true; } -consume_comment :: proc(p: ^Parser) -> (tok: token.Token, end_line: int) { +consume_comment :: proc(p: ^Parser) -> (tok: tokenizer.Token, end_line: int) { tok = p.curr_tok; - assert(tok.kind == token.Comment); + assert(tok.kind == .Comment); end_line = tok.pos.line; if tok.text[1] == '*' { @@ -210,11 +209,11 @@ consume_comment :: proc(p: ^Parser) -> (tok: token.Token, end_line: int) { } consume_comment_group :: proc(p: ^Parser, n: int) -> (comments: ^ast.Comment_Group, end_line: int) { - list: [dynamic]token.Token; + list: [dynamic]tokenizer.Token; end_line = p.curr_tok.pos.line; - for p.curr_tok.kind == token.Comment && + for p.curr_tok.kind == .Comment && p.curr_tok.pos.line <= end_line+n { - comment: token.Token; + comment: tokenizer.Token; comment, end_line = consume_comment(p); append(&list, comment); } @@ -228,31 +227,31 @@ consume_comment_group :: proc(p: ^Parser, n: int) -> (comments: ^ast.Comment_Gro return; } -consume_comment_groups :: proc(p: ^Parser, prev: token.Token) { - if p.curr_tok.kind == token.Comment { +consume_comment_groups :: proc(p: ^Parser, prev: tokenizer.Token) { + if p.curr_tok.kind == .Comment { comment: ^ast.Comment_Group; end_line := 0; if p.curr_tok.pos.line == prev.pos.line { comment, end_line = consume_comment_group(p, 0); - if p.curr_tok.pos.line != end_line || p.curr_tok.kind == token.EOF { + if p.curr_tok.pos.line != end_line || p.curr_tok.kind == .EOF { p.line_comment = comment; } } end_line = -1; - for p.curr_tok.kind == token.Comment { + for p.curr_tok.kind == .Comment { comment, end_line = consume_comment_group(p, 1); } if end_line+1 >= p.curr_tok.pos.line || end_line < 0 { p.lead_comment = comment; } - assert(p.curr_tok.kind != token.Comment); + assert(p.curr_tok.kind != .Comment); } } -advance_token :: proc(p: ^Parser) -> token.Token { +advance_token :: proc(p: ^Parser) -> tokenizer.Token { p.lead_comment = nil; p.line_comment = nil; p.prev_tok = p.curr_tok; @@ -264,39 +263,39 @@ advance_token :: proc(p: ^Parser) -> token.Token { return prev; } -expect_token :: proc(p: ^Parser, kind: token.Kind) -> token.Token { +expect_token :: proc(p: ^Parser, kind: tokenizer.Token_Kind) -> tokenizer.Token { prev := p.curr_tok; if prev.kind != kind { - e := token.to_string(kind); - g := token.to_string(prev.kind); + e := tokenizer.to_string(kind); + g := tokenizer.to_string(prev.kind); error(p, prev.pos, "expected '%s', got '%s'", e, g); } advance_token(p); return prev; } -expect_token_after :: proc(p: ^Parser, kind: token.Kind, msg: string) -> token.Token { +expect_token_after :: proc(p: ^Parser, kind: tokenizer.Token_Kind, msg: string) -> tokenizer.Token { prev := p.curr_tok; if prev.kind != kind { - e := token.to_string(kind); - g := token.to_string(prev.kind); + e := tokenizer.to_string(kind); + g := tokenizer.to_string(prev.kind); error(p, prev.pos, "expected '%s' after %s, got '%s'", e, msg, g); } advance_token(p); return prev; } -expect_operator :: proc(p: ^Parser) -> token.Token { +expect_operator :: proc(p: ^Parser) -> tokenizer.Token { prev := p.curr_tok; - if !token.is_operator(prev.kind) { - g := token.to_string(prev.kind); + if !tokenizer.is_operator(prev.kind) { + g := tokenizer.to_string(prev.kind); error(p, prev.pos, "expected an operator, got '%s'", g); } advance_token(p); return prev; } -allow_token :: proc(p: ^Parser, kind: token.Kind) -> bool { +allow_token :: proc(p: ^Parser, kind: tokenizer.Token_Kind) -> bool { if p.curr_tok.kind == kind { advance_token(p); return true; @@ -313,8 +312,8 @@ is_blank_ident :: proc{ is_blank_ident_string :: inline proc(str: string) -> bool { return str == "_"; } -is_blank_ident_token :: inline proc(tok: token.Token) -> bool { - if tok.kind == token.Ident { +is_blank_ident_token :: inline proc(tok: tokenizer.Token) -> bool { + if tok.kind == .Ident { return is_blank_ident_string(tok.text); } return false; @@ -347,7 +346,8 @@ is_semicolon_optional_for_node :: proc(p: ^Parser, node: ^ast.Node) -> bool { case ast.Pointer_Type: return is_semicolon_optional_for_node(p, n.elem); case ast.Struct_Type, ast.Union_Type, ast.Enum_Type, ast.Bit_Field_Type: - return true; + // Require semicolon within a procedure body + return p.curr_proc == nil; case ast.Proc_Lit: return true; @@ -371,16 +371,16 @@ is_semicolon_optional_for_node :: proc(p: ^Parser, node: ^ast.Node) -> bool { expect_semicolon :: proc(p: ^Parser, node: ^ast.Node) -> bool { - if allow_token(p, token.Semicolon) { + if allow_token(p, .Semicolon) { return true; } prev := p.prev_tok; - if prev.kind == token.Semicolon { + if prev.kind == .Semicolon { return true; } - if p.curr_tok.kind == token.EOF { + if p.curr_tok.kind == .EOF { return true; } @@ -391,20 +391,20 @@ expect_semicolon :: proc(p: ^Parser, node: ^ast.Node) -> bool { } } else { switch p.curr_tok.kind { - case token.Close_Brace: - case token.Close_Paren: - case token.Else: + case .Close_Brace: + case .Close_Paren: + case .Else: return true; } } } - error(p, prev.pos, "expected ';', got %s", token.to_string(prev.kind)); + error(p, prev.pos, "expected ';', got %s", tokenizer.to_string(prev.kind)); return false; } -new_blank_ident :: proc(p: ^Parser, pos: token.Pos) -> ^ast.Ident { - tok: token.Token; +new_blank_ident :: proc(p: ^Parser, pos: tokenizer.Pos) -> ^ast.Ident { + tok: tokenizer.Token; tok.pos = pos; i := ast.new(ast.Ident, pos, end_pos(tok)); i.name = "_"; @@ -415,11 +415,11 @@ parse_ident :: proc(p: ^Parser) -> ^ast.Ident { tok := p.curr_tok; pos := tok.pos; name := "_"; - if tok.kind == token.Ident { + if tok.kind == .Ident { name = tok.text; advance_token(p); } else { - expect_token(p, token.Ident); + expect_token(p, .Ident); } i := ast.new(ast.Ident, pos, end_pos(tok)); i.name = name; @@ -428,9 +428,9 @@ parse_ident :: proc(p: ^Parser) -> ^ast.Ident { parse_stmt_list :: proc(p: ^Parser) -> []^ast.Stmt { list: [dynamic]^ast.Stmt; - for p.curr_tok.kind != token.Case && - p.curr_tok.kind != token.Close_Brace && - p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .Case && + p.curr_tok.kind != .Close_Brace && + p.curr_tok.kind != .EOF { stmt := parse_stmt(p); if stmt != nil { if _, ok := stmt.derived.(ast.Empty_Stmt); !ok { @@ -454,7 +454,7 @@ parse_block_stmt :: proc(p: ^Parser, is_when: bool) -> ^ast.Stmt { } parse_when_stmt :: proc(p: ^Parser) -> ^ast.When_Stmt { - tok := expect_token(p, token.When); + tok := expect_token(p, .When); cond: ^ast.Expr; body: ^ast.Stmt; @@ -468,20 +468,20 @@ parse_when_stmt :: proc(p: ^Parser) -> ^ast.When_Stmt { if cond == nil { error(p, p.curr_tok.pos, "expected a condition for when statement"); } - if allow_token(p, token.Do) { + if allow_token(p, .Do) { body = convert_stmt_to_body(p, parse_stmt(p)); } else { body = parse_block_stmt(p, true); } - if allow_token(p, token.Else) { + if allow_token(p, .Else) { switch p.curr_tok.kind { - case token.When: + case .When: else_stmt = parse_when_stmt(p); - case token.Open_Brace: + case .Open_Brace: else_stmt = parse_block_stmt(p, true); - case token.Do: - expect_token(p, token.Do); + case .Do: + expect_token(p, .Do); else_stmt = convert_stmt_to_body(p, parse_stmt(p)); case: error(p, p.curr_tok.pos, "expected when statement block statement"); @@ -513,7 +513,7 @@ convert_stmt_to_expr :: proc(p: ^Parser, stmt: ^ast.Stmt, kind: string) -> ^ast. } parse_if_stmt :: proc(p: ^Parser) -> ^ast.If_Stmt { - tok := expect_token(p, token.If); + tok := expect_token(p, .If); init: ^ast.Stmt; cond: ^ast.Expr; @@ -524,11 +524,11 @@ parse_if_stmt :: proc(p: ^Parser) -> ^ast.If_Stmt { p.expr_level = -1; prev_allow_in_expr := p.allow_in_expr; p.allow_in_expr = true; - if allow_token(p, token.Semicolon) { + if allow_token(p, .Semicolon) { cond = parse_expr(p, false); } else { init = parse_simple_stmt(p, nil); - if allow_token(p, token.Semicolon) { + if allow_token(p, .Semicolon) { cond = parse_expr(p, false); } else { cond = convert_stmt_to_expr(p, init, "boolean expression"); @@ -543,20 +543,20 @@ parse_if_stmt :: proc(p: ^Parser) -> ^ast.If_Stmt { error(p, p.curr_tok.pos, "expected a condition for if statement"); } - if allow_token(p, token.Do) { + if allow_token(p, .Do) { body = convert_stmt_to_body(p, parse_stmt(p)); } else { body = parse_block_stmt(p, false); } - if allow_token(p, token.Else) { + if allow_token(p, .Else) { switch p.curr_tok.kind { - case token.If: + case .If: else_stmt = parse_if_stmt(p); - case token.Open_Brace: + case .Open_Brace: else_stmt = parse_block_stmt(p, false); - case token.Do: - expect_token(p, token.Do); + case .Do: + expect_token(p, .Do); else_stmt = convert_stmt_to_body(p, parse_stmt(p)); case: error(p, p.curr_tok.pos, "expected if statement block statement"); @@ -582,7 +582,7 @@ parse_for_stmt :: proc(p: ^Parser) -> ^ast.Stmt { error(p, p.curr_tok.pos, "you cannot use a for statement in the file scope"); } - tok := expect_token(p, token.For); + tok := expect_token(p, .For); init: ^ast.Stmt; cond: ^ast.Stmt; @@ -590,13 +590,13 @@ parse_for_stmt :: proc(p: ^Parser) -> ^ast.Stmt { body: ^ast.Stmt; is_range := false; - if p.curr_tok.kind != token.Open_Brace && p.curr_tok.kind != token.Do { + if p.curr_tok.kind != .Open_Brace && p.curr_tok.kind != .Do { prev_level := p.expr_level; defer p.expr_level = prev_level; p.expr_level = -1; - if p.curr_tok.kind == token.In { - in_tok := expect_token(p, token.In); + if p.curr_tok.kind == .In { + in_tok := expect_token(p, .In); rhs: ^ast.Expr; prev_allow_range := p.allow_range; @@ -604,7 +604,7 @@ parse_for_stmt :: proc(p: ^Parser) -> ^ast.Stmt { rhs = parse_expr(p, false); p.allow_range = prev_allow_range; - if allow_token(p, token.Do) { + if allow_token(p, .Do) { body = convert_stmt_to_body(p, parse_stmt(p)); } else { body = parse_body(p); @@ -618,27 +618,27 @@ parse_for_stmt :: proc(p: ^Parser) -> ^ast.Stmt { return range_stmt; } - if p.curr_tok.kind != token.Semicolon { + if p.curr_tok.kind != .Semicolon { cond = parse_simple_stmt(p, {Stmt_Allow_Flag.In}); - if as, ok := cond.derived.(ast.Assign_Stmt); ok && as.op.kind == token.In { + if as, ok := cond.derived.(ast.Assign_Stmt); ok && as.op.kind == .In { is_range = true; } } - if !is_range && allow_token(p, token.Semicolon) { + if !is_range && allow_token(p, .Semicolon) { init = cond; cond = nil; - if p.curr_tok.kind != token.Semicolon { + if p.curr_tok.kind != .Semicolon { cond = parse_simple_stmt(p, nil); } expect_semicolon(p, cond); - if p.curr_tok.kind != token.Open_Brace && p.curr_tok.kind != token.Do { + if p.curr_tok.kind != .Open_Brace && p.curr_tok.kind != .Do { post = parse_simple_stmt(p, nil); } } } - if allow_token(p, token.Do) { + if allow_token(p, .Do) { body = convert_stmt_to_body(p, parse_stmt(p)); } else { body = parse_body(p); @@ -686,11 +686,11 @@ parse_for_stmt :: proc(p: ^Parser) -> ^ast.Stmt { } parse_case_clause :: proc(p: ^Parser, is_type_switch: bool) -> ^ast.Case_Clause { - tok := expect_token(p, token.Case); + tok := expect_token(p, .Case); list: []^ast.Expr; - if p.curr_tok.kind != token.Colon { + if p.curr_tok.kind != .Colon { prev_allow_range, prev_allow_in_expr := p.allow_range, p.allow_in_expr; defer p.allow_range, p.allow_in_expr = prev_allow_range, prev_allow_in_expr; p.allow_range, p.allow_in_expr = !is_type_switch, !is_type_switch; @@ -698,7 +698,7 @@ parse_case_clause :: proc(p: ^Parser, is_type_switch: bool) -> ^ast.Case_Clause list = parse_rhs_expr_list(p); } - terminator := expect_token(p, token.Colon); + terminator := expect_token(p, .Colon); stmts := parse_stmt_list(p); @@ -710,20 +710,20 @@ parse_case_clause :: proc(p: ^Parser, is_type_switch: bool) -> ^ast.Case_Clause } parse_switch_stmt :: proc(p: ^Parser) -> ^ast.Stmt { - tok := expect_token(p, token.Switch); + tok := expect_token(p, .Switch); init: ^ast.Stmt; tag: ^ast.Stmt; is_type_switch := false; clauses: [dynamic]^ast.Stmt; - if p.curr_tok.kind != token.Open_Brace { + if p.curr_tok.kind != .Open_Brace { prev_level := p.expr_level; defer p.expr_level = prev_level; p.expr_level = -1; - if p.curr_tok.kind == token.In { - in_tok := expect_token(p, token.In); + if p.curr_tok.kind == .In { + in_tok := expect_token(p, .In); is_type_switch = true; lhs := make([]^ast.Expr, 1); @@ -738,12 +738,12 @@ parse_switch_stmt :: proc(p: ^Parser) -> ^ast.Stmt { tag = as; } else { tag = parse_simple_stmt(p, {Stmt_Allow_Flag.In}); - if as, ok := tag.derived.(ast.Assign_Stmt); ok && as.op.kind == token.In { + if as, ok := tag.derived.(ast.Assign_Stmt); ok && as.op.kind == .In { is_type_switch = true; - } else if allow_token(p, token.Semicolon) { + } else if allow_token(p, .Semicolon) { init = tag; tag = nil; - if p.curr_tok.kind != token.Open_Brace { + if p.curr_tok.kind != .Open_Brace { tag = parse_simple_stmt(p, nil); } } @@ -751,14 +751,14 @@ parse_switch_stmt :: proc(p: ^Parser) -> ^ast.Stmt { } - open := expect_token(p, token.Open_Brace); + open := expect_token(p, .Open_Brace); - for p.curr_tok.kind == token.Case { + for p.curr_tok.kind == .Case { clause := parse_case_clause(p, is_type_switch); append(&clauses, clause); } - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); body := ast.new(ast.Block_Stmt, open.pos, end_pos(close)); body.stmts = clauses[:]; @@ -778,23 +778,23 @@ parse_switch_stmt :: proc(p: ^Parser) -> ^ast.Stmt { } } -parse_attribute :: proc(p: ^Parser, tok: token.Token, open_kind, close_kind: token.Kind, docs: ^ast.Comment_Group) -> ^ast.Stmt { +parse_attribute :: proc(p: ^Parser, tok: tokenizer.Token, open_kind, close_kind: tokenizer.Token_Kind, docs: ^ast.Comment_Group) -> ^ast.Stmt { elems: [dynamic]^ast.Expr; - open, close: token.Token; + open, close: tokenizer.Token; - if p.curr_tok.kind == token.Ident { + if p.curr_tok.kind == .Ident { elem := parse_ident(p); append(&elems, elem); } else { open = expect_token(p, open_kind); p.expr_level += 1; for p.curr_tok.kind != close_kind && - p.curr_tok.kind != token.EOF { + p.curr_tok.kind != .EOF { elem: ^ast.Expr; elem = parse_ident(p); - if p.curr_tok.kind == token.Eq { - eq := expect_token(p, token.Eq); + if p.curr_tok.kind == .Eq { + eq := expect_token(p, .Eq); value := parse_value(p); fv := ast.new(ast.Field_Value, elem.pos, value.end); fv.field = elem; @@ -805,7 +805,7 @@ parse_attribute :: proc(p: ^Parser, tok: token.Token, open_kind, close_kind: tok } append(&elems, elem); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } @@ -852,12 +852,12 @@ parse_foreign_block_decl :: proc(p: ^Parser) -> ^ast.Stmt { } -parse_foreign_block :: proc(p: ^Parser, tok: token.Token) -> ^ast.Foreign_Block_Decl { +parse_foreign_block :: proc(p: ^Parser, tok: tokenizer.Token) -> ^ast.Foreign_Block_Decl { docs := p.lead_comment; foreign_library: ^ast.Expr; switch p.curr_tok.kind { - case token.Open_Brace: + case .Open_Brace: i := ast.new(ast.Ident, tok.pos, end_pos(tok)); i.name = "_"; foreign_library = i; @@ -871,14 +871,14 @@ parse_foreign_block :: proc(p: ^Parser, tok: token.Token) -> ^ast.Foreign_Block_ defer p.in_foreign_block = prev_in_foreign_block; p.in_foreign_block = true; - open := expect_token(p, token.Open_Brace); - for p.curr_tok.kind != token.Close_Brace && p.curr_tok.kind != token.EOF { + open := expect_token(p, .Open_Brace); + for p.curr_tok.kind != .Close_Brace && p.curr_tok.kind != .EOF { decl := parse_foreign_block_decl(p); if decl != nil { append(&decls, decl); } } - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); body := ast.new(ast.Block_Stmt, open.pos, end_pos(close)); body.open = open.pos; @@ -896,16 +896,16 @@ parse_foreign_block :: proc(p: ^Parser, tok: token.Token) -> ^ast.Foreign_Block_ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl { docs := p.lead_comment; - tok := expect_token(p, token.Foreign); + tok := expect_token(p, .Foreign); switch p.curr_tok.kind { - case token.Ident, token.Open_Brace: + case .Ident, .Open_Brace: return parse_foreign_block(p, tok); - case token.Import: - import_tok := expect_token(p, token.Import); + case .Import: + import_tok := expect_token(p, .Import); name: ^ast.Ident; - if p.curr_tok.kind == token.Ident { + if p.curr_tok.kind == .Ident { name = parse_ident(p); } @@ -914,19 +914,19 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl { } fullpaths: [dynamic]string; - if allow_token(p, token.Open_Brace) { - for p.curr_tok.kind != token.Close_Brace && - p.curr_tok.kind != token.EOF { - path := expect_token(p, token.String); + if allow_token(p, .Open_Brace) { + for p.curr_tok.kind != .Close_Brace && + p.curr_tok.kind != .EOF { + path := expect_token(p, .String); append(&fullpaths, path.text); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - expect_token(p, token.Close_Brace); + expect_token(p, .Close_Brace); } else { - path := expect_token(p, token.String); + path := expect_token(p, .String); reserve(&fullpaths, 1); append(&fullpaths, path.text); } @@ -954,29 +954,29 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl { parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { switch p.curr_tok.kind { // Operands - case token.Context, // Also allows for 'context = ' - token.Proc, - token.Inline, token.No_Inline, - token.Ident, - token.Integer, token.Float, token.Imag, - token.Rune, token.String, - token.Open_Paren, - token.Pointer, + case .Context, // Also allows for 'context = ' + .Proc, + .Inline, .No_Inline, + .Ident, + .Integer, .Float, .Imag, + .Rune, .String, + .Open_Paren, + .Pointer, // Unary Expressions - token.Add, token.Sub, token.Xor, token.Not, token.And: + .Add, .Sub, .Xor, .Not, .And: s := parse_simple_stmt(p, {Stmt_Allow_Flag.Label}); expect_semicolon(p, s); return s; - case token.Import: return parse_import_decl(p); - case token.Foreign: return parse_foreign_decl(p); - case token.If: return parse_if_stmt(p); - case token.When: return parse_when_stmt(p); - case token.For: return parse_for_stmt(p); - case token.Switch: return parse_switch_stmt(p); + case .Import: return parse_import_decl(p); + case .Foreign: return parse_foreign_decl(p); + case .If: return parse_if_stmt(p); + case .When: return parse_when_stmt(p); + case .For: return parse_for_stmt(p); + case .Switch: return parse_switch_stmt(p); - case token.Defer: + case .Defer: tok := advance_token(p); stmt := parse_stmt(p); switch s in stmt.derived { @@ -992,7 +992,7 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { ds.stmt = stmt; return ds; - case token.Return: + case .Return: tok := advance_token(p); if p.expr_level > 0 { @@ -1000,11 +1000,11 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { } results: [dynamic]^ast.Expr; - for p.curr_tok.kind != token.Semicolon { + for p.curr_tok.kind != .Semicolon { result := parse_expr(p, false); append(&results, result); - if p.curr_tok.kind != token.Comma || - p.curr_tok.kind == token.EOF { + if p.curr_tok.kind != .Comma || + p.curr_tok.kind == .EOF { break; } advance_token(p); @@ -1019,10 +1019,10 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { rs.results = results[:]; return rs; - case token.Break, token.Continue, token.Fallthrough: + case .Break, .Continue, .Fallthrough: tok := advance_token(p); label: ^ast.Expr; - if tok.kind != token.Fallthrough && p.curr_tok.kind == token.Ident { + if tok.kind != .Fallthrough && p.curr_tok.kind == .Ident { label = parse_ident(p); } end := label != nil ? label.end : end_pos(tok); @@ -1030,11 +1030,11 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { expect_semicolon(p, s); return s; - case token.Using: + case .Using: docs := p.lead_comment; - tok := expect_token(p, token.Using); + tok := expect_token(p, .Using); - if p.curr_tok.kind == token.Import { + if p.curr_tok.kind == .Import { return parse_import_decl(p, Import_Decl_Kind.Using); } @@ -1045,14 +1045,14 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { return ast.new(ast.Bad_Stmt, tok.pos, end_pos(p.prev_tok)); } - if p.curr_tok.kind != token.Colon { + if p.curr_tok.kind != .Colon { end := list[len(list)-1]; expect_semicolon(p, end); us := ast.new(ast.Using_Stmt, tok.pos, end.end); us.list = list; return us; } - expect_token_after(p, token.Colon, "identifier list"); + expect_token_after(p, .Colon, "identifier list"); decl := parse_value_decl(p, list, docs); if decl != nil do switch d in &decl.derived { case ast.Value_Decl: @@ -1063,14 +1063,14 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { error(p, tok.pos, "illegal use of 'using' statement"); return ast.new(ast.Bad_Stmt, tok.pos, end_pos(p.prev_tok)); - case token.At: + case .At: docs := p.lead_comment; tok := advance_token(p); - return parse_attribute(p, tok, token.Open_Paren, token.Close_Paren, docs); + return parse_attribute(p, tok, .Open_Paren, .Close_Paren, docs); - case token.Hash: - tok := expect_token(p, token.Hash); - tag := expect_token(p, token.Ident); + case .Hash: + tok := expect_token(p, .Hash); + tag := expect_token(p, .Ident); name := tag.text; switch name { @@ -1110,50 +1110,50 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { te.stmt = stmt; return te; } - case token.Open_Brace: + case .Open_Brace: return parse_block_stmt(p, false); - case token.Semicolon: + case .Semicolon: tok := advance_token(p); s := ast.new(ast.Empty_Stmt, tok.pos, end_pos(tok)); return s; } tok := advance_token(p); - error(p, tok.pos, "expected a statement, got %s", token.to_string(tok.kind)); + error(p, tok.pos, "expected a statement, got %s", tokenizer.to_string(tok.kind)); s := ast.new(ast.Bad_Stmt, tok.pos, end_pos(tok)); return s; } -token_precedence :: proc(p: ^Parser, kind: token.Kind) -> int { +token_precedence :: proc(p: ^Parser, kind: tokenizer.Token_Kind) -> int { switch kind { - case token.Question: + case .Question: return 1; - case token.Ellipsis, token.Range_Half: + case .Ellipsis, .Range_Half: if !p.allow_range { return 0; } return 2; - case token.Cmp_Or: + case .Cmp_Or: return 3; - case token.Cmp_And: + case .Cmp_And: return 4; - case token.Cmp_Eq, token.Not_Eq, - token.Lt, token.Gt, - token.Lt_Eq, token.Gt_Eq: + case .Cmp_Eq, .Not_Eq, + .Lt, .Gt, + .Lt_Eq, .Gt_Eq: return 5; - case token.In, token.Notin: + case .In, .Notin: if p.expr_level < 0 && !p.allow_in_expr { return 0; } fallthrough; - case token.Add, token.Sub, token.Or, token.Xor: + case .Add, .Sub, .Or, .Xor: return 6; - case token.Mul, token.Quo, - token.Mod, token.Mod_Mod, - token.And, token.And_Not, - token.Shl, token.Shr: + case .Mul, .Quo, + .Mod, .Mod_Mod, + .And, .And_Not, + .Shl, .Shr: return 7; } return 0; @@ -1187,9 +1187,9 @@ parse_body :: proc(p: ^Parser) -> ^ast.Block_Stmt { defer p.expr_level = prev_expr_level; p.expr_level = 0; - open := expect_token(p, token.Open_Brace); + open := expect_token(p, .Open_Brace); stmts := parse_stmt_list(p); - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); bs := ast.new(ast.Block_Stmt, open.pos, end_pos(close)); bs.open = open.pos; @@ -1216,7 +1216,7 @@ convert_stmt_to_body :: proc(p: ^Parser, stmt: ^ast.Stmt) -> ^ast.Stmt { } new_ast_field :: proc(names: []^ast.Expr, type: ^ast.Expr, default_value: ^ast.Expr) -> ^ast.Field { - pos, end: token.Pos; + pos, end: tokenizer.Pos; if len(names) > 0 { pos = names[0].pos; @@ -1306,22 +1306,22 @@ convert_to_ident_list :: proc(p: ^Parser, list: []Expr_And_Flags, ignore_flags, is_token_field_prefix :: proc(p: ^Parser) -> Field_Prefix { using Field_Prefix; switch p.curr_tok.kind { - case token.EOF: + case .EOF: return Invalid; - case token.Using: + case .Using: advance_token(p); return Using; - case token.In: + case .In: advance_token(p); return In; - case token.Auto_Cast: + case .Auto_Cast: advance_token(p); return Auto_Cast; - case token.Hash: + case .Hash: advance_token(p); defer advance_token(p); switch p.curr_tok.kind { - case token.Ident: + case .Ident: switch p.curr_tok.text { case "no_alias": return No_Alias; @@ -1414,7 +1414,7 @@ check_field_flag_prefixes :: proc(p: ^Parser, name_count: int, allowed_flags, se } parse_var_type :: proc(p: ^Parser, flags: ast.Field_Flags) -> ^ast.Expr { - if ast.Field_Flag.Ellipsis in flags && p.curr_tok.kind == token.Ellipsis { + if ast.Field_Flag.Ellipsis in flags && p.curr_tok.kind == .Ellipsis { tok := advance_token(p); type := parse_type_or_ident(p); if type == nil { @@ -1426,11 +1426,11 @@ parse_var_type :: proc(p: ^Parser, flags: ast.Field_Flags) -> ^ast.Expr { return e; } type: ^ast.Expr; - if ast.Field_Flag.Typeid_Token in flags && p.curr_tok.kind == token.Typeid { - tok := expect_token(p, token.Typeid); + if ast.Field_Flag.Typeid_Token in flags && p.curr_tok.kind == .Typeid { + tok := expect_token(p, .Typeid); specialization: ^ast.Expr; end := tok.pos; - if allow_token(p, token.Quo) { + if allow_token(p, .Quo) { specialization = parse_type(p); end = specialization.end; } @@ -1482,8 +1482,8 @@ parse_ident_list :: proc(p: ^Parser, allow_poly_names: bool) -> []^ast.Expr { list: [dynamic]^ast.Expr; for { - if allow_poly_names && p.curr_tok.kind == token.Dollar { - tok := expect_token(p, token.Dollar); + if allow_poly_names && p.curr_tok.kind == .Dollar { + tok := expect_token(p, .Dollar); ident := parse_ident(p); if is_blank_ident(ident) { error(p, ident.pos, "invalid polymorphic type definition with a blank identifier"); @@ -1495,8 +1495,8 @@ parse_ident_list :: proc(p: ^Parser, allow_poly_names: bool) -> []^ast.Expr { ident := parse_ident(p); append(&list, ident); } - if p.curr_tok.kind != token.Comma || - p.curr_tok.kind == token.EOF { + if p.curr_tok.kind != .Comma || + p.curr_tok.kind == .EOF { break; } advance_token(p); @@ -1507,7 +1507,7 @@ parse_ident_list :: proc(p: ^Parser, allow_poly_names: bool) -> []^ast.Expr { -parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Field_Flags) -> (field_list: ^ast.Field_List, total_name_count: int) { +parse_field_list :: proc(p: ^Parser, follow: tokenizer.Token_Kind, allowed_flags: ast.Field_Flags) -> (field_list: ^ast.Field_List, total_name_count: int) { handle_field :: proc(p: ^Parser, seen_ellipsis: ^bool, fields: ^[dynamic]^ast.Field, docs: ^ast.Comment_Group, @@ -1517,10 +1517,10 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel expect_field_separator :: proc(p: ^Parser, param: ^ast.Expr) -> bool { tok := p.curr_tok; - if allow_token(p, token.Comma) { + if allow_token(p, .Comma) { return true; } - if allow_token(p, token.Semicolon) { + if allow_token(p, .Semicolon) { error(p, tok.pos, "expected a comma, got a semicolon"); return true; } @@ -1539,10 +1539,10 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel type: ^ast.Expr; default_value: ^ast.Expr; - tag: token.Token; + tag: tokenizer.Token; - expect_token_after(p, token.Colon, "field list"); - if p.curr_tok.kind != token.Eq { + expect_token_after(p, .Colon, "field list"); + if p.curr_tok.kind != .Eq { type = parse_var_type(p, allowed_flags); tt := ast.unparen_expr(type); if is_signature && !any_polymorphic_names { @@ -1552,7 +1552,7 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel } } - if allow_token(p, token.Eq) { + if allow_token(p, .Eq) { default_value = parse_expr(p, false); if ast.Field_Flag.Default_Parameters notin allowed_flags { error(p, p.curr_tok.pos, "default parameters are only allowed for procedures"); @@ -1580,8 +1580,8 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel } if type != nil && default_value == nil { - if p.curr_tok.kind == token.String { - tag = expect_token(p, token.String); + if p.curr_tok.kind == .String { + tag = expect_token(p, .String); if .Tags notin allowed_flags { error(p, tag.pos, "Field tags are only allowed within structures"); } @@ -1616,8 +1616,8 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel allow_poly_names := allow_typeid_token; for p.curr_tok.kind != follow && - p.curr_tok.kind != token.Colon && - p.curr_tok.kind != token.EOF { + p.curr_tok.kind != .Colon && + p.curr_tok.kind != .EOF { prefix_flags := parse_field_prefixes(p); param := parse_var_type(p, allowed_flags & {ast.Field_Flag.Typeid_Token, ast.Field_Flag.Ellipsis}); if _, ok := param.derived.(ast.Ellipsis); ok { @@ -1631,15 +1631,15 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel eaf := Expr_And_Flags{param, prefix_flags}; append(&list, eaf); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - if p.curr_tok.kind != token.Colon { + if p.curr_tok.kind != .Colon { for eaf in list { type := eaf.expr; - tok: token.Token; + tok: tokenizer.Token; tok.pos = type.pos; if ast.Field_Flag.Results notin allowed_flags { tok.text = "_"; @@ -1670,7 +1670,7 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel total_name_count += len(names); handle_field(p, &seen_ellipsis, &fields, docs, names, allowed_flags, set_flags); - for p.curr_tok.kind != follow && p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != follow && p.curr_tok.kind != .EOF { docs = p.lead_comment; set_flags = parse_field_prefixes(p); names = parse_ident_list(p, allow_poly_names); @@ -1690,11 +1690,11 @@ parse_field_list :: proc(p: ^Parser, follow: token.Kind, allowed_flags: ast.Fiel parse_results :: proc(p: ^Parser) -> (list: ^ast.Field_List, diverging: bool) { - if !allow_token(p, token.Arrow_Right) { + if !allow_token(p, .Arrow_Right) { return; } - if allow_token(p, token.Not) { + if allow_token(p, .Not) { diverging = true; return; } @@ -1702,7 +1702,7 @@ parse_results :: proc(p: ^Parser) -> (list: ^ast.Field_List, diverging: bool) { prev_level := p.expr_level; defer p.expr_level = prev_level; - if p.curr_tok.kind != token.Open_Paren { + if p.curr_tok.kind != .Open_Paren { type := parse_type(p); field := new_ast_field(nil, type, nil); @@ -1712,9 +1712,9 @@ parse_results :: proc(p: ^Parser) -> (list: ^ast.Field_List, diverging: bool) { return; } - expect_token(p, token.Open_Paren); - list, _ = parse_field_list(p, token.Close_Paren, ast.Field_Flags_Signature_Results); - expect_token_after(p, token.Close_Paren, "parameter list"); + expect_token(p, .Open_Paren); + list, _ = parse_field_list(p, .Close_Paren, ast.Field_Flags_Signature_Results); + expect_token_after(p, .Close_Paren, "parameter list"); return; } @@ -1740,9 +1740,9 @@ string_to_calling_convention :: proc(s: string) -> ast.Proc_Calling_Convention { } parse_proc_tags :: proc(p: ^Parser) -> (tags: ast.Proc_Tags) { - for p.curr_tok.kind == token.Hash { - _ = expect_token(p, token.Hash); - ident := expect_token(p, token.Ident); + for p.curr_tok.kind == .Hash { + _ = expect_token(p, .Hash); + ident := expect_token(p, .Ident); switch ident.text { case "bounds_check": @@ -1760,10 +1760,10 @@ parse_proc_tags :: proc(p: ^Parser) -> (tags: ast.Proc_Tags) { return; } -parse_proc_type :: proc(p: ^Parser, tok: token.Token) -> ^ast.Proc_Type { +parse_proc_type :: proc(p: ^Parser, tok: tokenizer.Token) -> ^ast.Proc_Type { cc := ast.Proc_Calling_Convention.Invalid; - if p.curr_tok.kind == token.String { - str := expect_token(p, token.String); + if p.curr_tok.kind == .String { + str := expect_token(p, .String); cc = string_to_calling_convention(str.text); if cc == ast.Proc_Calling_Convention.Invalid { error(p, str.pos, "unknown calling convention '%s'", str.text); @@ -1778,9 +1778,9 @@ parse_proc_type :: proc(p: ^Parser, tok: token.Token) -> ^ast.Proc_Type { } } - expect_token(p, token.Open_Paren); - params, _ := parse_field_list(p, token.Close_Paren, ast.Field_Flags_Signature_Params); - expect_token(p, token.Close_Paren); + expect_token(p, .Open_Paren); + params, _ := parse_field_list(p, .Close_Paren, ast.Field_Flags_Signature_Params); + expect_token(p, .Close_Paren); results, diverging := parse_results(p); is_generic := false; @@ -1811,7 +1811,7 @@ parse_proc_type :: proc(p: ^Parser, tok: token.Token) -> ^ast.Proc_Type { return pt; } -check_poly_params_for_type :: proc(p: ^Parser, poly_params: ^ast.Field_List, tok: token.Token) { +check_poly_params_for_type :: proc(p: ^Parser, poly_params: ^ast.Field_List, tok: tokenizer.Token) { if poly_params == nil { return; } @@ -1830,46 +1830,46 @@ check_poly_params_for_type :: proc(p: ^Parser, poly_params: ^ast.Field_List, tok parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { switch p.curr_tok.kind { - case token.Ident: + case .Ident: return parse_ident(p); - case token.Undef: - tok := expect_token(p, token.Undef); + case .Undef: + tok := expect_token(p, .Undef); undef := ast.new(ast.Undef, tok.pos, end_pos(tok)); undef.tok = tok.kind; return undef; - case token.Context: - tok := expect_token(p, token.Context); + case .Context: + tok := expect_token(p, .Context); ctx := ast.new(ast.Implicit, tok.pos, end_pos(tok)); ctx.tok = tok; return ctx; - case token.Integer, token.Float, token.Imag, - token.Rune, token.String: + case .Integer, .Float, .Imag, + .Rune, .String: tok := advance_token(p); bl := ast.new(ast.Basic_Lit, tok.pos, end_pos(tok)); bl.tok = tok; return bl; - case token.Size_Of, token.Align_Of, token.Offset_Of: + case .Size_Of, .Align_Of, .Offset_Of: tok := advance_token(p); expr := ast.new(ast.Implicit, tok.pos, end_pos(tok)); expr.tok = tok; return parse_call_expr(p, expr); - case token.Open_Brace: + case .Open_Brace: if !lhs { return parse_literal_value(p, nil); } - case token.Open_Paren: - open := expect_token(p, token.Open_Paren); + case .Open_Paren: + open := expect_token(p, .Open_Paren); p.expr_level += 1; expr := parse_expr(p, false); p.expr_level -= 1; - close := expect_token(p, token.Close_Paren); + close := expect_token(p, .Close_Paren); pe := ast.new(ast.Paren_Expr, open.pos, end_pos(close)); pe.open = open.pos; @@ -1877,7 +1877,7 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { pe.close = close.pos; return pe; - case token.Distinct: + case .Distinct: tok := advance_token(p); type := parse_type(p); dt := ast.new(ast.Distinct_Type, tok.pos, type.end); @@ -1885,16 +1885,16 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { dt.type = type; return dt; - case token.Opaque: + case .Opaque: tok := advance_token(p); type := parse_type(p); ot := ast.new(ast.Opaque_Type, tok.pos, type.end); ot.tok = tok.kind; ot.type = type; return ot; - case token.Hash: - tok := expect_token(p, token.Hash); - name := expect_token(p, token.Ident); + case .Hash: + tok := expect_token(p, .Hash); + name := expect_token(p, .Ident); switch name.text { case "type": type := parse_type(p); @@ -1922,15 +1922,15 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { return te; } - case token.Inline, token.No_Inline: + case .Inline, .No_Inline: tok := advance_token(p); expr := parse_unary_expr(p, lhs); pi := ast.Proc_Inlining.None; switch tok.kind { - case token.Inline: + case .Inline: pi = ast.Proc_Inlining.Inline; - case token.No_Inline: + case .No_Inline: pi = ast.Proc_Inlining.No_Inline; } @@ -1951,25 +1951,25 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { } return expr; - case token.Proc: - tok := expect_token(p, token.Proc); + case .Proc: + tok := expect_token(p, .Proc); - if p.curr_tok.kind == token.Open_Brace { - open := expect_token(p, token.Open_Brace); + if p.curr_tok.kind == .Open_Brace { + open := expect_token(p, .Open_Brace); args: [dynamic]^ast.Expr; - for p.curr_tok.kind != token.Close_Brace && - p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .Close_Brace && + p.curr_tok.kind != .EOF { elem := parse_expr(p, false); append(&args, elem); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); if len(args) == 0 { error(p, tok.pos, "expected at least 1 argument in procedure group"); @@ -1985,10 +1985,10 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { type := parse_proc_type(p, tok); - where_token: token.Token; + where_token: tokenizer.Token; where_clauses: []^ast.Expr; - if (p.curr_tok.kind == token.Where) { - where_token = expect_token(p, token.Where); + if (p.curr_tok.kind == .Where) { + where_token = expect_token(p, .Where); prev_level := p.expr_level; p.expr_level = -1; where_clauses = parse_rhs_expr_list(p); @@ -1996,24 +1996,24 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { } if p.allow_type && p.expr_level < 0 { - if where_token.kind != token.Invalid { + if where_token.kind != .Invalid { error(p, where_token.pos, "'where' clauses are not allowed on procedure types"); } return type; } body: ^ast.Stmt; - if allow_token(p, token.Undef) { + if allow_token(p, .Undef) { // Okay - if where_token.kind != token.Invalid { + if where_token.kind != .Invalid { error(p, where_token.pos, "'where' clauses are not allowed on procedure literals without a defined body (replaced with ---"); } - } else if p.curr_tok.kind == token.Open_Brace { + } else if p.curr_tok.kind == .Open_Brace { prev_proc := p.curr_proc; p.curr_proc = type; body = parse_body(p); p.curr_proc = prev_proc; - } else if allow_token(p, token.Do) { + } else if allow_token(p, .Do) { prev_proc := p.curr_proc; p.curr_proc = type; body = convert_stmt_to_body(p, parse_stmt(p)); @@ -2029,13 +2029,13 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { pl.where_clauses = where_clauses; return pl; - case token.Dollar: + case .Dollar: tok := advance_token(p); type := parse_ident(p); end := type.end; specialization: ^ast.Expr; - if allow_token(p, token.Quo) { + if allow_token(p, .Quo) { specialization = parse_type(p); end = specialization.pos; } @@ -2049,19 +2049,19 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { pt.specialization = specialization; return pt; - case token.Typeid: + case .Typeid: tok := advance_token(p); ti := ast.new(ast.Typeid_Type, tok.pos, end_pos(tok)); ti.tok = tok.kind; ti.specialization = nil; return ti; - case token.Type_Of: + case .Type_Of: tok := advance_token(p); i := ast.new(ast.Implicit, tok.pos, end_pos(tok)); i.tok = tok; type: ^ast.Expr = parse_call_expr(p, i); - for p.curr_tok.kind == token.Period { + for p.curr_tok.kind == .Period { period := advance_token(p); field := parse_ident(p); @@ -2075,24 +2075,24 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { return type; - case token.Pointer: - tok := expect_token(p, token.Pointer); + case .Pointer: + tok := expect_token(p, .Pointer); elem := parse_type(p); ptr := ast.new(ast.Pointer_Type, tok.pos, elem.end); ptr.elem = elem; return ptr; - case token.Open_Bracket: - open := expect_token(p, token.Open_Bracket); + case .Open_Bracket: + open := expect_token(p, .Open_Bracket); count: ^ast.Expr; - if p.curr_tok.kind == token.Question { - tok := expect_token(p, token.Question); + if p.curr_tok.kind == .Question { + tok := expect_token(p, .Question); q := ast.new(ast.Unary_Expr, tok.pos, end_pos(tok)); q.op = tok; count = q; - } else if p.curr_tok.kind == token.Dynamic { - tok := expect_token(p, token.Dynamic); - close := expect_token(p, token.Close_Bracket); + } else if p.curr_tok.kind == .Dynamic { + tok := expect_token(p, .Dynamic); + close := expect_token(p, .Close_Bracket); elem := parse_type(p); da := ast.new(ast.Dynamic_Array_Type, open.pos, elem.end); da.open = open.pos; @@ -2101,12 +2101,12 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { da.elem = elem; return da; - } else if p.curr_tok.kind != token.Close_Bracket { + } else if p.curr_tok.kind != .Close_Bracket { p.expr_level += 1; count = parse_expr(p, false); p.expr_level -= 1; } - close := expect_token(p, token.Close_Bracket); + close := expect_token(p, .Close_Bracket); elem := parse_type(p); at := ast.new(ast.Array_Type, open.pos, elem.end); at.open = open.pos; @@ -2115,11 +2115,11 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { at.elem = elem; return at; - case token.Map: - tok := expect_token(p, token.Map); - expect_token(p, token.Open_Bracket); + case .Map: + tok := expect_token(p, .Map); + expect_token(p, .Open_Bracket); key := parse_type(p); - expect_token(p, token.Close_Bracket); + expect_token(p, .Close_Bracket); value := parse_type(p); mt := ast.new(ast.Map_Type, tok.pos, value.end); @@ -2128,8 +2128,8 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { mt.value = value; return mt; - case token.Struct: - tok := expect_token(p, token.Struct); + case .Struct: + tok := expect_token(p, .Struct); poly_params: ^ast.Field_List; align: ^ast.Expr; @@ -2138,21 +2138,21 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { fields: ^ast.Field_List; name_count: int; - if allow_token(p, token.Open_Paren) { + if allow_token(p, .Open_Paren) { param_count: int; - poly_params, param_count = parse_field_list(p, token.Close_Paren, ast.Field_Flags_Record_Poly_Params); + poly_params, param_count = parse_field_list(p, .Close_Paren, ast.Field_Flags_Record_Poly_Params); if param_count == 0 { error(p, poly_params.pos, "expected at least 1 polymorphic parameter"); poly_params = nil; } - expect_token_after(p, token.Close_Paren, "parameter list"); + expect_token_after(p, .Close_Paren, "parameter list"); check_poly_params_for_type(p, poly_params, tok); } prev_level := p.expr_level; p.expr_level = -1; - for allow_token(p, token.Hash) { - tag := expect_token_after(p, token.Ident, "#"); + for allow_token(p, .Hash) { + tag := expect_token_after(p, .Ident, "#"); switch tag.text { case "packed": if is_packed do error(p, tag.pos, "duplicate struct tag '#%s'", tag.text); @@ -2174,19 +2174,19 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { error(p, tok.pos, "'#raw_union' cannot also be '#packed"); } - where_token: token.Token; + where_token: tokenizer.Token; where_clauses: []^ast.Expr; - if (p.curr_tok.kind == token.Where) { - where_token = expect_token(p, token.Where); + if (p.curr_tok.kind == .Where) { + where_token = expect_token(p, .Where); prev_level := p.expr_level; p.expr_level = -1; where_clauses = parse_rhs_expr_list(p); p.expr_level = prev_level; } - expect_token(p, token.Open_Brace); - fields, name_count = parse_field_list(p, token.Close_Brace, ast.Field_Flags_Struct); - close := expect_token(p, token.Close_Brace); + expect_token(p, .Open_Brace); + fields, name_count = parse_field_list(p, .Close_Brace, ast.Field_Flags_Struct); + close := expect_token(p, .Close_Brace); st := ast.new(ast.Struct_Type, tok.pos, end_pos(close)); st.poly_params = poly_params; @@ -2199,26 +2199,26 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { st.where_clauses = where_clauses; return st; - case token.Union: - tok := expect_token(p, token.Union); + case .Union: + tok := expect_token(p, .Union); poly_params: ^ast.Field_List; align: ^ast.Expr; - if allow_token(p, token.Open_Paren) { + if allow_token(p, .Open_Paren) { param_count: int; - poly_params, param_count = parse_field_list(p, token.Close_Paren, ast.Field_Flags_Record_Poly_Params); + poly_params, param_count = parse_field_list(p, .Close_Paren, ast.Field_Flags_Record_Poly_Params); if param_count == 0 { error(p, poly_params.pos, "expected at least 1 polymorphic parameter"); poly_params = nil; } - expect_token_after(p, token.Close_Paren, "parameter list"); + expect_token_after(p, .Close_Paren, "parameter list"); check_poly_params_for_type(p, poly_params, tok); } prev_level := p.expr_level; p.expr_level = -1; - for allow_token(p, token.Hash) { - tag := expect_token_after(p, token.Ident, "#"); + for allow_token(p, .Hash) { + tag := expect_token_after(p, .Ident, "#"); switch tag.text { case "align": if align != nil do error(p, tag.pos, "duplicate union tag '#%s'", tag.text); @@ -2229,10 +2229,10 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { } p.expr_level = prev_level; - where_token: token.Token; + where_token: tokenizer.Token; where_clauses: []^ast.Expr; - if (p.curr_tok.kind == token.Where) { - where_token = expect_token(p, token.Where); + if (p.curr_tok.kind == .Where) { + where_token = expect_token(p, .Where); prev_level := p.expr_level; p.expr_level = -1; where_clauses = parse_rhs_expr_list(p); @@ -2241,19 +2241,19 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { variants: [dynamic]^ast.Expr; - expect_token_after(p, token.Open_Brace, "union"); + expect_token_after(p, .Open_Brace, "union"); - for p.curr_tok.kind != token.Close_Brace && p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .Close_Brace && p.curr_tok.kind != .EOF { type := parse_type(p); if _, ok := type.derived.(ast.Bad_Expr); !ok { append(&variants, type); } - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); ut := ast.new(ast.Union_Type, tok.pos, end_pos(close)); ut.poly_params = poly_params; @@ -2264,15 +2264,15 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { return ut; - case token.Enum: - tok := expect_token(p, token.Enum); + case .Enum: + tok := expect_token(p, .Enum); base_type: ^ast.Expr; - if p.curr_tok.kind != token.Open_Brace { + if p.curr_tok.kind != .Open_Brace { base_type = parse_type(p); } - open := expect_token(p, token.Open_Brace); + open := expect_token(p, .Open_Brace); fields := parse_elem_list(p); - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); et := ast.new(ast.Enum_Type, tok.pos, end_pos(close)); et.base_type = base_type; @@ -2281,8 +2281,8 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { et.close = close.pos; return et; - case token.Bit_Field: - tok := expect_token(p, token.Bit_Field); + case .Bit_Field: + tok := expect_token(p, .Bit_Field); fields: [dynamic]^ast.Field_Value; align: ^ast.Expr; @@ -2290,8 +2290,8 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { prev_level := p.expr_level; p.expr_level = -1; - for allow_token(p, token.Hash) { - tag := expect_token_after(p, token.Ident, "#"); + for allow_token(p, .Hash) { + tag := expect_token_after(p, .Ident, "#"); switch tag.text { case "align": if align != nil { @@ -2305,11 +2305,11 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { p.expr_level = prev_level; - open := expect_token_after(p, token.Open_Brace, "bit_field"); + open := expect_token_after(p, .Open_Brace, "bit_field"); - for p.curr_tok.kind != token.Close_Brace && p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .Close_Brace && p.curr_tok.kind != .EOF { name := parse_ident(p); - colon := expect_token(p, token.Colon); + colon := expect_token(p, .Colon); value := parse_expr(p, true); fv := ast.new(ast.Field_Value, name.pos, value.end); @@ -2318,12 +2318,12 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { fv.value = value; append(&fields, fv); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - close := expect_token(p, token.Close_Brace); + close := expect_token(p, .Close_Brace); bft := ast.new(ast.Bit_Field_Type, tok.pos, end_pos(close)); bft.tok_pos = tok.pos; @@ -2334,9 +2334,9 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { return bft; - case token.Bit_Set: - tok := expect_token(p, token.Bit_Set); - open := expect_token(p, token.Open_Bracket); + case .Bit_Set: + tok := expect_token(p, .Bit_Set); + open := expect_token(p, .Open_Bracket); elem, underlying: ^ast.Expr; prev_allow_range := p.allow_range; @@ -2344,12 +2344,12 @@ parse_operand :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { elem = parse_expr(p, false); p.allow_range = prev_allow_range; - if allow_token(p, token.Semicolon) { + if allow_token(p, .Semicolon) { underlying = parse_type(p); } - close := expect_token(p, token.Close_Bracket); + close := expect_token(p, .Close_Bracket); bst := ast.new(ast.Bit_Set_Type, tok.pos, end_pos(close)); bst.tok_pos = tok.pos; @@ -2388,7 +2388,7 @@ is_literal_type :: proc(expr: ^ast.Expr) -> bool { } parse_value :: proc(p: ^Parser) -> ^ast.Expr { - if p.curr_tok.kind == token.Open_Brace { + if p.curr_tok.kind == .Open_Brace { return parse_literal_value(p, nil); } return parse_expr(p, false); @@ -2397,10 +2397,10 @@ parse_value :: proc(p: ^Parser) -> ^ast.Expr { parse_elem_list :: proc(p: ^Parser) -> []^ast.Expr { elems: [dynamic]^ast.Expr; - for p.curr_tok.kind != token.Close_Brace && p.curr_tok.kind != token.EOF { + for p.curr_tok.kind != .Close_Brace && p.curr_tok.kind != .EOF { elem := parse_value(p); - if p.curr_tok.kind == token.Eq { - eq := expect_token(p, token.Eq); + if p.curr_tok.kind == .Eq { + eq := expect_token(p, .Eq); value := parse_value(p); fv := ast.new(ast.Field_Value, elem.pos, value.end); @@ -2413,7 +2413,7 @@ parse_elem_list :: proc(p: ^Parser) -> []^ast.Expr { append(&elems, elem); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } @@ -2423,14 +2423,14 @@ parse_elem_list :: proc(p: ^Parser) -> []^ast.Expr { parse_literal_value :: proc(p: ^Parser, type: ^ast.Expr) -> ^ast.Comp_Lit { elems: []^ast.Expr; - open := expect_token(p, token.Open_Brace); + open := expect_token(p, .Open_Brace); p.expr_level += 1; - if p.curr_tok.kind != token.Close_Brace { + if p.curr_tok.kind != .Close_Brace { elems = parse_elem_list(p); } p.expr_level -= 1; - close := expect_token_after(p, token.Close_Brace, "compound literal"); + close := expect_token_after(p, .Close_Brace, "compound literal"); pos := type != nil ? type.pos : open.pos; lit := ast.new(ast.Comp_Lit, pos, end_pos(close)); @@ -2444,30 +2444,30 @@ parse_literal_value :: proc(p: ^Parser, type: ^ast.Expr) -> ^ast.Comp_Lit { parse_call_expr :: proc(p: ^Parser, operand: ^ast.Expr) -> ^ast.Call_Expr { args: [dynamic]^ast.Expr; - ellipsis: token.Token; + ellipsis: tokenizer.Token; p.expr_level += 1; - open := expect_token(p, token.Open_Paren); + open := expect_token(p, .Open_Paren); - for p.curr_tok.kind != token.Close_Paren && - p.curr_tok.kind != token.EOF && + for p.curr_tok.kind != .Close_Paren && + p.curr_tok.kind != .EOF && ellipsis.pos.line == 0 { - if p.curr_tok.kind == token.Comma { + if p.curr_tok.kind == .Comma { error(p, p.curr_tok.pos, "expected an expression not ,"); - } else if p.curr_tok.kind == token.Eq { + } else if p.curr_tok.kind == .Eq { error(p, p.curr_tok.pos, "expected an expression not ="); } prefix_ellipsis := false; - if p.curr_tok.kind == token.Ellipsis { + if p.curr_tok.kind == .Ellipsis { prefix_ellipsis = true; - ellipsis = expect_token(p, token.Ellipsis); + ellipsis = expect_token(p, .Ellipsis); } arg := parse_expr(p, false); - if p.curr_tok.kind == token.Eq { - eq := expect_token(p, token.Eq); + if p.curr_tok.kind == .Eq { + eq := expect_token(p, .Eq); if prefix_ellipsis { error(p, ellipsis.pos, "'..' must be applied to value rather than a field name"); @@ -2484,12 +2484,12 @@ parse_call_expr :: proc(p: ^Parser, operand: ^ast.Expr) -> ^ast.Call_Expr { append(&args, arg); - if !allow_token(p, token.Comma) { + if !allow_token(p, .Comma) { break; } } - close := expect_token_after(p, token.Close_Paren, "argument list"); + close := expect_token_after(p, .Close_Paren, "argument list"); p.expr_level -= 1; ce := ast.new(ast.Call_Expr, operand.pos, end_pos(close)); @@ -2520,23 +2520,23 @@ parse_atom_expr :: proc(p: ^Parser, value: ^ast.Expr, lhs: bool) -> (operand: ^a case: loop = false; - case token.Open_Paren: + case .Open_Paren: operand = parse_call_expr(p, operand); - case token.Open_Bracket: + case .Open_Bracket: prev_allow_range := p.allow_range; defer p.allow_range = prev_allow_range; p.allow_range = false; indicies: [2]^ast.Expr; - interval: token.Token; + interval: tokenizer.Token; is_slice_op := false; p.expr_level += 1; - open := expect_token(p, token.Open_Bracket); + open := expect_token(p, .Open_Bracket); switch p.curr_tok.kind { - case token.Colon, token.Ellipsis, token.Range_Half: + case .Colon, .Ellipsis, .Range_Half: // NOTE(bill): Do not err yet break; case: @@ -2544,18 +2544,18 @@ parse_atom_expr :: proc(p: ^Parser, value: ^ast.Expr, lhs: bool) -> (operand: ^a } switch p.curr_tok.kind { - case token.Ellipsis, token.Range_Half: + case .Ellipsis, .Range_Half: error(p, p.curr_tok.pos, "expected a colon, not a range"); fallthrough; - case token.Colon: + case .Colon: interval = advance_token(p); is_slice_op = true; - if (p.curr_tok.kind != token.Close_Bracket && p.curr_tok.kind != token.EOF) { + if (p.curr_tok.kind != .Close_Bracket && p.curr_tok.kind != .EOF) { indicies[1] = parse_expr(p, false); } } - close := expect_token(p, token.Close_Bracket); + close := expect_token(p, .Close_Bracket); p.expr_level -= 1; if is_slice_op { @@ -2579,10 +2579,10 @@ parse_atom_expr :: proc(p: ^Parser, value: ^ast.Expr, lhs: bool) -> (operand: ^a } - case token.Period: - tok := expect_token(p, token.Period); + case .Period: + tok := expect_token(p, .Period); switch p.curr_tok.kind { - case token.Ident: + case .Ident: field := parse_ident(p); sel := ast.new(ast.Selector_Expr, operand.pos, field.end); @@ -2591,10 +2591,10 @@ parse_atom_expr :: proc(p: ^Parser, value: ^ast.Expr, lhs: bool) -> (operand: ^a operand = sel; - case token.Open_Paren: - open := expect_token(p, token.Open_Paren); + case .Open_Paren: + open := expect_token(p, .Open_Paren); type := parse_type(p); - close := expect_token(p, token.Close_Paren); + close := expect_token(p, .Close_Paren); ta := ast.new(ast.Type_Assertion, operand.pos, end_pos(close)); ta.expr = operand; @@ -2610,15 +2610,15 @@ parse_atom_expr :: proc(p: ^Parser, value: ^ast.Expr, lhs: bool) -> (operand: ^a operand = ast.new(ast.Bad_Expr, operand.pos, end_pos(tok)); } - case token.Pointer: - op := expect_token(p, token.Pointer); + case .Pointer: + op := expect_token(p, .Pointer); deref := ast.new(ast.Deref_Expr, operand.pos, end_pos(op)); deref.expr = operand; deref.op = op; operand = deref; - case token.Open_Brace: + case .Open_Brace: if !is_lhs && is_literal_type(operand) && p.expr_level >= 0 { operand = parse_literal_value(p, operand); } else { @@ -2639,11 +2639,11 @@ parse_expr :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { } parse_unary_expr :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { switch p.curr_tok.kind { - case token.Transmute, token.Cast: + case .Transmute, .Cast: tok := advance_token(p); - open := expect_token(p, token.Open_Paren); + open := expect_token(p, .Open_Paren); type := parse_type(p); - close := expect_token(p, token.Close_Paren); + close := expect_token(p, .Close_Paren); expr := parse_unary_expr(p, lhs); tc := ast.new(ast.Type_Cast, tok.pos, expr.end); @@ -2654,7 +2654,7 @@ parse_unary_expr :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { tc.expr = expr; return tc; - case token.Auto_Cast: + case .Auto_Cast: op := advance_token(p); expr := parse_unary_expr(p, lhs); @@ -2663,9 +2663,9 @@ parse_unary_expr :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { ac.expr = expr; return ac; - case token.Add, token.Sub, - token.Not, token.Xor, - token.And: + case .Add, .Sub, + .Not, .Xor, + .And: op := advance_token(p); expr := parse_unary_expr(p, lhs); @@ -2674,7 +2674,7 @@ parse_unary_expr :: proc(p: ^Parser, lhs: bool) -> ^ast.Expr { ue.expr = expr; return ue; - case token.Period: + case .Period: op := advance_token(p); field := parse_ident(p); ise := ast.new(ast.Implicit_Selector_Expr, op.pos, field.end); @@ -2695,10 +2695,10 @@ parse_binary_expr :: proc(p: ^Parser, lhs: bool, prec_in: int) -> ^ast.Expr { } expect_operator(p); - if op.kind == token.Question { + if op.kind == .Question { cond := expr; x := parse_expr(p, lhs); - colon := expect_token(p, token.Colon); + colon := expect_token(p, .Colon); y := parse_expr(p, lhs); te := ast.new(ast.Ternary_Expr, expr.pos, end_pos(p.prev_tok)); te.cond = cond; @@ -2732,7 +2732,7 @@ parse_expr_list :: proc(p: ^Parser, lhs: bool) -> ([]^ast.Expr) { for { expr := parse_expr(p, lhs); append(&list, expr); - if p.curr_tok.kind != token.Comma || p.curr_tok.kind == token.EOF { + if p.curr_tok.kind != .Comma || p.curr_tok.kind == .EOF { break; } advance_token(p); @@ -2754,7 +2754,7 @@ parse_simple_stmt :: proc(p: ^Parser, flags: Stmt_Allow_Flags) -> ^ast.Stmt { lhs := parse_lhs_expr_list(p); op := p.curr_tok; switch { - case token.is_assignment_operator(op.kind): + case tokenizer.is_assignment_operator(op.kind): // if p.curr_proc == nil { // error(p, p.curr_tok.pos, "simple statements are not allowed at the file scope"); // return ast.new(ast.Bad_Stmt, start_tok.pos, end_pos(p.curr_tok)); @@ -2771,9 +2771,9 @@ parse_simple_stmt :: proc(p: ^Parser, flags: Stmt_Allow_Flags) -> ^ast.Stmt { stmt.rhs = rhs; return stmt; - case op.kind == token.In: + case op.kind == .In: if .In in flags { - allow_token(p, token.In); + allow_token(p, .In); prev_allow_range := p.allow_range; p.allow_range = true; expr := parse_expr(p, false); @@ -2788,11 +2788,11 @@ parse_simple_stmt :: proc(p: ^Parser, flags: Stmt_Allow_Flags) -> ^ast.Stmt { stmt.rhs = rhs; return stmt; } - case op.kind == token.Colon: - expect_token_after(p, token.Colon, "identifier list"); + case op.kind == .Colon: + expect_token_after(p, .Colon, "identifier list"); if .Label in flags && len(lhs) == 1 { switch p.curr_tok.kind { - case token.Open_Brace, token.If, token.For, token.Switch: + case .Open_Brace, .If, .For, .Switch: label := lhs[0]; stmt := parse_stmt(p); @@ -2827,9 +2827,9 @@ parse_value_decl :: proc(p: ^Parser, names: []^ast.Expr, docs: ^ast.Comment_Grou type := parse_type_or_ident(p); switch p.curr_tok.kind { - case token.Eq, token.Colon: + case .Eq, .Colon: sep := advance_token(p); - is_mutable = sep.kind != token.Colon; + is_mutable = sep.kind != .Colon; values = parse_rhs_expr_list(p); if len(values) > len(names) { @@ -2858,7 +2858,7 @@ parse_value_decl :: proc(p: ^Parser, names: []^ast.Expr, docs: ^ast.Comment_Grou if !is_mutable && len(values) > 0 { end = values[len(values)-1]; } - if p.curr_tok.kind == token.Close_Brace && + if p.curr_tok.kind == .Close_Brace && p.curr_tok.pos.line == p.prev_tok.pos.line { } else { @@ -2884,13 +2884,13 @@ parse_value_decl :: proc(p: ^Parser, names: []^ast.Expr, docs: ^ast.Comment_Grou parse_import_decl :: proc(p: ^Parser, kind := Import_Decl_Kind.Standard) -> ^ast.Import_Decl { docs := p.lead_comment; - tok := expect_token(p, token.Import); + tok := expect_token(p, .Import); - import_name: token.Token; + import_name: tokenizer.Token; is_using := kind != Import_Decl_Kind.Standard; switch p.curr_tok.kind { - case token.Ident: + case .Ident: import_name = advance_token(p); case: import_name.pos = p.curr_tok.pos; @@ -2900,7 +2900,7 @@ parse_import_decl :: proc(p: ^Parser, kind := Import_Decl_Kind.Standard) -> ^ast error(p, import_name.pos, "illegal import name: '_'"); } - path := expect_token_after(p, token.String, "import"); + path := expect_token_after(p, .String, "import"); decl := ast.new(ast.Import_Decl, tok.pos, end_pos(path)); decl.docs = docs; diff --git a/core/odin/token/token.odin b/core/odin/token/token.odin deleted file mode 100644 index 737ff3586..000000000 --- a/core/odin/token/token.odin +++ /dev/null @@ -1,333 +0,0 @@ -package odin_token - -import "core:strings" - -Token :: struct { - kind: Kind, - text: string, - pos: Pos, -} - -Pos :: struct { - file: string, - offset: int, // starting at 0 - line: int, // starting at 1 - column: int, // starting at 1 -} - -pos_compare :: proc(lhs, rhs: Pos) -> int { - if lhs.offset != rhs.offset { - return (lhs.offset < rhs.offset) ? -1 : +1; - } - if lhs.line != rhs.line { - return (lhs.line < rhs.line) ? -1 : +1; - } - if lhs.column != rhs.column { - return (lhs.column < rhs.column) ? -1 : +1; - } - return strings.compare(lhs.file, rhs.file); -} - -using Kind :: enum u32 { - Invalid, - EOF, - Comment, - - B_Literal_Begin, - Ident, - Integer, - Float, - Imag, - Rune, - String, - B_Literal_End, - - B_Operator_Begin, - Eq, - Not, - Hash, - At, - Dollar, - Pointer, - Question, - Add, - Sub, - Mul, - Quo, - Mod, - Mod_Mod, - And, - Or, - Xor, - And_Not, - Shl, - Shr, - - Cmp_And, - Cmp_Or, - - B_Assign_Op_Begin, - Add_Eq, - Sub_Eq, - Mul_Eq, - Quo_Eq, - Mod_Eq, - Mod_Mod_Eq, - And_Eq, - Or_Eq, - Xor_Eq, - And_Not_Eq, - Shl_Eq, - Shr_Eq, - Cmp_And_Eq, - Cmp_Or_Eq, - B_Assign_Op_End, - - Arrow_Right, - Arrow_Left, - Double_Arrow_Right, - Undef, - - B_Comparison_Begin, - Cmp_Eq, - Not_Eq, - Lt, - Gt, - Lt_Eq, - Gt_Eq, - B_Comparison_End, - - Open_Paren, - Close_Paren, - Open_Bracket, - Close_Bracket, - Open_Brace, - Close_Brace, - Colon, - Semicolon, - Period, - Comma, - Ellipsis, - Range_Half, - Back_Slash, - B_Operator_End, - - B_Keyword_Begin, - Import, - Foreign, - Package, - Typeid, - When, - Where, - If, - Else, - For, - Switch, - In, - Notin, - Do, - Case, - Break, - Continue, - Fallthrough, - Defer, - Return, - Proc, - Macro, - Struct, - Union, - Enum, - Bit_Field, - Bit_Set, - Map, - Dynamic, - Auto_Cast, - Cast, - Transmute, - Distinct, - Opaque, - Using, - Inline, - No_Inline, - Context, - Size_Of, - Align_Of, - Offset_Of, - Type_Of, - Const, - B_Keyword_End, - - COUNT, - - B_Custom_Keyword_Begin = COUNT+1, - // ... Custom keywords -}; - -tokens := [Kind.COUNT]string { - "Invalid", - "EOF", - "Comment", - - "", - "identifier", - "integer", - "float", - "imaginary", - "rune", - "string", - "", - - "", - "=", - "!", - "#", - "@", - "$", - "^", - "?", - "+", - "-", - "*", - "/", - "%", - "%%", - "&", - "|", - "~", - "&~", - "<<", - ">>", - - "&&", - "||", - - "", - "+=", - "-=", - "*=", - "/=", - "%=", - "%%=", - "&=", - "|=", - "~=", - "&~=", - "<<=", - ">>=", - "&&=", - "||=", - "", - - "->", - "<-", - "=>", - "---", - - "", - "==", - "!=", - "<", - ">", - "<=", - ">=", - "", - - "(", - ")", - "[", - "]", - "{", - "}", - ":", - ";", - ".", - ",", - "..", - "..<", - "\\", - "", - - "", - "import", - "foreign", - "package", - "typeid", - "when", - "where", - "if", - "else", - "for", - "switch", - "in", - "notin", - "do", - "case", - "break", - "continue", - "fallthrough", - "defer", - "return", - "proc", - "macro", - "struct", - "union", - "enum", - "bit_field", - "bit_set", - "map", - "dynamic", - "auto_cast", - "cast", - "transmute", - "distinct", - "opaque", - "using", - "inline", - "no_inline", - "context", - "size_of", - "align_of", - "offset_of", - "type_of", - "const", - "", -}; - -custom_keyword_tokens: []string; - -to_string :: proc(kind: Kind) -> string { - if Invalid <= kind && kind < COUNT { - return tokens[kind]; - } - if B_Custom_Keyword_Begin < kind { - n := int(u16(kind)-u16(B_Custom_Keyword_Begin)); - if n < len(custom_keyword_tokens) { - return custom_keyword_tokens[n]; - } - } - - return "Invalid"; -} - -is_literal :: proc(kind: Kind) -> bool { return B_Literal_Begin < kind && kind < B_Literal_End; } -is_operator :: proc(kind: Kind) -> bool { - switch kind { - case B_Operator_Begin..B_Operator_End: - return true; - case In, Notin: - return true; - } - return false; -} -is_assignment_operator :: proc(kind: Kind) -> bool { - return B_Assign_Op_Begin < kind && kind < B_Assign_Op_End || kind == Eq; -} -is_keyword :: proc(kind: Kind) -> bool { - switch { - case B_Keyword_Begin < kind && kind < B_Keyword_End: - return true; - case B_Custom_Keyword_Begin < kind: - return true; - } - return false; -} diff --git a/core/odin/tokenizer/tokenizer.odin b/core/odin/tokenizer/tokenizer.odin index 98aef62b0..26aab7c1c 100644 --- a/core/odin/tokenizer/tokenizer.odin +++ b/core/odin/tokenizer/tokenizer.odin @@ -1,10 +1,9 @@ package odin_tokenizer import "core:fmt" -import "core:odin/token" import "core:unicode/utf8" -Error_Handler :: #type proc(pos: token.Pos, fmt: string, args: ..any); +Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any); Tokenizer :: struct { // Immutable data @@ -41,11 +40,11 @@ init :: proc(t: ^Tokenizer, src: []byte, path: string, err: Error_Handler = defa } @(private) -offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> token.Pos { +offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> Pos { line := t.line_count; column := offset - t.line_offset + 1; - return token.Pos { + return Pos { file = t.path, offset = offset, line = line, @@ -53,7 +52,7 @@ offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> token.Pos { }; } -default_error_handler :: proc(pos: token.Pos, msg: string, args: ..any) { +default_error_handler :: proc(pos: Pos, msg: string, args: ..any) { fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column); fmt.eprintf(msg, ..args); fmt.eprintf("\n"); @@ -322,15 +321,15 @@ scan_rune :: proc(t: ^Tokenizer) -> string { return string(t.src[offset : t.offset]); } -scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (token.Kind, string) { +scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) { scan_mantissa :: proc(t: ^Tokenizer, base: int) { for digit_val(t.ch) < base || t.ch == '_' { advance_rune(t); } } - scan_exponent :: proc(t: ^Tokenizer, kind: ^token.Kind) { + scan_exponent :: proc(t: ^Tokenizer, kind: ^Token_Kind) { if t.ch == 'e' || t.ch == 'E' { - kind^ = token.Float; + kind^ = .Float; advance_rune(t); if t.ch == '-' || t.ch == '+' { advance_rune(t); @@ -345,16 +344,16 @@ scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (token.Kind, str // NOTE(bill): This needs to be here for sanity's sake switch t.ch { case 'i', 'j', 'k': - kind^ = token.Imag; + kind^ = .Imag; advance_rune(t); } } - scan_fraction :: proc(t: ^Tokenizer, kind: ^token.Kind) -> (early_exit: bool) { + scan_fraction :: proc(t: ^Tokenizer, kind: ^Token_Kind) -> (early_exit: bool) { if t.ch == '.' && peek_byte(t) == '.' { return true; } if t.ch == '.' { - kind^ = token.Float; + kind^ = .Float; advance_rune(t); scan_mantissa(t, 10); } @@ -363,22 +362,22 @@ scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (token.Kind, str offset := t.offset; - kind := token.Integer; + kind := Token_Kind.Integer; seen_point := seen_decimal_point; if seen_point { offset -= 1; - kind = token.Float; + kind = .Float; scan_mantissa(t, 10); scan_exponent(t, &kind); } else { if t.ch == '0' { - int_base :: inline proc(t: ^Tokenizer, kind: ^token.Kind, base: int, msg: string) { + int_base :: inline proc(t: ^Tokenizer, kind: ^Token_Kind, base: int, msg: string) { prev := t.offset; advance_rune(t); scan_mantissa(t, base); if t.offset - prev <= 1 { - kind^ = token.Invalid; + kind^ = .Invalid; error(t, t.offset, msg); } } @@ -395,7 +394,7 @@ scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (token.Kind, str advance_rune(t); scan_mantissa(t, 16); if t.offset - prev <= 1 { - kind = token.Invalid; + kind = .Invalid; error(t, t.offset, "illegal hexadecimal floating-point number"); } else { sub := t.src[prev+1 : t.offset]; @@ -440,15 +439,15 @@ scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (token.Kind, str } -scan :: proc(t: ^Tokenizer) -> token.Token { - switch2 :: proc(t: ^Tokenizer, tok0, tok1: token.Kind) -> token.Kind { +scan :: proc(t: ^Tokenizer) -> Token { + switch2 :: proc(t: ^Tokenizer, tok0, tok1: Token_Kind) -> Token_Kind { if t.ch == '=' { advance_rune(t); return tok1; } return tok0; } - switch3 :: proc(t: ^Tokenizer, tok0, tok1: token.Kind, ch2: rune, tok2: token.Kind) -> token.Kind { + switch3 :: proc(t: ^Tokenizer, tok0, tok1: Token_Kind, ch2: rune, tok2: Token_Kind) -> Token_Kind { if t.ch == '=' { advance_rune(t); return tok1; @@ -459,7 +458,7 @@ scan :: proc(t: ^Tokenizer) -> token.Token { } return tok0; } - switch4 :: proc(t: ^Tokenizer, tok0, tok1: token.Kind, ch2: rune, tok2, tok3: token.Kind) -> token.Kind { + switch4 :: proc(t: ^Tokenizer, tok0, tok1: Token_Kind, ch2: rune, tok2, tok3: Token_Kind) -> Token_Kind { if t.ch == '=' { advance_rune(t); return tok1; @@ -480,25 +479,25 @@ scan :: proc(t: ^Tokenizer) -> token.Token { offset := t.offset; - kind: token.Kind; + kind: Token_Kind; lit: string; pos := offset_to_pos(t, offset); switch ch := t.ch; true { case is_letter(ch): lit = scan_identifier(t); - kind = token.Ident; + kind = .Ident; check_keyword: if len(lit) > 1 { // TODO(bill): Maybe have a hash table lookup rather than this linear search - for i in token.B_Keyword_Begin .. token.B_Keyword_End { - if lit == token.tokens[i] { - kind = token.Kind(i); + for i in Token_Kind.B_Keyword_Begin .. Token_Kind.B_Keyword_End { + if lit == tokens[i] { + kind = Token_Kind(i); break check_keyword; } } - for keyword, i in token.custom_keyword_tokens { + for keyword, i in custom_keyword_tokens { if lit == keyword { - kind = token.Kind(i+1)+token.B_Custom_Keyword_Begin; + kind = Token_Kind(i+1) + .B_Custom_Keyword_Begin; break check_keyword; } } @@ -509,115 +508,115 @@ scan :: proc(t: ^Tokenizer) -> token.Token { advance_rune(t); switch ch { case -1: - kind = token.EOF; + kind = .EOF; case '"': - kind = token.String; + kind = .String; lit = scan_string(t); case '\'': - kind = token.Rune; + kind = .Rune; lit = scan_rune(t); case '`': - kind = token.String; + kind = .String; lit = scan_raw_string(t); case '=': if t.ch == '>' { advance_rune(t); - kind = token.Double_Arrow_Right; + kind = .Double_Arrow_Right; } else { - kind = switch2(t, token.Eq, token.Cmp_Eq); + kind = switch2(t, .Eq, .Cmp_Eq); } - case '!': kind = switch2(t, token.Not, token.Not_Eq); + case '!': kind = switch2(t, .Not, .Not_Eq); case '#': - kind = token.Hash; + kind = .Hash; if t.ch == '!' { - kind = token.Comment; + kind = .Comment; lit = scan_comment(t); } - case '?': kind = token.Question; - case '@': kind = token.At; - case '$': kind = token.Dollar; - case '^': kind = token.Pointer; - case '+': kind = switch2(t, token.Add, token.Add_Eq); + case '?': kind = .Question; + case '@': kind = .At; + case '$': kind = .Dollar; + case '^': kind = .Pointer; + case '+': kind = switch2(t, .Add, .Add_Eq); case '-': if t.ch == '>' { advance_rune(t); - kind = token.Arrow_Right; + kind = .Arrow_Right; } else if t.ch == '-' && peek_byte(t) == '-' { advance_rune(t); advance_rune(t); - kind = token.Undef; + kind = .Undef; } else { - kind = switch2(t, token.Sub, token.Sub_Eq); + kind = switch2(t, .Sub, .Sub_Eq); } - case '*': kind = switch2(t, token.Mul, token.Mul_Eq); + case '*': kind = switch2(t, .Mul, .Mul_Eq); case '/': if t.ch == '/' || t.ch == '*' { - kind = token.Comment; + kind = .Comment; lit = scan_comment(t); } else { - kind = switch2(t, token.Quo, token.Quo_Eq); + kind = switch2(t, .Quo, .Quo_Eq); } - case '%': kind = switch4(t, token.Mod, token.Mod_Eq, '%', token.Mod_Mod, token.Mod_Mod_Eq); + case '%': kind = switch4(t, .Mod, .Mod_Eq, '%', .Mod_Mod, .Mod_Mod_Eq); case '&': if t.ch == '~' { advance_rune(t); - kind = switch2(t, token.And_Not, token.And_Not_Eq); + kind = switch2(t, .And_Not, .And_Not_Eq); } else { - kind = switch3(t, token.And, token.And_Eq, '&', token.Cmp_And); + kind = switch3(t, .And, .And_Eq, '&', .Cmp_And); } - case '|': kind = switch3(t, token.Or, token.Or_Eq, '|', token.Cmp_Or); - case '~': kind = token.Xor; + case '|': kind = switch3(t, .Or, .Or_Eq, '|', .Cmp_Or); + case '~': kind = .Xor; case '<': if t.ch == '-' { advance_rune(t); - kind = token.Arrow_Left; + kind = .Arrow_Left; } else { - kind = switch4(t, token.Lt, token.Lt_Eq, '<', token.Shl, token.Shl_Eq); + kind = switch4(t, .Lt, .Lt_Eq, '<', .Shl, .Shl_Eq); } - case '>': kind = switch4(t, token.Gt, token.Gt_Eq, '>', token.Shr,token.Shr_Eq); + case '>': kind = switch4(t, .Gt, .Gt_Eq, '>', .Shr,.Shr_Eq); - case '≠': kind = token.Not_Eq; - case '≤': kind = token.Lt_Eq; - case '≥': kind = token.Gt_Eq; - case '∈': kind = token.In; - case '∉': kind = token.Notin; + case '≠': kind = .Not_Eq; + case '≤': kind = .Lt_Eq; + case '≥': kind = .Gt_Eq; + case '∈': kind = .In; + case '∉': kind = .Notin; case '.': if '0' <= t.ch && t.ch <= '9' { kind, lit = scan_number(t, true); } else { - kind = token.Period; + kind = .Period; if t.ch == '.' { advance_rune(t); - kind = token.Ellipsis; + kind = .Ellipsis; if t.ch == '<' { advance_rune(t); - kind = token.Range_Half; + kind = .Range_Half; } } } - case ':': kind = token.Colon; - case ',': kind = token.Comma; - case ';': kind = token.Semicolon; - case '(': kind = token.Open_Paren; - case ')': kind = token.Close_Paren; - case '[': kind = token.Open_Bracket; - case ']': kind = token.Close_Bracket; - case '{': kind = token.Open_Brace; - case '}': kind = token.Close_Brace; - - case '\\': kind = token.Back_Slash; + case ':': kind = .Colon; + case ',': kind = .Comma; + case ';': kind = .Semicolon; + case '(': kind = .Open_Paren; + case ')': kind = .Close_Paren; + case '[': kind = .Open_Bracket; + case ']': kind = .Close_Bracket; + case '{': kind = .Open_Brace; + case '}': kind = .Close_Brace; + + case '\\': kind = .Back_Slash; case: if ch != utf8.RUNE_BOM { error(t, t.offset, "illegal character '%r': %d", ch, ch); } - kind = token.Invalid; + kind = .Invalid; } } if lit == "" { lit = string(t.src[offset : t.offset]); } - return token.Token{kind, lit, pos}; + return Token{kind, lit, pos}; } diff --git a/core/runtime/core.odin b/core/runtime/core.odin index c401dbdf3..729932781 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -814,7 +814,7 @@ __get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> Map_Header { key: Map_Key, next: int, value: V, - } + }; _, is_string := type_info_base(type_info_of(K)).variant.(Type_Info_String); header.is_key_string = is_string; diff --git a/core/strconv/generic_float.odin b/core/strconv/generic_float.odin index f0e43bcc9..59eaa20d5 100644 --- a/core/strconv/generic_float.odin +++ b/core/strconv/generic_float.odin @@ -105,7 +105,7 @@ format_digits :: proc(buf: []byte, shortest: bool, neg: bool, digs: Decimal_Slic Buffer :: struct { b: []byte, n: int, - } + }; to_bytes :: proc(b: Buffer) -> []byte do return b.b[:b.n]; add_bytes :: proc(buf: ^Buffer, bytes: ..byte) { diff --git a/src/parser.cpp b/src/parser.cpp index 29254d7e8..e17d19787 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1344,7 +1344,8 @@ bool is_semicolon_optional_for_node(AstFile *f, Ast *s) { case Ast_UnionType: case Ast_EnumType: case Ast_BitFieldType: - return true; + // Require semicolon within a procedure body + return f->curr_proc == false; case Ast_ProcLit: return true; -- cgit v1.2.3 From 1b8c3ca22a28b8ca36175d3e20a3993cba315dae Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Oct 2019 20:28:45 +0100 Subject: Fix typos and make demo work with -vet --- core/fmt/fmt.odin | 4 ++-- core/strings/strings.odin | 6 +++--- examples/demo/demo.odin | 1 - src/check_type.cpp | 6 ++++-- src/parser.cpp | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 6903b9bdb..e6830e734 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -69,8 +69,8 @@ eprintf :: proc(fmt: string, args: ..any) -> int { return fprintf(context.stder @(deprecated="prefer eprint") print_err :: proc(args: ..any) -> int { return eprint(..args); } -@(deprecated="prefer eprintf") println_err :: proc(args: ..any) -> int { return eprintln(..args); } -@(deprecated="prefer eprintln") printf_err :: proc(fmt: string, args: ..any) -> int { return eprintf(fmt, ..args); } +@(deprecated="prefer eprintf") printf_err :: proc(fmt: string, args: ..any) -> int { return eprintf(fmt, ..args); } +@(deprecated="prefer eprintln") println_err :: proc(args: ..any) -> int { return eprintln(..args); } // aprint* procedures return a string that was allocated with the current context diff --git a/core/strings/strings.odin b/core/strings/strings.odin index 0c9ffa7d3..921e4f009 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -169,9 +169,9 @@ _split :: proc(s_, sep: string, sep_save, n_: int, allocator := context.allocato n = l; } - res := make([dynamic]string, n); + res := make([dynamic]string, n, allocator); for i := 0; i < n-1; i += 1 { - r, w := utf8.decode_rune_in_string(s); + _, w := utf8.decode_rune_in_string(s); res[i] = s[:w]; s = s[w:]; } @@ -185,7 +185,7 @@ _split :: proc(s_, sep: string, sep_save, n_: int, allocator := context.allocato n = count(s, sep) + 1; } - res := make([dynamic]string, n); + res := make([dynamic]string, n, allocator); n -= 1; diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index a5f3500f1..ff861b1de 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -4,7 +4,6 @@ import "core:fmt" import "core:mem" import "core:os" import "core:reflect" -import "core:strings" import "intrinsics" when os.OS == "windows" { diff --git a/src/check_type.cpp b/src/check_type.cpp index 9f5ed0ed3..65e0bc880 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2016,8 +2016,10 @@ Type *type_to_abi_compat_result_type(gbAllocator a, Type *original_type, ProcCal break; } } - } else if (build_context.ODIN_OS == "linux") { - + } else if (build_context.ODIN_OS == "linux" || build_context.ODIN_OS == "darwin") { + if (build_context.ODIN_ARCH == "amd64") { + + } } else { // IMPORTANT TODO(bill): figure out the ABI settings for Linux, OSX etc. for // their architectures diff --git a/src/parser.cpp b/src/parser.cpp index e17d19787..025a181ba 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1345,7 +1345,7 @@ bool is_semicolon_optional_for_node(AstFile *f, Ast *s) { case Ast_EnumType: case Ast_BitFieldType: // Require semicolon within a procedure body - return f->curr_proc == false; + return f->curr_proc == nullptr; case Ast_ProcLit: return true; -- cgit v1.2.3 From 2a6d9e8927ad1eb1e5f3a79fc9ed068a02cbfdfc Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Oct 2019 12:38:23 +0100 Subject: `#panic`; Minor change to demo.odin; Fix `#assert` bug at file scope --- core/odin/parser/parser.odin | 2 +- core/runtime/core.odin | 54 +++++++++++++++++++----------------- examples/demo/demo.odin | 20 +++++++------- src/check_expr.cpp | 22 +++++++++++++-- src/check_type.cpp | 37 ++++++++++++++++++++----- src/checker.cpp | 13 +++++++-- src/checker.hpp | 1 + src/main.cpp | 14 ++++------ src/parser.cpp | 38 +++---------------------- src/types.cpp | 66 ++++++++++++++++++++++++-------------------- 10 files changed, 145 insertions(+), 122 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index 213041924..fee9b7bdc 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -1091,7 +1091,7 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { case: error(p, stmt.pos, "#complete can only be applied to a switch statement"); } return stmt; - case "assert": + case "assert", "panic": bd := ast.new(ast.Basic_Directive, tok.pos, end_pos(tag)); bd.tok = tok; bd.name = name; diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 729932781..9ad5d3d8a 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -6,6 +6,7 @@ package runtime import "core:os" import "core:mem" import "core:log" +import "intrinsics" // Naming Conventions: // In general, Ada_Case for types and snake_case for values @@ -187,12 +188,13 @@ Typeid_Kind :: enum u8 { #assert(len(Typeid_Kind) < 32); Typeid_Bit_Field :: bit_field #align align_of(uintptr) { - index: 8*size_of(align_of(uintptr)) - 8, + index: 8*size_of(uintptr) - 8, kind: 5, // Typeid_Kind named: 1, special: 1, // signed, cstring, etc reserved: 1, } +#assert(size_of(Typeid_Bit_Field) == size_of(uintptr)); // NOTE(bill): only the ones that are needed (not all types) // This will be set by the compiler @@ -678,7 +680,7 @@ assert :: proc(condition: bool, message := "", loc := #caller_location) -> bool if p == nil { p = default_assertion_failure_proc; } - p("Runtime assertion", message, loc); + p("runtime assertion", message, loc); }(message, loc); } return condition; @@ -690,7 +692,7 @@ panic :: proc(message: string, loc := #caller_location) -> ! { if p == nil { p = default_assertion_failure_proc; } - p("Panic", message, loc); + p("panic", message, loc); } @builtin @@ -816,8 +818,7 @@ __get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> Map_Header { value: V, }; - _, is_string := type_info_base(type_info_of(K)).variant.(Type_Info_String); - header.is_key_string = is_string; + header.is_key_string = intrinsics.type_is_string(K); header.entry_size = int(size_of(Entry)); header.entry_align = int(align_of(Entry)); header.value_offset = uintptr(offset_of(Entry, value)); @@ -828,33 +829,34 @@ __get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> Map_Header { __get_map_key :: proc "contextless" (k: $K) -> Map_Key { key := k; map_key: Map_Key; - ti := type_info_base_without_enum(type_info_of(K)); - switch _ in ti.variant { - case Type_Info_Integer: - switch 8*size_of(key) { - case 8: map_key.hash = u64(( ^u8)(&key)^); - case 16: map_key.hash = u64(( ^u16)(&key)^); - case 32: map_key.hash = u64(( ^u32)(&key)^); - case 64: map_key.hash = u64(( ^u64)(&key)^); - case: panic("Unhandled integer size"); - } - case Type_Info_Rune: + + T :: intrinsics.type_core_type(K); + + when intrinsics.type_is_integer(T) { + sz :: 8*size_of(T); + when sz == 8 do map_key.hash = u64(( ^u8)(&key)^); + else when sz == 16 do map_key.hash = u64((^u16)(&key)^); + else when sz == 32 do map_key.hash = u64((^u32)(&key)^); + else when sz == 64 do map_key.hash = u64((^u64)(&key)^); + else do #assert(false, "Unhandled integer size"); + } else when intrinsics.type_is_rune(T) { map_key.hash = u64((^rune)(&key)^); - case Type_Info_Pointer: + } else when intrinsics.type_is_pointer(T) { map_key.hash = u64(uintptr((^rawptr)(&key)^)); - case Type_Info_Float: - switch 8*size_of(key) { - case 32: map_key.hash = u64((^u32)(&key)^); - case 64: map_key.hash = u64((^u64)(&key)^); - case: panic("Unhandled float size"); - } - case Type_Info_String: + } else when intrinsics.type_is_float(T) { + sz :: 8*size_of(T); + when sz == 32 do map_key.hash = u64((^u32)(&key)^); + else when sz == 64 do map_key.hash = u64((^u64)(&key)^); + else do #assert(false, "Unhandled float size"); + } else when intrinsics.type_is_string(T) { + #assert(T == string); str := (^string)(&key)^; map_key.hash = default_hash_string(str); map_key.str = str; - case: - panic("Unhandled map key type"); + } else { + #assert(false, "Unhandled map key type"); } + return map_key; } diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index ff861b1de..5ee9ded84 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -28,9 +28,9 @@ when os.OS == "windows" { Answers to common questions about Odin. */ -@(link_name="general_stuff") -general_stuff :: proc() { - fmt.println("# general_stuff"); +@(link_name="extra_general_stuff") +extra_general_stuff :: proc() { + fmt.println("# extra_general_stuff"); { // `do` for inline statements rather than block foo :: proc() do fmt.println("Foo!"); if false do foo(); @@ -209,8 +209,8 @@ union_type :: proc() { } } - Vector3 :: struct {x, y, z: f32}; - Quaternion :: struct {x, y, z, w: f32}; + Vector3 :: distinct [3]f32; + Quaternion :: distinct quaternion128; // More realistic examples { @@ -320,17 +320,17 @@ union_type :: proc() { /* Entity :: struct { - .. + ... derived: union{^Frog, ^Monster}, } Frog :: struct { using entity: Entity, - .. + ... } Monster :: struct { using entity: Entity, - .. + ... } new_entity :: proc(T: type) -> ^Entity { @@ -539,7 +539,7 @@ parametric_polymorphism :: proc() { fmt.println(r); r = 123; fmt.println(r); - r = Error.Foo0; + r = Error.Foo0; // r = .Foo0; is allow too, see implicit selector expressions below fmt.println(r); } @@ -1190,7 +1190,7 @@ where_clauses :: proc() { main :: proc() { when true { - general_stuff(); + extra_general_stuff(); union_type(); parametric_polymorphism(); threading_example(); diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 6a0d8221f..7c418c4f0 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -3071,7 +3071,6 @@ Entity *check_selector(CheckerContext *c, Operand *operand, Ast *node, Type *typ } if (selector->kind != Ast_Ident) { - // if (selector->kind != Ast_Ident) { error(selector, "Illegal selector kind: '%.*s'", LIT(ast_strings[selector->kind])); operand->mode = Addressing_Invalid; operand->expr = node; @@ -3544,6 +3543,25 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 operand->type = t_untyped_bool; operand->mode = Addressing_Constant; + } else if (name == "panic") { + if (ce->args.count != 1) { + error(call, "'#panic' expects 1 argument, got %td", ce->args.count); + return false; + } + if (!is_type_string(operand->type) && operand->mode != Addressing_Constant) { + gbString str = expr_to_string(ce->args[0]); + error(call, "'%s' is not a constant string", str); + gb_string_free(str); + return false; + } + error(call, "Compile time panic: %.*s", LIT(operand->value.value_string)); + if (c->proc_name != "") { + gbString str = type_to_string(c->curr_proc_sig); + error_line("\tCalled within '%.*s' :: %s\n", LIT(c->proc_name), str); + gb_string_free(str); + } + operand->type = t_invalid; + operand->mode = Addressing_NoValue; } else if (name == "defined") { if (ce->args.count != 1) { error(call, "'#defined' expects 1 argument, got %td", ce->args.count); @@ -6349,7 +6367,7 @@ ExprKind check_call_expr(CheckerContext *c, Operand *operand, Ast *call) { ce->proc->kind == Ast_BasicDirective) { ast_node(bd, BasicDirective, ce->proc); String name = bd->name; - if (name == "location" || name == "assert" || name == "defined" || name == "load") { + if (name == "location" || name == "assert" || name == "panic" || name == "defined" || name == "load") { operand->mode = Addressing_Builtin; operand->builtin_id = BuiltinProc_DIRECTIVE; operand->expr = ce->proc; diff --git a/src/check_type.cpp b/src/check_type.cpp index 329755ea5..4d107a9ac 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -1847,7 +1847,7 @@ Array systemv_distribute_struct_fields(Type *t) { } auto distributed = array_make(heap_allocator(), 0, distributed_cap); - + i64 sz = type_size_of(bt); switch (bt->kind) { case Type_Basic: switch (bt->Basic.kind){ @@ -1925,9 +1925,9 @@ Array systemv_distribute_struct_fields(Type *t) { array_add(&distributed, t_int); break; + case Type_Union: case Type_DynamicArray: case Type_Map: - case Type_Union: case Type_BitField: // TODO(bill): Ignore? // NOTE(bill, 2019-10-10): Odin specific, don't worry about C calling convention yet goto DEFAULT; @@ -1937,7 +1937,7 @@ Array systemv_distribute_struct_fields(Type *t) { case Type_SimdVector: // TODO(bill): Is this correct logic? default: DEFAULT:; - if (type_size_of(bt) > 0) { + if (sz > 0) { array_add(&distributed, bt); } break; @@ -1959,13 +1959,22 @@ Type *handle_single_distributed_type_parameter(Array const &types, bool if (types.count == 1) { if (offset) *offset = 1; + + i64 sz = type_size_of(types[0]); + if (is_type_float(types[0])) { return types[0]; - } else if (type_size_of(types[0]) == 8) { + } + switch (sz) { + case 0: + GB_PANIC("Zero sized type found!"); + case 1: + case 2: + case 4: + case 8: return types[0]; - } else { - return t_u64; } + return t_u64; } else if (types.count >= 2) { if (types[0] == t_f32 && types[1] == t_f32) { if (offset) *offset = 2; @@ -2050,7 +2059,7 @@ Type *handle_struct_system_v_amd64_abi_type(Type *t) { Type *final_type = nullptr; if (field_types.count == 0) { - return t; + final_type = t; } else if (field_types.count == 1) { final_type = field_types[0]; } else { @@ -2072,8 +2081,22 @@ Type *handle_struct_system_v_amd64_abi_type(Type *t) { variables[1] = alloc_entity_param(nullptr, empty_token, two_types[1], false, false); final_type = alloc_type_tuple(); final_type->Tuple.variables = variables; + if (t->kind == Type_Struct) { + // NOTE(bill): Make this packed + final_type->Tuple.is_packed = t->Struct.is_packed; + } } } + + + GB_ASSERT(final_type != nullptr); + i64 ftsz = type_size_of(final_type); + i64 otsz = type_size_of(original_type); + if (ftsz != otsz) { + // TODO(bill): Handle this case which will be caused by #packed most likely + GB_PANIC("Incorrectly handled case for handle_struct_system_v_amd64_abi_type, %lld vs %lld", ftsz, otsz); + } + return final_type; } } diff --git a/src/checker.cpp b/src/checker.cpp index f30273439..8ce9d8ec2 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -3367,8 +3367,9 @@ bool collect_file_decls(CheckerContext *ctx, Array const &decls) { if (es->expr->kind == Ast_CallExpr) { ast_node(ce, CallExpr, es->expr); if (ce->proc->kind == Ast_BasicDirective) { - Operand o = {}; - check_expr(ctx, &o, es->expr); + if (ctx->collect_delayed_decls) { + array_add(&ctx->scope->delayed_directives, es->expr); + } } } case_end; @@ -3525,12 +3526,18 @@ void check_import_entities(Checker *c) { for_array(i, pkg->files) { AstFile *f = pkg->files[i]; CheckerContext ctx = c->init_ctx; - add_curr_ast_file(&ctx, f); + for_array(j, f->scope->delayed_imports) { Ast *decl = f->scope->delayed_imports[j]; check_add_import_decl(&ctx, decl); } + } + for_array(i, pkg->files) { + AstFile *f = pkg->files[i]; + CheckerContext ctx = c->init_ctx; + add_curr_ast_file(&ctx, f); + for_array(j, f->scope->delayed_directives) { Ast *expr = f->scope->delayed_directives[j]; Operand o = {}; diff --git a/src/checker.hpp b/src/checker.hpp index 58cb01a82..f86acce5c 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -222,6 +222,7 @@ struct ForeignContext { typedef Array CheckerTypePath; typedef Array CheckerPolyPath; + // CheckerInfo stores all the symbol information for a type-checked program struct CheckerInfo { Map untyped; // Key: Ast * | Expression -> ExprInfo diff --git a/src/main.cpp b/src/main.cpp index b294e7f96..78859e04e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -445,7 +445,7 @@ bool parse_build_flags(Array args) { path = substring(path, 0, string_extension_position(path)); } #endif - build_context.out_filepath = path; + build_context.out_filepath = path_to_full_path(heap_allocator(), path); } else { gb_printf_err("Invalid -out path, got %.*s\n", LIT(path)); bad_flags = true; @@ -624,7 +624,7 @@ bool parse_build_flags(Array args) { break; } - if (str == "dll") { + if (str == "dll" || str == "shared") { build_context.is_dll = true; } else if (str == "exe") { build_context.is_dll = false; @@ -1112,7 +1112,7 @@ int main(int arg_count, char **arg_ptr) { if (0) { #ifdef GB_SYSTEM_UNIX } else if (selected_target_metrics->metrics == &target_essence_amd64) { - system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s", + system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s", LIT(output_base), LIT(output_base), LIT(build_context.link_flags)); #endif } else { @@ -1239,10 +1239,7 @@ int main(int arg_count, char **arg_ptr) { // This allows you to specify '-f' in a #foreign_system_library, // without having to implement any new syntax specifically for MacOS. #if defined(GB_SYSTEM_OSX) - if (lib.len > 2 && lib[0] == '-' && lib[1] == 'f') { - // framework thingie - lib_str = gb_string_append_fmt(lib_str, " -framework %.*s ", (int)(lib.len) - 2, lib.text + 2); - } else if (string_ends_with(lib, str_lit(".framework"))) { + if (string_ends_with(lib, str_lit(".framework"))) { // framework thingie String lib_name = lib; lib_name = remove_extension_from_path(lib_name); @@ -1251,8 +1248,7 @@ int main(int arg_count, char **arg_ptr) { // static libs, absolute full path relative to the file in which the lib was imported from lib_str = gb_string_append_fmt(lib_str, " %.*s ", LIT(lib)); } else if (string_ends_with(lib, str_lit(".dylib"))) { - // dynamic lib, relative path to executable - // lib_str = gb_string_append_fmt(lib_str, " -l:%s/%.*s ", cwd, LIT(lib)); + // dynamic lib lib_str = gb_string_append_fmt(lib_str, " -l%.*s ", LIT(lib)); } else { // dynamic or static system lib, just link regularly searching system library paths diff --git a/src/parser.cpp b/src/parser.cpp index 025a181ba..f69efc0ce 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -3970,34 +3970,6 @@ Ast *parse_stmt(AstFile *f) { return s; } - // case Token_static: { - // CommentGroup *docs = f->lead_comment; - // Token token = expect_token(f, Token_static); - - // Ast *decl = nullptr; - // Array list = parse_lhs_expr_list(f); - // if (list.count == 0) { - // syntax_error(token, "Illegal use of 'static' statement"); - // expect_semicolon(f, nullptr); - // return ast_bad_stmt(f, token, f->curr_token); - // } - - // expect_token_after(f, Token_Colon, "identifier list"); - // decl = parse_value_decl(f, list, docs); - - // if (decl != nullptr && decl->kind == Ast_ValueDecl) { - // if (decl->ValueDecl.is_mutable) { - // decl->ValueDecl.is_static = true; - // } else { - // error(token, "'static' may only be currently used with variable declaration"); - // } - // return decl; - // } - - // syntax_error(token, "Illegal use of 'static' statement"); - // return ast_bad_stmt(f, token, f->curr_token); - // } break; - case Token_using: { CommentGroup *docs = f->lead_comment; Token token = expect_token(f, Token_using); @@ -4071,12 +4043,10 @@ Ast *parse_stmt(AstFile *f) { } else if (tag == "assert") { Ast *t = ast_basic_directive(f, hash_token, tag); return ast_expr_stmt(f, parse_call_expr(f, t)); - } /* else if (name.string == "no_deferred") { - s = parse_stmt(f); - s->stmt_state_flags |= StmtStateFlag_no_deferred; - } */ - - if (tag == "include") { + } else if (tag == "panic") { + Ast *t = ast_basic_directive(f, hash_token, tag); + return ast_expr_stmt(f, parse_call_expr(f, t)); + } else if (tag == "include") { syntax_error(token, "#include is not a valid import declaration kind. Did you mean 'import'?"); s = ast_bad_stmt(f, token, f->curr_token); } else { diff --git a/src/types.cpp b/src/types.cpp index d8a579510..8ad352f62 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -113,21 +113,22 @@ struct BasicType { struct TypeStruct { Array fields; Array tags; - Ast *node; - Scope * scope; - - Array offsets; - bool are_offsets_set; - bool are_offsets_being_processed; - bool is_packed; - bool is_raw_union; - bool is_polymorphic; - bool is_poly_specialized; + Array offsets; + Ast * node; + Scope * scope; + Type * polymorphic_params; // Type_Tuple Type * polymorphic_parent; - i64 custom_align; // NOTE(bill): Only used in structs at the moment + i64 custom_align; Entity * names; + + bool are_offsets_set; + bool are_offsets_being_processed; + bool is_packed; + bool is_raw_union; + bool is_polymorphic; + bool is_poly_specialized; }; struct TypeUnion { @@ -137,12 +138,11 @@ struct TypeUnion { i64 variant_block_size; i64 custom_align; i64 tag_size; + Type * polymorphic_params; // Type_Tuple + Type * polymorphic_parent; bool no_nil; - - bool is_polymorphic; - bool is_poly_specialized; - Type * polymorphic_params; // Type_Tuple - Type * polymorphic_parent; + bool is_polymorphic; + bool is_poly_specialized; }; #define TYPE_KINDS \ @@ -190,7 +190,9 @@ struct TypeUnion { TYPE_KIND(Tuple, struct { \ Array variables; /* Entity_Variable */ \ Array offsets; \ + bool are_offsets_being_processed; \ bool are_offsets_set; \ + bool is_packed; \ }) \ TYPE_KIND(Proc, struct { \ Ast *node; \ @@ -201,9 +203,8 @@ struct TypeUnion { i32 result_count; \ Array abi_compat_params; \ Type * abi_compat_result_type; \ - bool return_by_pointer; \ - bool variadic; \ i32 variadic_index; \ + bool variadic; \ bool require_results; \ bool c_vararg; \ bool is_polymorphic; \ @@ -211,6 +212,7 @@ struct TypeUnion { bool has_proc_default_values; \ bool has_named_results; \ bool diverging; /* no return */ \ + bool return_by_pointer; \ u64 tags; \ isize specialization_count; \ ProcCallingConvention calling_convention; \ @@ -1782,7 +1784,8 @@ bool are_types_identical(Type *x, Type *y) { case Type_Tuple: if (y->kind == Type_Tuple) { - if (x->Tuple.variables.count == y->Tuple.variables.count) { + if (x->Tuple.variables.count == y->Tuple.variables.count && + x->Tuple.is_packed == y->Tuple.is_packed) { for_array(i, x->Tuple.variables) { Entity *xe = x->Tuple.variables[i]; Entity *ye = y->Tuple.variables[i]; @@ -2231,19 +2234,22 @@ Selection lookup_field_with_selection(Type *type_, String field_name, bool is_ty if (type->Array.count <= 4) { // HACK(bill): Memory leak switch (type->Array.count) { - #define _ARRAY_FIELD_CASE(_length, _name) \ - case (_length): \ - if (field_name == _name) { \ + #define _ARRAY_FIELD_CASE_IF(_length, _name) \ + if (field_name == (_name)) { \ selection_add_index(&sel, (_length)-1); \ sel.entity = alloc_entity_array_elem(nullptr, make_token_ident(str_lit(_name)), type->Array.elem, (_length)-1); \ return sel; \ - } \ + } + #define _ARRAY_FIELD_CASE(_length, _name0, _name1) \ + case (_length): \ + _ARRAY_FIELD_CASE_IF(_length, _name0); \ + _ARRAY_FIELD_CASE_IF(_length, _name1); \ /*fallthrough*/ - _ARRAY_FIELD_CASE(4, "w"); - _ARRAY_FIELD_CASE(3, "z"); - _ARRAY_FIELD_CASE(2, "y"); - _ARRAY_FIELD_CASE(1, "x"); + _ARRAY_FIELD_CASE(4, "w", "a"); + _ARRAY_FIELD_CASE(3, "z", "b"); + _ARRAY_FIELD_CASE(2, "y", "g"); + _ARRAY_FIELD_CASE(1, "x", "r"); default: break; #undef _ARRAY_FIELD_CASE @@ -2590,9 +2596,9 @@ bool type_set_offsets(Type *t) { } } else if (is_type_tuple(t)) { if (!t->Tuple.are_offsets_set) { - t->Struct.are_offsets_being_processed = true; - t->Tuple.offsets = type_set_offsets_of(t->Tuple.variables, false, false); - t->Struct.are_offsets_being_processed = false; + t->Tuple.are_offsets_being_processed = true; + t->Tuple.offsets = type_set_offsets_of(t->Tuple.variables, t->Tuple.is_packed, false); + t->Tuple.are_offsets_being_processed = false; t->Tuple.are_offsets_set = true; return true; } -- cgit v1.2.3 From b53fe14c22fe2e7063979353735ca0aa5b0e2605 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Oct 2019 16:06:41 +0100 Subject: Change `error` to `syntax_error` in parser --- src/check_type.cpp | 1 - src/parser.cpp | 56 +++++++++++++++++++++++++++--------------------------- src/tokenizer.cpp | 9 +++++++++ 3 files changed, 37 insertions(+), 29 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/check_type.cpp b/src/check_type.cpp index f366c42d1..da6419877 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -1975,7 +1975,6 @@ Type *handle_single_distributed_type_parameter(Array const &types, bool default: return types[0]; } - return t_u64; } else if (types.count >= 2) { if (types[0] == t_f32 && types[1] == t_f32) { if (offset) *offset = 2; diff --git a/src/parser.cpp b/src/parser.cpp index f69efc0ce..7f866922a 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -427,7 +427,7 @@ void syntax_error(Ast *node, char *fmt, ...) { bool ast_node_expect(Ast *node, AstKind kind) { if (node->kind != kind) { - error(node, "Expected %.*s, got %.*s", LIT(ast_strings[kind]), LIT(ast_strings[node->kind])); + syntax_error(node, "Expected %.*s, got %.*s", LIT(ast_strings[kind]), LIT(ast_strings[node->kind])); return false; } return true; @@ -1662,7 +1662,7 @@ void check_polymorphic_params_for_type(AstFile *f, Ast *polymorphic_params, Toke for_array(i, field->Field.names) { Ast *name = field->Field.names[i]; if (name->kind == Ast_PolyType) { - error(name, "Polymorphic names are not needed for %.*s parameters", LIT(token.string)); + syntax_error(name, "Polymorphic names are not needed for %.*s parameters", LIT(token.string)); return; // TODO(bill): Err multiple times or just the once? } } @@ -4207,7 +4207,7 @@ void parser_add_package(Parser *p, AstPackage *pkg) { if (found) { GB_ASSERT(pkg->files.count > 0); AstFile *f = pkg->files[0]; - error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name)); + syntax_error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name)); GB_ASSERT((*found)->files.count > 0); TokenPos pos = (*found)->files[0]->package_token.pos; error_line("\tpreviously declared at %.*s(%td:%td)\n", LIT(pos.file), pos.line, pos.column); @@ -4283,22 +4283,22 @@ bool try_add_import_path(Parser *p, String const &path, String const &rel_path, switch (rd_err) { case ReadDirectory_InvalidPath: - error(pos, "Invalid path: %.*s", LIT(rel_path)); + syntax_error(pos, "Invalid path: %.*s", LIT(rel_path)); return false; case ReadDirectory_NotExists: - error(pos, "Path does not exist: %.*s", LIT(rel_path)); + syntax_error(pos, "Path does not exist: %.*s", LIT(rel_path)); return false; case ReadDirectory_Permission: - error(pos, "Unknown error whilst reading path %.*s", LIT(rel_path)); + syntax_error(pos, "Unknown error whilst reading path %.*s", LIT(rel_path)); return false; case ReadDirectory_NotDir: - error(pos, "Expected a directory for a package, got a file: %.*s", LIT(rel_path)); + syntax_error(pos, "Expected a directory for a package, got a file: %.*s", LIT(rel_path)); return false; case ReadDirectory_Empty: - error(pos, "Empty directory: %.*s", LIT(rel_path)); + syntax_error(pos, "Empty directory: %.*s", LIT(rel_path)); return false; case ReadDirectory_Unknown: - error(pos, "Unknown error whilst reading path %.*s", LIT(rel_path)); + syntax_error(pos, "Unknown error whilst reading path %.*s", LIT(rel_path)); return false; } @@ -4400,7 +4400,7 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, String file_str = {}; if (colon_pos == 0) { - error(node, "Expected a collection name"); + syntax_error(node, "Expected a collection name"); return false; } @@ -4415,11 +4415,11 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, if (has_windows_drive) { String sub_file_path = substring(file_str, 3, file_str.len); if (!is_import_path_valid(sub_file_path)) { - error(node, "Invalid import path: '%.*s'", LIT(file_str)); + syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str)); return false; } } else if (!is_import_path_valid(file_str)) { - error(node, "Invalid import path: '%.*s'", LIT(file_str)); + syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str)); return false; } @@ -4427,7 +4427,7 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, if (collection_name.len > 0) { if (collection_name == "system") { if (node->kind != Ast_ForeignImportDecl) { - error(node, "The library collection 'system' is restrict for 'foreign_library'"); + syntax_error(node, "The library collection 'system' is restrict for 'foreign_library'"); return false; } else { *path = file_str; @@ -4435,7 +4435,7 @@ bool determine_path_from_string(gbMutex *file_mutex, Ast *node, String base_dir, } } else if (!find_library_collection_path(collection_name, &base_dir)) { // NOTE(bill): It's a naughty name - error(node, "Unknown library collection: '%.*s'", LIT(collection_name)); + syntax_error(node, "Unknown library collection: '%.*s'", LIT(collection_name)); return false; } } else { @@ -4556,7 +4556,7 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Array array_add(&fl->fullpaths, fullpath); } if (fl->fullpaths.count == 0) { - error(decls[i], "No foreign paths found"); + syntax_error(decls[i], "No foreign paths found"); decls[i] = ast_bad_decl(f, fl->filepaths[0], fl->filepaths[fl->filepaths.count-1]); goto end; } @@ -4605,7 +4605,7 @@ bool parse_build_tag(Token token_for_pos, String s) { is_notted = true; p = substring(p, 1, p.len); if (p.len == 0) { - error(token_for_pos, "Expected a build platform after '!'"); + syntax_error(token_for_pos, "Expected a build platform after '!'"); break; } } @@ -4634,7 +4634,7 @@ bool parse_build_tag(Token token_for_pos, String s) { } } if (os == TargetOs_Invalid && arch == TargetArch_Invalid) { - error(token_for_pos, "Invalid build tag platform: %.*s", LIT(p)); + syntax_error(token_for_pos, "Invalid build tag platform: %.*s", LIT(p)); break; } } @@ -4680,11 +4680,11 @@ bool parse_file(Parser *p, AstFile *f) { Token package_name = expect_token_after(f, Token_Ident, "package"); if (package_name.kind == Token_Ident) { if (package_name.string == "_") { - error(package_name, "Invalid package name '_'"); + syntax_error(package_name, "Invalid package name '_'"); } else if (f->pkg->kind != Package_Runtime && package_name.string == "runtime") { - error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); + syntax_error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); } else if (is_package_name_reserved(package_name.string)) { - error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); + syntax_error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); } } f->package_name = package_name.string; @@ -4751,28 +4751,28 @@ ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_fil if (err != ParseFile_None) { if (err == ParseFile_EmptyFile) { if (fi->fullpath == p->init_fullpath) { - error(pos, "Initial file is empty - %.*s\n", LIT(p->init_fullpath)); + syntax_error(pos, "Initial file is empty - %.*s\n", LIT(p->init_fullpath)); gb_exit(1); } } else { switch (err) { case ParseFile_WrongExtension: - error(pos, "Failed to parse file: %.*s; invalid file extension: File must have the extension '.odin'", LIT(fi->name)); + syntax_error(pos, "Failed to parse file: %.*s; invalid file extension: File must have the extension '.odin'", LIT(fi->name)); break; case ParseFile_InvalidFile: - error(pos, "Failed to parse file: %.*s; invalid file or cannot be found", LIT(fi->name)); + syntax_error(pos, "Failed to parse file: %.*s; invalid file or cannot be found", LIT(fi->name)); break; case ParseFile_Permission: - error(pos, "Failed to parse file: %.*s; file permissions problem", LIT(fi->name)); + syntax_error(pos, "Failed to parse file: %.*s; file permissions problem", LIT(fi->name)); break; case ParseFile_NotFound: - error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi->name), LIT(fi->fullpath)); + syntax_error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi->name), LIT(fi->fullpath)); break; case ParseFile_InvalidToken: - error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi->name)); + syntax_error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi->name)); break; case ParseFile_EmptyFile: - error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name)); + syntax_error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name)); break; } @@ -4789,7 +4789,7 @@ ParseFileError process_imported_file(Parser *p, ImportedFile const &imported_fil if (pkg->name.len == 0) { pkg->name = file->package_name; } else if (file->tokens.count > 0 && pkg->name != file->package_name) { - error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(pkg->name), LIT(file->package_name)); + syntax_error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(pkg->name), LIT(file->package_name)); } p->total_line_count += file->tokenizer.line_count; diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index d5e04aa1e..4b0db7ac4 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -401,6 +401,15 @@ void syntax_error(Token token, char *fmt, ...) { va_end(va); } +void syntax_error(TokenPos pos, char *fmt, ...) { + va_list va; + va_start(va, fmt); + Token token = {}; + token.pos = pos; + syntax_error_va(token, fmt, va); + va_end(va); +} + void syntax_warning(Token token, char *fmt, ...) { va_list va; va_start(va, fmt); -- cgit v1.2.3 From 7fae890ef9eb72f3131914f9af2469f9bfb1e59e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 26 Oct 2019 14:06:29 +0100 Subject: Allow ranges for array-like compound literals --- core/odin/parser/parser.odin | 3 + src/check_expr.cpp | 229 ++++++++++++++++++++++++++++++++++--------- src/check_stmt.cpp | 84 +--------------- src/common.cpp | 3 + src/exact_value.cpp | 2 +- src/ir.cpp | 124 ++++++++++++++++++----- src/ir_print.cpp | 48 +++++++-- src/parser.cpp | 11 ++- src/parser.hpp | 1 + 9 files changed, 340 insertions(+), 165 deletions(-) (limited to 'src/parser.cpp') diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index fee9b7bdc..fb3a87e8b 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -2391,6 +2391,9 @@ parse_value :: proc(p: ^Parser) -> ^ast.Expr { if p.curr_tok.kind == .Open_Brace { return parse_literal_value(p, nil); } + prev_allow_range = p.allow_range; + defer p.allow_range = prev_allow_range; + p.allow_range = true; return parse_expr(p, false); } diff --git a/src/check_expr.cpp b/src/check_expr.cpp index ea9248089..f07ebe09c 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -6635,6 +6635,98 @@ bool ternary_compare_types(Type *x, Type *y) { } +bool check_range(CheckerContext *c, Ast *node, Operand *x, Operand *y, ExactValue *inline_for_depth_) { + if (!is_ast_range(node)) { + return false; + } + + ast_node(ie, BinaryExpr, node); + + check_expr(c, x, ie->left); + if (x->mode == Addressing_Invalid) { + return false; + } + check_expr(c, y, ie->right); + if (y->mode == Addressing_Invalid) { + return false; + } + + convert_to_typed(c, x, y->type); + if (x->mode == Addressing_Invalid) { + return false; + } + convert_to_typed(c, y, x->type); + if (y->mode == Addressing_Invalid) { + return false; + } + + convert_to_typed(c, x, default_type(y->type)); + if (x->mode == Addressing_Invalid) { + return false; + } + convert_to_typed(c, y, default_type(x->type)); + if (y->mode == Addressing_Invalid) { + return false; + } + + if (!are_types_identical(x->type, y->type)) { + if (x->type != t_invalid && + y->type != t_invalid) { + gbString xt = type_to_string(x->type); + gbString yt = type_to_string(y->type); + gbString expr_str = expr_to_string(x->expr); + error(ie->op, "Mismatched types in interval expression '%s' : '%s' vs '%s'", expr_str, xt, yt); + gb_string_free(expr_str); + gb_string_free(yt); + gb_string_free(xt); + } + return false; + } + + Type *type = x->type; + if (!is_type_integer(type) && !is_type_float(type) && !is_type_pointer(type) && !is_type_enum(type)) { + error(ie->op, "Only numerical and pointer types are allowed within interval expressions"); + return false; + } + + if (x->mode == Addressing_Constant && + y->mode == Addressing_Constant) { + ExactValue a = x->value; + ExactValue b = y->value; + + GB_ASSERT(are_types_identical(x->type, y->type)); + + TokenKind op = Token_Lt; + switch (ie->op.kind) { + case Token_Ellipsis: op = Token_LtEq; break; + case Token_RangeHalf: op = Token_Lt; break; + default: error(ie->op, "Invalid range operator"); break; + } + bool ok = compare_exact_values(op, a, b); + if (!ok) { + // TODO(bill): Better error message + error(ie->op, "Invalid interval range"); + return false; + } + + ExactValue inline_for_depth = exact_value_sub(b, a); + if (ie->op.kind == Token_Ellipsis) { + inline_for_depth = exact_value_increment_one(inline_for_depth); + } + + if (inline_for_depth_) *inline_for_depth_ = inline_for_depth; + } else { + error(ie->op, "Interval expressions must be constant"); + return false; + } + + add_type_and_value(&c->checker->info, ie->left, x->mode, x->type, x->value); + add_type_and_value(&c->checker->info, ie->right, y->mode, y->type, y->value); + + return true; +} + + ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast *node, Type *type_hint) { ExprKind kind = Expr_Stmt; @@ -6697,35 +6789,26 @@ ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast *node, Type case_ast_node(bl, BasicLit, node); - // NOTE(bill, 2018-06-17): Placing this in the parser is slower than - // placing it here for some reason. So don't move it to the parsing - // stage if you _think_ it will be faster, only do it if you _know_ it - // will be faster. Type *t = t_invalid; - switch (bl->token.kind) { - case Token_Integer: t = t_untyped_integer; break; - case Token_Float: t = t_untyped_float; break; - case Token_String: t = t_untyped_string; break; - case Token_Rune: t = t_untyped_rune; break; - case Token_Imag: { - String s = bl->token.string; - Rune r = s[s.len-1]; - // NOTE(bill, 2019-08-25): Allow for quaternions by having j and k imaginary numbers - switch (r) { - case 'i': t = t_untyped_complex; break; - case 'j': t = t_untyped_quaternion; break; - case 'k': t = t_untyped_quaternion; break; + switch (bl->value.kind) { + case ExactValue_String: t = t_untyped_string; break; + case ExactValue_Float: t = t_untyped_float; break; + case ExactValue_Complex: t = t_untyped_complex; break; + case ExactValue_Quaternion: t = t_untyped_quaternion; break; + case ExactValue_Integer: + t = t_untyped_integer; + if (bl->token.kind == Token_Rune) { + t = t_untyped_rune; } - break; - } default: - GB_PANIC("Unknown literal"); + GB_PANIC("Unhandled value type for basic literal"); break; } + o->mode = Addressing_Constant; o->type = t; - o->value = exact_value_from_basic_literal(bl->token); + o->value = bl->value; case_end; case_ast_node(bd, BasicDirective, node); @@ -7090,9 +7173,8 @@ ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast *node, Type if (is_type_simd_vector(t)) { error(cl->elems[0], "'field = value' is not allowed for SIMD vector literals"); } else { - Map seen = {}; - map_init(&seen, heap_allocator()); - defer (map_destroy(&seen)); + RangeCache rc = range_cache_make(heap_allocator()); + defer (range_cache_destroy(&rc)); for_array(i, cl->elems) { Ast *elem = cl->elems[i]; @@ -7102,36 +7184,89 @@ ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast *node, Type } ast_node(fv, FieldValue, elem); - Operand op_index = {}; - check_expr(c, &op_index, fv->field); + if (is_ast_range(fv->field)) { + Token op = fv->field->BinaryExpr.op; - if (op_index.mode != Addressing_Constant || !is_type_integer(core_type(op_index.type))) { - error(elem, "Expected a constant integer as an array field"); - continue; - } + Operand x = {}; + Operand y = {}; + bool ok = check_range(c, fv->field, &x, &y, nullptr); + if (!ok) { + continue; + } + if (x.mode != Addressing_Constant || !is_type_integer(core_type(x.type))) { + error(x.expr, "Expected a constant integer as an array field"); + continue; + } - i64 index = exact_value_to_i64(op_index.value); + if (y.mode != Addressing_Constant || !is_type_integer(core_type(y.type))) { + error(y.expr, "Expected a constant integer as an array field"); + continue; + } - if (max_type_count >= 0 && (index < 0 || index >= max_type_count)) { - error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", index, max_type_count, LIT(context_name)); - continue; - } + i64 lo = exact_value_to_i64(x.value); + i64 hi = exact_value_to_i64(y.value); + if (op.kind == Token_RangeHalf) { + hi -= 1; + } + i64 max_index = hi; - if (map_get(&seen, hash_integer(u64(index))) != nullptr) { - error(elem, "Duplicate field index %lld for %.*s", index, LIT(context_name)); - continue; - } - map_set(&seen, hash_integer(u64(index)), true); + bool new_range = range_cache_add_range(&rc, lo, hi); + if (!new_range) { + error(elem, "Overlapping field range index %lld %.*s %lld for %.*s", lo, LIT(op.string), hi, LIT(context_name)); + continue; + } - if (max < index) { - max = index; - } - Operand operand = {}; - check_expr_with_type_hint(c, &operand, fv->value, elem_type); - check_assignment(c, &operand, elem_type, context_name); + if (max_type_count >= 0 && (lo < 0 || lo >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", lo, max_type_count, LIT(context_name)); + continue; + } + if (max_type_count >= 0 && (hi < 0 || hi >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", hi, max_type_count, LIT(context_name)); + continue; + } + + if (max < max_index) { + max = max_index; + } + + Operand operand = {}; + check_expr_with_type_hint(c, &operand, fv->value, elem_type); + check_assignment(c, &operand, elem_type, context_name); + + is_constant = is_constant && operand.mode == Addressing_Constant; + } else { + Operand op_index = {}; + check_expr(c, &op_index, fv->field); + + if (op_index.mode != Addressing_Constant || !is_type_integer(core_type(op_index.type))) { + error(elem, "Expected a constant integer as an array field"); + continue; + } - is_constant = is_constant && operand.mode == Addressing_Constant; + i64 index = exact_value_to_i64(op_index.value); + + if (max_type_count >= 0 && (index < 0 || index >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", index, max_type_count, LIT(context_name)); + continue; + } + + bool new_index = range_cache_add_index(&rc, index); + if (!new_index) { + error(elem, "Duplicate field index %lld for %.*s", index, LIT(context_name)); + continue; + } + + if (max < index) { + max = index; + } + + Operand operand = {}; + check_expr_with_type_hint(c, &operand, fv->value, elem_type); + check_assignment(c, &operand, elem_type, context_name); + + is_constant = is_constant && operand.mode == Addressing_Constant; + } } cl->max_index = max; diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index 72913903d..d4398664b 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -605,89 +605,15 @@ void check_inline_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) { if (is_ast_range(expr)) { ast_node(ie, BinaryExpr, expr); - Operand x = {Addressing_Invalid}; - Operand y = {Addressing_Invalid}; + Operand x = {}; + Operand y = {}; - check_expr(ctx, &x, ie->left); - if (x.mode == Addressing_Invalid) { - goto skip_expr; - } - check_expr(ctx, &y, ie->right); - if (y.mode == Addressing_Invalid) { - goto skip_expr; - } - - convert_to_typed(ctx, &x, y.type); - if (x.mode == Addressing_Invalid) { - goto skip_expr; - } - convert_to_typed(ctx, &y, x.type); - if (y.mode == Addressing_Invalid) { - goto skip_expr; - } - - convert_to_typed(ctx, &x, default_type(y.type)); - if (x.mode == Addressing_Invalid) { - goto skip_expr; - } - convert_to_typed(ctx, &y, default_type(x.type)); - if (y.mode == Addressing_Invalid) { - goto skip_expr; - } - - if (!are_types_identical(x.type, y.type)) { - if (x.type != t_invalid && - y.type != t_invalid) { - gbString xt = type_to_string(x.type); - gbString yt = type_to_string(y.type); - gbString expr_str = expr_to_string(x.expr); - error(ie->op, "Mismatched types in interval expression '%s' : '%s' vs '%s'", expr_str, xt, yt); - gb_string_free(expr_str); - gb_string_free(yt); - gb_string_free(xt); - } - goto skip_expr; - } - - Type *type = x.type; - if (!is_type_integer(type) && !is_type_float(type) && !is_type_pointer(type) && !is_type_enum(type)) { - error(ie->op, "Only numerical and pointer types are allowed within interval expressions"); - goto skip_expr; - } - - if (x.mode == Addressing_Constant && - y.mode == Addressing_Constant) { - ExactValue a = x.value; - ExactValue b = y.value; - - GB_ASSERT(are_types_identical(x.type, y.type)); - - TokenKind op = Token_Lt; - switch (ie->op.kind) { - case Token_Ellipsis: op = Token_LtEq; break; - case Token_RangeHalf: op = Token_Lt; break; - default: error(ie->op, "Invalid range operator"); break; - } - bool ok = compare_exact_values(op, a, b); - if (!ok) { - // TODO(bill): Better error message - error(ie->op, "Invalid interval range"); - goto skip_expr; - } - - inline_for_depth = exact_value_sub(b, a); - if (ie->op.kind == Token_Ellipsis) { - inline_for_depth = exact_value_increment_one(inline_for_depth); - } - - } else { - error(ie->op, "Interval expressions must be constant"); + bool ok = check_range(ctx, expr, &x, &y, &inline_for_depth); + if (!ok) { goto skip_expr; } - add_type_and_value(&ctx->checker->info, ie->left, x.mode, x.type, x.value); - add_type_and_value(&ctx->checker->info, ie->right, y.mode, y.type, y.value); - val0 = type; + val0 = x.type; val1 = t_int; } else { Operand operand = {Addressing_Invalid}; diff --git a/src/common.cpp b/src/common.cpp index db6505a36..b034ad720 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -143,6 +143,9 @@ GB_ALLOCATOR_PROC(heap_allocator_proc) { #define for_array(index_, array_) for (isize index_ = 0; index_ < (array_).count; index_++) +#include "range_cache.cpp" + + u64 fnv64a(void const *data, isize len) { u8 const *bytes = cast(u8 const *)data; diff --git a/src/exact_value.cpp b/src/exact_value.cpp index 95d04d93b..42b22c8ef 100644 --- a/src/exact_value.cpp +++ b/src/exact_value.cpp @@ -293,12 +293,12 @@ ExactValue exact_value_from_basic_literal(Token token) { case 'i': return exact_value_complex(0, imag); case 'j': return exact_value_quaternion(0, 0, imag, 0); case 'k': return exact_value_quaternion(0, 0, 0, imag); + default: GB_PANIC("Invalid imaginary basic literal"); } } case Token_Rune: { Rune r = GB_RUNE_INVALID; gb_utf8_decode(token.string.text, token.string.len, &r); - // gb_printf("%.*s rune: %d\n", LIT(token.string), r); return exact_value_i64(r); } default: diff --git a/src/ir.cpp b/src/ir.cpp index c654e2bf8..5eeba91a2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -7865,14 +7865,40 @@ irAddr ir_build_addr(irProcedure *proc, Ast *expr) { if (ir_is_elem_const(proc->module, fv->value, et)) { continue; } - auto tav = fv->field->tav; - GB_ASSERT(tav.mode == Addressing_Constant); - i64 index = exact_value_to_i64(tav.value); + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); + + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op == Token_Ellipsis) { + hi += 1; + } - irCompoundLitElemTempData data = {}; - data.expr = fv->value; - data.elem_index = cast(i32)index; - array_add(&temp_data, data); + irValue *value = ir_build_expr(proc, fv->value); + + for (i64 k = lo; k < hi; k++) { + irCompoundLitElemTempData data = {}; + data.value = value; + data.elem_index = cast(i32)k; + array_add(&temp_data, data); + } + + } else { + auto tav = fv->field->tav; + GB_ASSERT(tav.mode == Addressing_Constant); + i64 index = exact_value_to_i64(tav.value); + + irCompoundLitElemTempData data = {}; + data.value = ir_emit_conv(proc, ir_build_expr(proc, fv->value), et); + data.expr = fv->value; + data.elem_index = cast(i32)index; + array_add(&temp_data, data); + } } else { if (ir_is_elem_const(proc->module, elem, et)) { @@ -7897,15 +7923,15 @@ irAddr ir_build_addr(irProcedure *proc, Ast *expr) { defer (proc->return_ptr_hint_value = return_ptr_hint_value); defer (proc->return_ptr_hint_used = return_ptr_hint_used); + irValue *field_expr = temp_data[i].value; Ast *expr = temp_data[i].expr; - if (expr == nullptr) { - continue; - } proc->return_ptr_hint_value = temp_data[i].gep; proc->return_ptr_hint_ast = unparen_expr(expr); - irValue *field_expr = ir_build_expr(proc, expr); + if (field_expr == nullptr) { + field_expr = ir_build_expr(proc, expr); + } Type *t = ir_type(field_expr); GB_ASSERT(t->kind != Type_Tuple); irValue *ev = ir_emit_conv(proc, field_expr, et); @@ -7945,19 +7971,43 @@ irAddr ir_build_addr(irProcedure *proc, Ast *expr) { continue; } + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); + + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op == Token_Ellipsis) { + hi += 1; + } - GB_ASSERT(fv->field->tav.mode == Addressing_Constant); - i64 index = exact_value_to_i64(fv->field->tav.value); + irValue *value = ir_emit_conv(proc, ir_build_expr(proc, fv->value), et); - irValue *field_expr = ir_build_expr(proc, fv->value); - GB_ASSERT(!is_type_tuple(ir_type(field_expr))); + for (i64 k = lo; k < hi; k++) { + irCompoundLitElemTempData data = {}; + data.value = value; + data.elem_index = cast(i32)k; + array_add(&temp_data, data); + } - irValue *ev = ir_emit_conv(proc, field_expr, et); + } else { + GB_ASSERT(fv->field->tav.mode == Addressing_Constant); + i64 index = exact_value_to_i64(fv->field->tav.value); - irCompoundLitElemTempData data = {}; - data.value = ev; - data.elem_index = cast(i32)index; - array_add(&temp_data, data); + irValue *field_expr = ir_build_expr(proc, fv->value); + GB_ASSERT(!is_type_tuple(ir_type(field_expr))); + + irValue *ev = ir_emit_conv(proc, field_expr, et); + + irCompoundLitElemTempData data = {}; + data.value = ev; + data.elem_index = cast(i32)index; + array_add(&temp_data, data); + } } else { if (ir_is_elem_const(proc->module, elem, et)) { continue; @@ -8015,14 +8065,36 @@ irAddr ir_build_addr(irProcedure *proc, Ast *expr) { Ast *elem = cl->elems[i]; if (elem->kind == Ast_FieldValue) { ast_node(fv, FieldValue, elem); - GB_ASSERT(fv->field->tav.mode == Addressing_Constant); + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); + + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op == Token_Ellipsis) { + hi += 1; + } - i64 field_index = exact_value_to_i64(fv->field->tav.value); + irValue *value = ir_emit_conv(proc, ir_build_expr(proc, fv->value), et); - irValue *ev = ir_build_expr(proc, fv->value); - irValue *value = ir_emit_conv(proc, ev, et); - irValue *ep = ir_emit_array_epi(proc, items, cast(i32)field_index); - ir_emit_store(proc, ep, value); + for (i64 k = lo; k < hi; k++) { + irValue *ep = ir_emit_array_epi(proc, items, cast(i32)k); + ir_emit_store(proc, ep, value); + } + } else { + GB_ASSERT(fv->field->tav.mode == Addressing_Constant); + + i64 field_index = exact_value_to_i64(fv->field->tav.value); + + irValue *ev = ir_build_expr(proc, fv->value); + irValue *value = ir_emit_conv(proc, ev, et); + irValue *ep = ir_emit_array_epi(proc, items, cast(i32)field_index); + ir_emit_store(proc, ep, value); + } } else { irValue *value = ir_emit_conv(proc, ir_build_expr(proc, elem), et); irValue *ep = ir_emit_array_epi(proc, items, cast(i32)i); diff --git a/src/ir_print.cpp b/src/ir_print.cpp index cece0c1db..1d0c8af35 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -887,17 +887,47 @@ void ir_print_exact_value(irFileBuffer *f, irModule *m, ExactValue value, Type * for (isize j = 0; j < elem_count; j++) { Ast *elem = cl->elems[j]; ast_node(fv, FieldValue, elem); - TypeAndValue index_tav = fv->field->tav; - GB_ASSERT(index_tav.mode == Addressing_Constant); - i64 index = exact_value_to_i64(index_tav.value); - if (index == i) { - TypeAndValue tav = fv->value->tav; - if (tav.mode != Addressing_Constant) { + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); + + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op == Token_Ellipsis) { + hi += 1; + } + if (lo == i) { + TypeAndValue tav = fv->value->tav; + if (tav.mode != Addressing_Constant) { + break; + } + for (i64 k = lo; k < hi; k++) { + if (k > lo) ir_write_str_lit(f, ", "); + + ir_print_compound_element(f, m, tav.value, elem_type); + } + + found = true; + i += (hi-lo-1); + break; + } + } else { + TypeAndValue index_tav = fv->field->tav; + GB_ASSERT(index_tav.mode == Addressing_Constant); + i64 index = exact_value_to_i64(index_tav.value); + if (index == i) { + TypeAndValue tav = fv->value->tav; + if (tav.mode != Addressing_Constant) { + break; + } + ir_print_compound_element(f, m, tav.value, elem_type); + found = true; break; } - ir_print_compound_element(f, m, tav.value, elem_type); - found = true; - break; } } diff --git a/src/parser.cpp b/src/parser.cpp index 7f866922a..10aa10119 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -588,6 +588,7 @@ Ast *ast_undef(AstFile *f, Token token) { Ast *ast_basic_lit(AstFile *f, Token basic_lit) { Ast *result = alloc_ast_node(f, Ast_BasicLit); result->BasicLit.token = basic_lit; + result->BasicLit.value = exact_value_from_basic_literal(basic_lit); return result; } @@ -1509,8 +1510,11 @@ Ast *parse_value(AstFile *f) { if (f->curr_token.kind == Token_OpenBrace) { return parse_literal_value(f, nullptr); } - - Ast *value = parse_expr(f, false); + Ast *value; + bool prev_allow_range = f->allow_range; + f->allow_range = true; + value = parse_expr(f, false); + f->allow_range = prev_allow_range; return value; } @@ -1735,7 +1739,8 @@ Ast *parse_operand(AstFile *f, bool lhs) { operand = ast_bad_expr(f, token, f->curr_token); } operand->stmt_state_flags |= StmtStateFlag_no_deferred; - } */ else if (name.string == "file") { return ast_basic_directive(f, token, name.string); + } */ else if (name.string == "file") { + return ast_basic_directive(f, token, name.string); } else if (name.string == "line") { return ast_basic_directive(f, token, name.string); } else if (name.string == "procedure") { return ast_basic_directive(f, token, name.string); } else if (name.string == "caller_location") { return ast_basic_directive(f, token, name.string); diff --git a/src/parser.hpp b/src/parser.hpp index 9c3f733e5..83df7a9d6 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -222,6 +222,7 @@ enum StmtAllowFlag { AST_KIND(Undef, "undef", Token) \ AST_KIND(BasicLit, "basic literal", struct { \ Token token; \ + ExactValue value; \ }) \ AST_KIND(BasicDirective, "basic directive", struct { \ Token token; \ -- cgit v1.2.3