From b7858a66b94e87830c0c777547c4f411c4d8d259 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 28 May 2018 12:06:50 +0100 Subject: Parallelize per file rather than per package --- src/parser.cpp | 343 +++++++++++++++++++++++++++------------------------------ 1 file changed, 161 insertions(+), 182 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 4f35568d7..b1b1f8e05 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -3885,10 +3885,11 @@ void destroy_ast_file(AstFile *f) { bool init_parser(Parser *p) { GB_ASSERT(p != nullptr); - map_init(&p->imported_files, heap_allocator()); + string_set_init(&p->imported_files, heap_allocator()); map_init(&p->package_map, heap_allocator()); array_init(&p->packages, heap_allocator()); - array_init(&p->imports, heap_allocator()); + array_init(&p->package_imports, heap_allocator()); + array_init(&p->files_to_process, heap_allocator()); gb_mutex_init(&p->file_add_mutex); gb_mutex_init(&p->file_decl_mutex); return true; @@ -3898,47 +3899,144 @@ void destroy_parser(Parser *p) { GB_ASSERT(p != nullptr); // TODO(bill): Fix memory leak for_array(i, p->packages) { - AstPackage *package = p->packages[i]; - for_array(j, package->files) { - destroy_ast_file(package->files[j]); + AstPackage *pkg = p->packages[i]; + for_array(j, pkg->files) { + destroy_ast_file(pkg->files[j]); } - array_free(&package->files); + array_free(&pkg->files); } #if 0 - for_array(i, p->imports) { - // gb_free(heap_allocator(), p->imports[i].text); + for_array(i, p->package_imports) { + // gb_free(heap_allocator(), p->package_imports[i].text); } #endif array_free(&p->packages); - array_free(&p->imports); - map_destroy(&p->imported_files); + array_free(&p->package_imports); + array_free(&p->files_to_process); + string_set_destroy(&p->imported_files); map_destroy(&p->package_map); gb_mutex_destroy(&p->file_add_mutex); gb_mutex_destroy(&p->file_decl_mutex); } + +void parser_add_package(Parser *p, AstPackage *pkg) { + pkg->id = p->packages.count+1; + array_add(&p->packages, pkg); + if (pkg->name.len > 0) { + HashKey key = hash_string(pkg->name); + auto found = map_get(&p->package_map, key); + if (found) { + GB_ASSERT(pkg->files.count > 0); + AstFile *f = pkg->files[0]; + error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name)); + GB_ASSERT((*found)->files.count > 0); + TokenPos pos = (*found)->files[0]->package_token.pos; + gb_printf_err("\tpreviously declared at %.*s(%td:%td)", LIT(pos.file), pos.line, pos.column); + } else { + map_set(&p->package_map, key, pkg); + } + } +} + +void parser_add_file_to_process(Parser *p, AstPackage *pkg, FileInfo fi, TokenPos pos) { + ImportedFile f = {pkg, fi, pos, p->files_to_process.count}; + array_add(&p->files_to_process, f); +} + + // NOTE(bill): Returns true if it's added -bool try_add_import_path(Parser *p, String path, String rel_path, TokenPos pos, PackageKind kind = Package_Normal) { +bool try_add_import_path(Parser *p, String const &path, String const &rel_path, TokenPos pos, PackageKind kind = Package_Normal) { if (build_context.generate_docs) { return false; } - path = string_trim_whitespace(path); - rel_path = string_trim_whitespace(rel_path); + String const FILE_EXT = str_lit(".odin"); + + gb_mutex_lock(&p->file_add_mutex); + defer (gb_mutex_unlock(&p->file_add_mutex)); - HashKey key = hash_string(path); - if (map_get(&p->imported_files, key) != nullptr) { + if (string_set_exists(&p->imported_files, path)) { return false; } - map_set(&p->imported_files, key, true); + string_set_add(&p->imported_files, path); + + + AstPackage *pkg = gb_alloc_item(heap_allocator(), AstPackage); + pkg->kind = kind; + pkg->fullpath = path; + array_init(&pkg->files, heap_allocator()); + + // NOTE(bill): Single file initial package + if (kind == Package_Init && string_ends_with(path, FILE_EXT)) { + FileInfo fi = {}; + fi.name = filename_from_path(path); + fi.fullpath = path; + fi.size = get_file_size(path); + fi.is_dir = false; + + parser_add_file_to_process(p, pkg, fi, pos); + parser_add_package(p, pkg); + return true; + } + + + Array list = {}; + ReadDirectoryError rd_err = read_directory(path, &list); + defer (array_free(&list)); + + if (list.count == 1) { + GB_ASSERT(path != list[0].fullpath); + } + + + if (rd_err != ReadDirectory_None) { + if (pos.line != 0) { + gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column); + } + gb_mutex_lock(&global_error_collector.mutex); + defer (gb_mutex_unlock(&global_error_collector.mutex)); + global_error_collector.count++; + + + switch (rd_err) { + case ReadDirectory_InvalidPath: + gb_printf_err("Invalid path: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_NotExists: + gb_printf_err("Path does not exist: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_NotDir: + gb_printf_err("Expected a directory for a package, got a file: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_Unknown: + gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(rel_path)); + return false; + case ReadDirectory_Permission: + gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_Empty: + gb_printf_err("Empty directory: %.*s\n", LIT(rel_path)); + return false; + } + } + + for_array(list_index, list) { + FileInfo fi = list[list_index]; + String name = fi.name; + if (string_ends_with(name, FILE_EXT)) { + if (is_excluded_target_filename(name)) { + continue; + } + parser_add_file_to_process(p, pkg, fi, pos); + } + } - ImportedPackage item = {}; - item.kind = kind; - item.path = path; - item.rel_path = rel_path; - item.pos = pos; - item.index = p->imports.count; - array_add(&p->imports, item); + parser_add_package(p, pkg); return true; } @@ -4074,7 +4172,8 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Arraykind == AstNode_CallExpr && expr->CallExpr.proc->kind == AstNode_BasicDirective && expr->CallExpr.proc->BasicDirective.name == "assert") { - // NOTE(bill): Okay! + + f->assert_decl_count += 1; continue; } } @@ -4083,13 +4182,14 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Arraykind == AstNode_ImportDecl) { ast_node(id, ImportDecl, node); - String original_string = id->relpath.string; + String original_string = string_trim_whitespace(id->relpath.string); String import_path = {}; bool ok = determine_path_from_string(p, node, base_dir, original_string, &import_path); if (!ok) { decls[i] = ast_bad_decl(f, id->relpath, id->relpath); continue; } + import_path = string_trim_whitespace(import_path); id->fullpath = import_path; try_add_import_path(p, import_path, original_string, ast_node_token(node).pos); @@ -4141,7 +4241,7 @@ bool parse_file(Parser *p, AstFile *f) { if (package_name.kind == Token_Ident) { if (package_name.string == "_") { error(package_name, "Invalid package name '_'"); - } else if (f->package->kind != Package_Runtime && package_name.string == "runtime") { + } else if (f->pkg->kind != Package_Runtime && package_name.string == "runtime") { error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); } } @@ -4161,17 +4261,24 @@ bool parse_file(Parser *p, AstFile *f) { } -ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, TokenPos pos) { +ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) { + AstPackage *pkg = imported_file.pkg; + FileInfo *fi = &imported_file.fi; + TokenPos pos = imported_file.pos; + AstFile *file = gb_alloc_item(heap_allocator(), AstFile); - file->package = package; + file->pkg = pkg; - p->file_index += 1; - file->id = p->file_index; + file->id = imported_file.index+1; TokenPos err_pos = {0}; ParseFileError err = init_ast_file(file, fi->fullpath, &err_pos); if (err != ParseFile_None) { + gb_mutex_lock(&global_error_collector.mutex); + defer (gb_mutex_unlock(&global_error_collector.mutex)); + global_error_collector.count++; + if (err == ParseFile_EmptyFile) { if (fi->fullpath == p->init_fullpath) { gb_printf_err("Initial file is empty - %.*s\n", LIT(p->init_fullpath)); @@ -4206,9 +4313,6 @@ ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, } gb_printf_err("\n"); - gb_mutex_lock(&global_error_collector.mutex); - global_error_collector.count++; - gb_mutex_unlock(&global_error_collector.mutex); return err; } @@ -4217,164 +4321,39 @@ ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, skip: if (parse_file(p, file)) { gb_mutex_lock(&p->file_add_mutex); - array_add(&package->files, file); + defer (gb_mutex_unlock(&p->file_add_mutex)); - if (package->name.len == 0) { - package->name = file->package_name; - } else if (file->tokens.count > 0 && package->name != file->package_name) { - error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(package->name), LIT(file->package_name)); + array_add(&pkg->files, file); + + if (pkg->name.len == 0) { + pkg->name = file->package_name; + } else if (file->tokens.count > 0 && pkg->name != file->package_name) { + error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(pkg->name), LIT(file->package_name)); } p->total_line_count += file->tokenizer.line_count; p->total_token_count += file->tokens.count; - gb_mutex_unlock(&p->file_add_mutex); } return ParseFile_None; } -void parser_add_package(Parser *p, AstPackage *package) { - package->id = p->packages.count+1; - array_add(&p->packages, package); - if (package->name.len > 0) { - HashKey key = hash_string(package->name); - auto found = map_get(&p->package_map, key); - if (found) { - GB_ASSERT(package->files.count > 0); - AstFile *f = package->files[0]; - error(f->package_token, "Non-unique package name '%.*s'", LIT(package->name)); - GB_ASSERT((*found)->files.count > 0); - TokenPos pos = (*found)->files[0]->package_token.pos; - gb_printf_err("\tpreviously declared at %.*s(%td:%td)", LIT(pos.file), pos.line, pos.column); - } else { - map_set(&p->package_map, key, package); - } - } -} - -ParseFileError parse_import(Parser *p, ImportedPackage imported_package) { - String import_path = imported_package.path; - String import_rel_path = imported_package.rel_path; - TokenPos pos = imported_package.pos; - String const file_ext = str_lit(".odin"); - - // NOTE(bill): Single file initial package - if (imported_package.kind == Package_Init && string_ends_with(import_path, file_ext)) { - AstPackage *package = gb_alloc_item(heap_allocator(), AstPackage); - package->kind = imported_package.kind; - package->fullpath = import_path; - array_init(&package->files, heap_allocator()); - - FileInfo fi = {}; - fi.name = filename_from_path(import_path); - fi.fullpath = import_path; - fi.size = get_file_size(import_path); - fi.is_dir = false; - - ParseFileError err = parse_imported_file(p, package, &fi, pos); - if (err != ParseFile_None) { - return err; - } - - parser_add_package(p, package); - - return ParseFile_None; - } - - - Array list = {}; - ReadDirectoryError rd_err = read_directory(import_path, &list); - defer (array_free(&list)); - - if (list.count == 1) { - GB_ASSERT(import_path != list[0].fullpath); - } - - - if (rd_err != ReadDirectory_None) { - if (pos.line != 0) { - gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column); - } - gb_mutex_lock(&global_error_collector.mutex); - defer (gb_mutex_unlock(&global_error_collector.mutex)); - global_error_collector.count++; - - - switch (rd_err) { - case ReadDirectory_InvalidPath: - gb_printf_err("Invalid path: %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_NotExists: - gb_printf_err("Path does not exist: %.*s\n", LIT(import_rel_path)); - return ParseFile_NotFound; - - case ReadDirectory_NotDir: - gb_printf_err("Expected a directory for a package, got a file: %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_Unknown: - gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - case ReadDirectory_Permission: - gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_Empty: - gb_printf_err("Empty directory: %.*s\n", LIT(import_rel_path)); - return ParseFile_EmptyFile; - } - } - - AstPackage *package = gb_alloc_item(heap_allocator(), AstPackage); - package->kind = imported_package.kind; - package->fullpath = import_path; - array_init(&package->files, heap_allocator()); - - // TODO(bill): Fix concurrency - for_array(list_index, list) { - FileInfo *fi = &list[list_index]; - String name = fi->name; - if (string_ends_with(name, file_ext)) { - if (is_excluded_target_filename(name)) { - continue; - } - ParseFileError err = parse_imported_file(p, package, fi, pos); - if (err != ParseFile_None) { - return err; - } - } - } - - parser_add_package(p, package); - - return ParseFile_None; -} - GB_THREAD_PROC(parse_worker_file_proc) { if (thread == nullptr) return 0; auto *p = cast(Parser *)thread->user_data; isize index = thread->user_index; - ImportedPackage imported_package = p->imports[index]; - ParseFileError err = parse_import(p, imported_package); + ParseFileError err = process_imported_file(p, p->files_to_process[index]); return cast(isize)err; } +void add_shared_package(Parser *p, String name, TokenPos pos) { -struct ParserThreadWork { - Parser *parser; - isize import_index; -}; - -void add_shared_package(Parser *p, String name, TokenPos pos, PackageKind kind) { - String s = get_fullpath_core(heap_allocator(), name); - try_add_import_path(p, s, s, pos, kind); } ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); - char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char *)&init_filename[0]); + char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char const *)&init_filename[0]); String init_fullpath = string_trim_whitespace(make_string_c(fullpath_str)); if (!path_is_directory(init_fullpath)) { String const ext = str_lit(".odin"); @@ -4385,25 +4364,24 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } TokenPos init_pos = {}; - ImportedPackage init_imported_package = {Package_Init, init_fullpath, init_fullpath, init_pos}; - isize shared_package_count = 0; if (!build_context.generate_docs) { - add_shared_package(p, str_lit("runtime"), init_pos, Package_Runtime); shared_package_count++; + String s = get_fullpath_core(heap_allocator(), str_lit("runtime")); + try_add_import_path(p, s, s, init_pos, Package_Runtime); } - array_add(&p->imports, init_imported_package); + try_add_import_path(p, init_fullpath, init_fullpath, init_pos, Package_Init); p->init_fullpath = init_fullpath; // IMPORTANT TODO(bill): Figure out why this doesn't work on *nix sometimes -#if defined(GB_SYSTEM_WINDOWS) +#if 1 && defined(GB_SYSTEM_WINDOWS) isize thread_count = gb_max(build_context.thread_count, 1); if (thread_count > 1) { isize volatile curr_import_index = 0; - + isize initial_file_count = p->files_to_process.count; // NOTE(bill): Make sure that these are in parsed in this order - for (isize i = 0; i < shared_package_count; i++) { - ParseFileError err = parse_import(p, p->imports[i]); + for (isize i = 0; i < initial_file_count; i++) { + ParseFileError err = process_imported_file(p, p->files_to_process[i]); if (err != ParseFile_None) { return err; } @@ -4429,7 +4407,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { gbThread *t = &worker_threads[i]; if (gb_thread_is_running(t)) { are_any_alive = true; - } else if (curr_import_index < p->imports.count) { + } else if (curr_import_index < p->files_to_process.count) { auto curr_err = cast(ParseFileError)t->return_value; if (curr_err != ParseFile_None) { array_add(&errors, curr_err); @@ -4441,7 +4419,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } } } - if (!are_any_alive && curr_import_index >= p->imports.count) { + if (!are_any_alive && curr_import_index >= p->files_to_process.count) { break; } } @@ -4450,16 +4428,17 @@ ParseFileError parse_packages(Parser *p, String init_filename) { return errors[errors.count-1]; } } else { - for_array(i, p->imports) { - ParseFileError err = parse_import(p, p->imports[i]); + for_array(i, p->files_to_process) { + ParseFileError err = process_imported_file(p, p->files_to_process[i]); if (err != ParseFile_None) { return err; } } } #else - for_array(i, p->imports) { - ParseFileError err = parse_import(p, p->imports[i]); + for_array(i, p->files_to_process) { + ImportedFile f = p->files_to_process[i]; + ParseFileError err = process_imported_file(p, f); if (err != ParseFile_None) { return err; } -- cgit v1.2.3