From 042dbda47f8a428c1be2b1af2937f0cbff109c11 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 13 Sep 2021 01:30:30 +0100 Subject: Replace many uses of `heap_allocator()` with `permanent_allocator()` --- src/tokenizer.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 237179ca5..35d6775a3 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -371,7 +371,7 @@ void begin_error_block(void) { void end_error_block(void) { if (global_error_collector.error_buffer.count > 0) { isize n = global_error_collector.error_buffer.count; - u8 *text = gb_alloc_array(heap_allocator(), u8, n+1); + u8 *text = gb_alloc_array(permanent_allocator(), u8, n+1); gb_memmove(text, global_error_collector.error_buffer.data, n); text[n] = 0; String s = {text, n}; @@ -404,7 +404,7 @@ ERROR_OUT_PROC(default_error_out_va) { } else { mutex_lock(&global_error_collector.error_out_mutex); { - u8 *text = gb_alloc_array(heap_allocator(), u8, n+1); + u8 *text = gb_alloc_array(permanent_allocator(), u8, n+1); gb_memmove(text, buf, n); text[n] = 0; array_add(&global_error_collector.errors, make_string(text, n)); @@ -838,12 +838,6 @@ TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &full return err; } -gb_inline void destroy_tokenizer(Tokenizer *t) { - if (t->start != nullptr) { - gb_free(heap_allocator(), t->start); - } -} - gb_inline i32 digit_value(Rune r) { switch (r) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': -- cgit v1.2.3 From 3713f11461ed2cd42af3cfa00afd536b98f8ca54 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 18 Sep 2021 11:58:27 +0100 Subject: Refactor `init_tokenizer_with_data` to file memory mapping (Windows only currently) --- src/common.cpp | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/tokenizer.cpp | 54 +++++++++++------------- 2 files changed, 145 insertions(+), 30 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/common.cpp b/src/common.cpp index b132c26b0..8a220c799 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -850,6 +850,127 @@ ReadDirectoryError read_directory(String path, Array *fi) { +struct MemoryMappedFile { + void *handle; + + void *data; + i32 size; +}; +enum MemoryMappedFileError { + MemoryMappedFile_None, + + MemoryMappedFile_Empty, + MemoryMappedFile_FileTooLarge, + MemoryMappedFile_Invalid, + MemoryMappedFile_NotExists, + MemoryMappedFile_Permission, + + MemoryMappedFile_COUNT, +}; + +MemoryMappedFileError memory_map_file_32(char const *fullpath, MemoryMappedFile *memory_mapped_file) { + MemoryMappedFileError err = MemoryMappedFile_None; + +#if defined(GB_SYSTEM_WINDOWS) + isize w_len = 0; + wchar_t *w_str = gb__alloc_utf8_to_ucs2(temporary_allocator(), fullpath, &w_len); + if (w_str == nullptr) { + return MemoryMappedFile_Invalid; + } + i64 file_size = 0; + LARGE_INTEGER li_file_size = {}; + HANDLE handle = nullptr; + HANDLE file_mapping = nullptr; + void *file_data = nullptr; + + handle = CreateFileW(w_str, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); + if (handle == INVALID_HANDLE_VALUE) { + handle = nullptr; + goto window_handle_file_error; + } + + li_file_size = {}; + if (!GetFileSizeEx(handle, &li_file_size)) { + goto window_handle_file_error; + } + file_size = cast(i64)li_file_size.QuadPart; + if (file_size > I32_MAX) { + CloseHandle(handle); + return MemoryMappedFile_FileTooLarge; + } + + if (file_size == 0) { + CloseHandle(handle); + err = MemoryMappedFile_Empty; + memory_mapped_file->handle = nullptr; + memory_mapped_file->data = nullptr; + memory_mapped_file->size = 0; + return err; + } + + file_mapping = CreateFileMappingW(handle, nullptr, PAGE_READONLY, 0, 0, nullptr); + CloseHandle(handle); + + file_data = MapViewOfFileEx(file_mapping, FILE_MAP_READ, 0, 0, 0/*file_size*/, nullptr/*base address*/); + memory_mapped_file->handle = cast(void *)file_mapping; + memory_mapped_file->data = file_data; + memory_mapped_file->size = cast(i32)file_size; + return err; + +window_handle_file_error:; + { + DWORD handle_err = GetLastError(); + CloseHandle(handle); + err = MemoryMappedFile_Invalid; + switch (handle_err) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_DRIVE: + err = MemoryMappedFile_NotExists; + break; + case ERROR_ACCESS_DENIED: + case ERROR_INVALID_ACCESS: + err = MemoryMappedFile_Permission; + break; + } + return err; + } + +#else + // TODO(bill): Memory map rather than copy contents + gbFileContents fc = gb_file_read_contents(heap_allocator(), true, fullpath); + + if (fc.size > I32_MAX) { + err = MemoryMappedFile_FileTooLarge; + gb_file_free_contents(&fc); + } else if (fc.data != nullptr) { + memory_mapped_file->handle = nullptr; + memory_mapped_file->data = fc.data; + memory_mapped_file->size = cast(i32)fc.size; + } else { + gbFile f = {}; + gbFileError file_err = gb_file_open(&f, fullpath); + defer (gb_file_close(&f)); + + switch (file_err) { + case gbFileError_Invalid: err = MemoryMappedFile_Invalid; break; + case gbFileError_NotExists: err = MemoryMappedFile_NotExists; break; + case gbFileError_Permission: err = MemoryMappedFile_Permission; break; + } + + if (err == MemoryMappedFile_None && gb_file_size(&f) == 0) { + err = MemoryMappedFile_Empty; + } + } + return err; +#endif +} + + + + + + #define USE_DAMERAU_LEVENSHTEIN 1 isize levenstein_distance_case_insensitive(String const &a, String const &b) { diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 35d6775a3..2251b4f96 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -802,39 +802,33 @@ void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, } } -TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) { - TokenizerInitError err = TokenizerInit_None; - - char *c_str = alloc_cstring(temporary_allocator(), fullpath); - - // TODO(bill): Memory map rather than copy contents - gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str); +TokenizerInitError memory_mapped_file_error_map_to_tokenizer[MemoryMappedFile_COUNT] = { + TokenizerInit_None, /*MemoryMappedFile_None*/ + TokenizerInit_Empty, /*MemoryMappedFile_Empty*/ + TokenizerInit_FileTooLarge, /*MemoryMappedFile_FileTooLarge*/ + TokenizerInit_Invalid, /*MemoryMappedFile_Invalid*/ + TokenizerInit_NotExists, /*MemoryMappedFile_NotExists*/ + TokenizerInit_Permission, /*MemoryMappedFile_Permission*/ +}; - if (fc.size > I32_MAX) { - t->fullpath = fullpath; - t->line_count = 1; - err = TokenizerInit_FileTooLarge; - gb_file_free_contents(&fc); - } else if (fc.data != nullptr) { - init_tokenizer_with_data(t, fullpath, fc.data, fc.size); - } else { +TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) { + MemoryMappedFile memory_mapped_file = {}; + MemoryMappedFileError mmf_err = memory_map_file_32( + alloc_cstring(temporary_allocator(), fullpath), + &memory_mapped_file + ); + + TokenizerInitError err = memory_mapped_file_error_map_to_tokenizer[mmf_err]; + switch (mmf_err) { + case MemoryMappedFile_None: + init_tokenizer_with_data(t, fullpath, memory_mapped_file.data, cast(isize)memory_mapped_file.size); + break; + case MemoryMappedFile_FileTooLarge: + case MemoryMappedFile_Empty: t->fullpath = fullpath; t->line_count = 1; - gbFile f = {}; - gbFileError file_err = gb_file_open(&f, c_str); - defer (gb_file_close(&f)); - - switch (file_err) { - case gbFileError_Invalid: err = TokenizerInit_Invalid; break; - case gbFileError_NotExists: err = TokenizerInit_NotExists; break; - case gbFileError_Permission: err = TokenizerInit_Permission; break; - } - - if (err == TokenizerInit_None && gb_file_size(&f) == 0) { - err = TokenizerInit_Empty; - } - } - + break; + } return err; } -- cgit v1.2.3 From 05ac2002e0296c3acccca1d8cffaafb002e43120 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 18 Sep 2021 12:52:43 +0100 Subject: Force file copy on `odin strip-semicolon` --- src/common.cpp | 126 +++++++++++++++++++++++++++--------------------------- src/parser.cpp | 3 +- src/tokenizer.cpp | 10 +++-- 3 files changed, 71 insertions(+), 68 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/common.cpp b/src/common.cpp index 8a220c799..9497aaf18 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -868,76 +868,77 @@ enum MemoryMappedFileError { MemoryMappedFile_COUNT, }; -MemoryMappedFileError memory_map_file_32(char const *fullpath, MemoryMappedFile *memory_mapped_file) { +MemoryMappedFileError memory_map_file_32(char const *fullpath, MemoryMappedFile *memory_mapped_file, bool copy_file_contents) { MemoryMappedFileError err = MemoryMappedFile_None; -#if defined(GB_SYSTEM_WINDOWS) - isize w_len = 0; - wchar_t *w_str = gb__alloc_utf8_to_ucs2(temporary_allocator(), fullpath, &w_len); - if (w_str == nullptr) { - return MemoryMappedFile_Invalid; - } - i64 file_size = 0; - LARGE_INTEGER li_file_size = {}; - HANDLE handle = nullptr; - HANDLE file_mapping = nullptr; - void *file_data = nullptr; - - handle = CreateFileW(w_str, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); - if (handle == INVALID_HANDLE_VALUE) { - handle = nullptr; - goto window_handle_file_error; - } - - li_file_size = {}; - if (!GetFileSizeEx(handle, &li_file_size)) { - goto window_handle_file_error; - } - file_size = cast(i64)li_file_size.QuadPart; - if (file_size > I32_MAX) { - CloseHandle(handle); - return MemoryMappedFile_FileTooLarge; - } - - if (file_size == 0) { + if (!copy_file_contents) { + #if defined(GB_SYSTEM_WINDOWS) + isize w_len = 0; + wchar_t *w_str = gb__alloc_utf8_to_ucs2(temporary_allocator(), fullpath, &w_len); + if (w_str == nullptr) { + return MemoryMappedFile_Invalid; + } + i64 file_size = 0; + LARGE_INTEGER li_file_size = {}; + HANDLE handle = nullptr; + HANDLE file_mapping = nullptr; + void *file_data = nullptr; + + handle = CreateFileW(w_str, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); + if (handle == INVALID_HANDLE_VALUE) { + handle = nullptr; + goto window_handle_file_error; + } + + li_file_size = {}; + if (!GetFileSizeEx(handle, &li_file_size)) { + goto window_handle_file_error; + } + file_size = cast(i64)li_file_size.QuadPart; + if (file_size > I32_MAX) { + CloseHandle(handle); + return MemoryMappedFile_FileTooLarge; + } + + if (file_size == 0) { + CloseHandle(handle); + err = MemoryMappedFile_Empty; + memory_mapped_file->handle = nullptr; + memory_mapped_file->data = nullptr; + memory_mapped_file->size = 0; + return err; + } + + file_mapping = CreateFileMappingW(handle, nullptr, PAGE_READONLY, 0, 0, nullptr); CloseHandle(handle); - err = MemoryMappedFile_Empty; - memory_mapped_file->handle = nullptr; - memory_mapped_file->data = nullptr; - memory_mapped_file->size = 0; + + file_data = MapViewOfFileEx(file_mapping, FILE_MAP_READ, 0, 0, 0/*file_size*/, nullptr/*base address*/); + memory_mapped_file->handle = cast(void *)file_mapping; + memory_mapped_file->data = file_data; + memory_mapped_file->size = cast(i32)file_size; return err; - } - - file_mapping = CreateFileMappingW(handle, nullptr, PAGE_READONLY, 0, 0, nullptr); - CloseHandle(handle); - - file_data = MapViewOfFileEx(file_mapping, FILE_MAP_READ, 0, 0, 0/*file_size*/, nullptr/*base address*/); - memory_mapped_file->handle = cast(void *)file_mapping; - memory_mapped_file->data = file_data; - memory_mapped_file->size = cast(i32)file_size; - return err; -window_handle_file_error:; - { - DWORD handle_err = GetLastError(); - CloseHandle(handle); - err = MemoryMappedFile_Invalid; - switch (handle_err) { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_INVALID_DRIVE: - err = MemoryMappedFile_NotExists; - break; - case ERROR_ACCESS_DENIED: - case ERROR_INVALID_ACCESS: - err = MemoryMappedFile_Permission; - break; + window_handle_file_error:; + { + DWORD handle_err = GetLastError(); + CloseHandle(handle); + err = MemoryMappedFile_Invalid; + switch (handle_err) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_DRIVE: + err = MemoryMappedFile_NotExists; + break; + case ERROR_ACCESS_DENIED: + case ERROR_INVALID_ACCESS: + err = MemoryMappedFile_Permission; + break; + } + return err; } - return err; + #endif } -#else - // TODO(bill): Memory map rather than copy contents gbFileContents fc = gb_file_read_contents(heap_allocator(), true, fullpath); if (fc.size > I32_MAX) { @@ -963,7 +964,6 @@ window_handle_file_error:; } } return err; -#endif } diff --git a/src/parser.cpp b/src/parser.cpp index e33531fad..f72d8a73c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4645,7 +4645,8 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) { zero_item(&f->tokenizer); f->tokenizer.curr_file_id = f->id; - TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath); + bool copy_file_contents = build_context.command_kind == Command_strip_semicolon; + TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath, copy_file_contents); if (err != TokenizerInit_None) { switch (err) { case TokenizerInit_Empty: diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 2251b4f96..97836bd1b 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -722,6 +722,8 @@ struct Tokenizer { i32 error_count; bool insert_semicolon; + + MemoryMappedFile memory_mapped_file; }; @@ -811,17 +813,17 @@ TokenizerInitError memory_mapped_file_error_map_to_tokenizer[MemoryMappedFile_CO TokenizerInit_Permission, /*MemoryMappedFile_Permission*/ }; -TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) { - MemoryMappedFile memory_mapped_file = {}; +TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath, bool copy_file_contents) { MemoryMappedFileError mmf_err = memory_map_file_32( alloc_cstring(temporary_allocator(), fullpath), - &memory_mapped_file + &t->memory_mapped_file, + copy_file_contents ); TokenizerInitError err = memory_mapped_file_error_map_to_tokenizer[mmf_err]; switch (mmf_err) { case MemoryMappedFile_None: - init_tokenizer_with_data(t, fullpath, memory_mapped_file.data, cast(isize)memory_mapped_file.size); + init_tokenizer_with_data(t, fullpath, t->memory_mapped_file.data, cast(isize)t->memory_mapped_file.size); break; case MemoryMappedFile_FileTooLarge: case MemoryMappedFile_Empty: -- cgit v1.2.3