From 6ac0fb80a6eab6ba28838e8e577dc8ae8cba06a7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 May 2020 12:32:11 +0100 Subject: Minor tokenizer performance improvements --- src/parser.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/parser.cpp') diff --git a/src/parser.cpp b/src/parser.cpp index 3d2041183..5bdadfd9a 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -4284,8 +4284,15 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) { } isize file_size = f->tokenizer.end - f->tokenizer.start; - isize init_token_cap = cast(isize)gb_max(next_pow2(cast(i64)(file_size/2ll)), 16); + + // NOTE(bill): Determine allocation size required for tokens + isize token_cap = file_size/3ll; + isize pow2_cap = gb_max(cast(isize)prev_pow2(cast(i64)token_cap)/2, 16); + token_cap = ((token_cap + pow2_cap-1)/pow2_cap) * pow2_cap; + + isize init_token_cap = gb_max(token_cap, 16); array_init(&f->tokens, heap_allocator(), 0, gb_max(init_token_cap, 16)); + isize cap0 = f->tokens.capacity; if (err == TokenizerInit_Empty) { Token token = {Token_EOF}; @@ -4314,8 +4321,8 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) { f->prev_token = f->tokens[f->curr_token_index]; f->curr_token = f->tokens[f->curr_token_index]; - array_init(&f->comments, heap_allocator()); - array_init(&f->imports, heap_allocator()); + array_init(&f->comments, heap_allocator(), 0, 0); + array_init(&f->imports, heap_allocator(), 0, 0); f->curr_proc = nullptr; -- cgit v1.2.3