complete refractor of the project into packages

5 files changed, 242 insertions, 0 deletions

diff --git a/src/index/build.odin b/src/index/build.odin
new file mode 100644
index 0000000..d52cbca
--- /dev/null
+++ b/src/index/build.odin
@@ -0,0 +1,86 @@
+package index
+
+import "core:path/filepath"
+import "core:os"
+import "core:fmt"
+import "core:odin/parser"
+import "core:odin/ast"
+import "core:odin/tokenizer"
+
+import "shared:common"
+
+/*
+    Not fully sure how to handle rebuilding, but one thing is for sure, dynamic indexing has to have a background thread
+    rebuilding every minute or less to fight against stale information
+ */
+
+
+//test version for static indexing
+
+symbol_collection: SymbolCollection;
+
+build_static_index :: proc(allocator := context.allocator, config: ^common.Config) {
+
+    //right now just collect the symbols from core
+
+    core_path := config.collections["core"];
+
+
+    symbol_collection = make_symbol_collection(allocator);
+
+    walk_static_index_build := proc(info: os.File_Info, in_err: os.Errno) -> (err: os.Errno, skip_dir: bool) {
+
+        if info.is_dir {
+            return 0, false;
+        }
+
+        //fmt.println(info.fullpath);
+
+        //bit worried about using temp allocator here since we might overwrite all our temp allocator budget
+        data, ok := os.read_entire_file(info.fullpath, context.allocator);
+
+        if !ok {
+            fmt.println("failed to read");
+            return 1, false;
+        }
+
+        p := parser.Parser {
+            err  = no_error_handler,
+            warn = no_warning_handler,
+        };
+
+        file := ast.File {
+            fullpath = info.fullpath,
+            src = data,
+        };
+
+        parser.parse_file(&p, &file);
+
+        uri := common.create_uri(info.fullpath, context.temp_allocator);
+
+        collect_symbols(&symbol_collection, file, uri.uri);
+
+        delete(data);
+
+
+        return 0, false;
+    };
+
+    filepath.walk(core_path, walk_static_index_build);
+
+    indexer.static_index = make_memory_index(symbol_collection);
+}
+
+
+no_error_handler :: proc(pos: tokenizer.Pos, msg: string, args: ..any) {
+
+}
+
+no_warning_handler :: proc(pos: tokenizer.Pos, msg: string, args: ..any) {
+
+}
+
+
+
+
+
diff --git a/src/index/file_index.odin b/src/index/file_index.odin
new file mode 100644
index 0000000..fe787e5
--- /dev/null
+++ b/src/index/file_index.odin
@@ -0,0 +1,7 @@
+package index
+
+/*
+    This is indexer for static files operating on a file database to index symbols and files.
+
+    NOTE(Daniel, Let's be honest probably will not be made any time soon)
+ */
\ No newline at end of file
diff --git a/src/index/indexer.odin b/src/index/indexer.odin
new file mode 100644
index 0000000..46a2192
--- /dev/null
+++ b/src/index/indexer.odin
@@ -0,0 +1,54 @@
+package index
+
+import "core:odin/ast"
+import "core:fmt"
+import "core:strings"
+
+
+/*
+    Concept ideas:
+
+    static indexing:
+
+    is responsible for implementing the indexing of symbols for static files.
+
+    This is to solve the scaling problem of large projects with many files and symbols, as most of these files will be static.
+
+    Possible scopes for static files:
+    global scope (we don't have hiarachy of namespaces and therefore only need to look at the global scope)
+
+    Scopes not part of the indexer:
+    function scope, file scope, package scope(these are only relevant for dynamic active files in your project, that use the ast instead of indexing)
+
+    Potential features:
+        Allow for saving the indexer, instead of recreating it everytime the lsp starts(but you would have to account for stale data).
+
+
+    dynamic indexing:
+
+    When the user modifies files we need some smaller index to handle everything the user is using right now. This will allow
+    us to rebuild parts of the index without too much of a performance hit.
+
+    This index is first searched and if nothing is found look in the static index.
+
+    interface ideas:
+
+    index_search_fuzzy(symbol: string, scope: [] string) -> [] SymbolResult
+
+    TODO(Daniel, Look into data structure for fuzzy searching)
+
+ */
+
+
+Indexer :: struct {
+    static_index: MemoryIndex,
+};
+
+indexer: Indexer;
+
+
+lookup :: proc(id: string) -> (Symbol, bool) {
+    return memory_index_lookup(&indexer.static_index, id);
+}
+
+//indexer_fuzzy_search :: proc(name: string, scope: [] string, )
diff --git a/src/index/memory_index.odin b/src/index/memory_index.odin
new file mode 100644
index 0000000..c5ce416
--- /dev/null
+++ b/src/index/memory_index.odin
@@ -0,0 +1,29 @@
+package index
+
+import "core:hash"
+
+/*
+    This is a in memory index designed for the dynamic indexing of symbols and files.
+    Designed for few files and should be fast at rebuilding.
+
+    Right now the implementation is quite naive.
+ */
+MemoryIndex :: struct {
+    collection: SymbolCollection,
+};
+
+
+make_memory_index :: proc(collection: SymbolCollection) -> MemoryIndex {
+
+    return MemoryIndex {
+        collection = collection,
+    };
+
+}
+
+memory_index_lookup :: proc(index: ^MemoryIndex, id: string) -> (Symbol, bool) {
+
+    hashed := hash.murmur64(transmute([]u8)id);
+
+    return index.collection.symbols[hashed];
+}
\ No newline at end of file
diff --git a/src/index/symbol.odin b/src/index/symbol.odin
new file mode 100644
index 0000000..2cbd58c
--- /dev/null
+++ b/src/index/symbol.odin
@@ -0,0 +1,66 @@
+package index
+
+import "core:odin/ast"
+import "core:hash"
+import "core:strings"
+import "core:mem"
+
+import "shared:common"
+
+Symbol :: struct {
+    id: u64,
+    range: common.Range,
+    uri: string,
+};
+
+SymbolCollection :: struct {
+    allocator: mem.Allocator,
+    symbols: map[u64] Symbol,
+    unique_strings: map[u64] string,
+};
+
+make_symbol_collection :: proc(allocator := context.allocator) -> SymbolCollection {
+    return SymbolCollection {
+        allocator = allocator,
+        symbols = make(map[u64] Symbol, 16, allocator),
+        unique_strings = make(map[u64] string, 16, allocator),
+    };
+}
+
+collect_symbols :: proc(collection: ^SymbolCollection, file: ast.File, uri: string) -> common.Error {
+
+    for decl in file.decls {
+
+        if value_decl, ok := decl.derived.(ast.Value_Decl); ok {
+
+            name := string(file.src[value_decl.names[0].pos.offset:value_decl.names[0].end.offset]);
+
+            if len(value_decl.values) == 1 {
+
+                if proc_lit, ok := value_decl.values[0].derived.(ast.Proc_Lit); ok {
+
+                    symbol: Symbol;
+
+                    symbol.range = common.get_token_range(proc_lit, file.src);
+
+                    uri_id := hash.murmur64(transmute([]u8)uri);
+
+                    if _, ok := collection.unique_strings[uri_id]; !ok {
+                        collection.unique_strings[uri_id] = strings.clone(uri);
+                    }
+
+                    symbol.uri = collection.unique_strings[uri_id];
+
+                    id := hash.murmur64(transmute([]u8)strings.concatenate({file.pkg_name, name}, context.temp_allocator));
+
+                    collection.symbols[id] = symbol;
+
+                }
+
+            }
+
+        }
+    }
+
+    return .None;
+}