refractoring and more semantic tokens

7 files changed, 485 insertions, 70 deletions

diff --git a/src/common/ast.odin b/src/common/ast.odin
index ca26726..ef98acb 100644
--- a/src/common/ast.odin
+++ b/src/common/ast.odin
@@ -15,7 +15,9 @@ keyword_map : map [string] bool =
          "bool" = true,
          "rawptr" = true,
          "any" = true,
-         "u32" = true};
+         "u32" = true,
+         "true" = true,
+         "false" = true};
 
 get_ast_node_string :: proc(node: ^ast.Node, src: [] byte) -> string {
     return string(src[node.pos.offset:node.end.offset]);
diff --git a/src/common/position.odin b/src/common/position.odin
index 7eb4173..220f347 100644
--- a/src/common/position.odin
+++ b/src/common/position.odin
@@ -7,7 +7,9 @@ import "core:odin/ast"
 
 /*
     This file handles the conversion between utf-16 and utf-8 offsets in the text document
- */
+*/
+
+//TODO(Optimize by calculating all the newlines at parse instead of calculating them)
 
 Position :: struct {
 	line: int,
@@ -53,6 +55,42 @@ get_absolute_position :: proc(position: Position, document_text: [] u8) -> (Abso
     return absolute, true;
 }
 
+get_relative_token_position :: proc(offset: int, document_text: [] u8, current_start: int) -> Position {
+
+    start_index := current_start;
+
+    data := document_text[start_index:];
+
+    i: int;
+
+    position: Position;
+
+    for i + start_index < offset {
+
+        r, w := utf8.decode_rune(data[i:]);
+
+        if r == '\n' { //\r?
+            position.character = 0;
+            position.line += 1;
+            i += 1;
+        }
+
+        else {
+            if r < 0x10000 {
+                position.character += 1;
+            }
+
+            else {
+                position.character += 2;
+            }
+
+            i += w;
+        }
+    }
+
+    return position;
+}
+
 /*
     Get the range of a token in utf16 space
  */
diff --git a/src/index/indexer.odin b/src/index/indexer.odin
index 2d80751..88e5419 100644
--- a/src/index/indexer.odin
+++ b/src/index/indexer.odin
@@ -46,7 +46,7 @@ indexer: Indexer;
 
 lookup :: proc(name: string, scope: string) -> (Symbol, bool) {
     symbol, ok := memory_index_lookup(&indexer.static_index, name, scope);
-    log.infof("lookup name: %v scope: %v, symbol %v", name, scope, symbol);
+    //log.infof("lookup name: %v scope: %v, symbol %v", name, scope, symbol);
     return symbol, ok;
 }
 
diff --git a/src/server/analysis.odin b/src/server/analysis.odin
index 0db0287..248849a 100644
--- a/src/server/analysis.odin
+++ b/src/server/analysis.odin
@@ -39,6 +39,7 @@ DocumentPositionContext :: struct {
 AstContext :: struct {
     locals: map [string] ^ast.Expr, //locals all the way to the document position
     globals: map [string] ^ast.Expr,
+    variables: map [string] bool,
     usings: [dynamic] string,
     file: ast.File,
     allocator: mem.Allocator,
@@ -55,6 +56,7 @@ make_ast_context :: proc(file: ast.File, imports: [] Package, package_name: stri
     ast_context := AstContext {
         locals = make(map [string] ^ast.Expr, 0, allocator),
         globals = make(map [string] ^ast.Expr, 0, allocator),
+        variables = make(map [string] bool, 0, allocator),
         usings = make([dynamic] string, allocator),
         file = file,
         imports = imports,
@@ -289,6 +291,10 @@ resolve_generic_function_symbol :: proc(ast_context: ^AstContext, params: []^ast
 
         for name in param.names {
 
+            if len(call_expr.args) >= i {
+                break;
+            }
+
             if poly, ok := name.derived.(Poly_Type); ok {
                 poly_map[poly.type.name] = call_expr.args[i];
             }
@@ -299,7 +305,7 @@ resolve_generic_function_symbol :: proc(ast_context: ^AstContext, params: []^ast
 
             if poly, ok := param.type.derived.(Poly_Type); ok {
 
-                if arg_eval, ok := resolve_type_expression(ast_context, call_expr.args[i], false); ok {
+                if arg_eval, ok := resolve_type_expression(ast_context, call_expr.args[i]); ok {
 
                     if value, ok := arg_eval.value.(index.SymbolGenericValue); ok {
                         resolve_poly_spec_node(ast_context, value.expr, poly.specialization, &poly_map);
@@ -413,7 +419,7 @@ resolve_function_overload :: proc(ast_context: ^AstContext, group: ast.Proc_Grou
 
     for arg_expr in group.args {
 
-        next_fn: if f, ok := resolve_type_expression(ast_context, arg_expr, false); ok {
+        next_fn: if f, ok := resolve_type_expression(ast_context, arg_expr); ok {
 
             if procedure, ok := f.value.(index.SymbolProcedureValue); ok {
 
@@ -423,7 +429,7 @@ resolve_function_overload :: proc(ast_context: ^AstContext, group: ast.Proc_Grou
 
                 for arg, i in call_expr.args {
 
-                    if eval_call_expr, ok := resolve_type_expression(ast_context, arg, false); ok {
+                    if eval_call_expr, ok := resolve_type_expression(ast_context, arg); ok {
 
                         #partial switch v in eval_call_expr.value {
                         case index.SymbolProcedureValue:
@@ -487,34 +493,38 @@ resolve_basic_lit :: proc(ast_context: ^AstContext, basic_lit: ast.Basic_Lit) ->
     return symbol, true;
 }
 
-resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expect_identifier := true) -> (index.Symbol, bool) {
+resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr) -> (index.Symbol, bool) {
+
+    if node == nil {
+        return {}, false;
+    }
 
     using ast;
 
     switch v in node.derived {
     case Ident:
-        return resolve_type_identifier(ast_context, v, expect_identifier);
+        return resolve_type_identifier(ast_context, v);
     case Basic_Lit:
         return resolve_basic_lit(ast_context, v);
     case Pointer_Type:
         if v2, ok := v.elem.derived.(ast.Pointer_Type); !ok {
-            return resolve_type_expression(ast_context, v.elem, false);
+            return resolve_type_expression(ast_context, v.elem);
         }
 
         else {
-            return resolve_type_expression(ast_context, node, false);
+            return resolve_type_expression(ast_context, node);
         }
 
     case Index_Expr:
-        indexed, ok := resolve_type_expression(ast_context, v.expr, false);
+        indexed, ok := resolve_type_expression(ast_context, v.expr);
 
         if generic, ok := indexed.value.(index.SymbolGenericValue); ok {
 
             switch c in generic.expr.derived {
             case Array_Type:
-                return resolve_type_expression(ast_context, c.elem, false);
+                return resolve_type_expression(ast_context, c.elem);
             case Dynamic_Array_Type:
-                return resolve_type_expression(ast_context, c.elem, false);
+                return resolve_type_expression(ast_context, c.elem);
 
             }
 
@@ -523,12 +533,20 @@ resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expec
         return index.Symbol {}, false;
     case Call_Expr:
         ast_context.call = node;
-        return resolve_type_expression(ast_context, v.expr, false);
+        return resolve_type_expression(ast_context, v.expr);
     case Implicit_Selector_Expr:
         log.info(v);
         return index.Symbol {}, false;
     case Selector_Expr:
 
+        if ident, ok := v.expr.derived.(Ident); ok {
+
+            if !resolve_ident_is_variable(ast_context, ident) && !resolve_ident_is_package(ast_context, ident) {
+                return {}, false;
+            }
+
+        }
+
         if selector, ok := resolve_type_expression(ast_context, v.expr); ok {
 
             ast_context.use_locals = false;
@@ -546,7 +564,7 @@ resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expec
 
                 for name, i in s.names {
                     if v.field != nil && strings.compare(name, v.field.name) == 0 {
-                        return resolve_type_expression(ast_context, s.types[i], false);
+                        return resolve_type_expression(ast_context, s.types[i]);
                     }
                 }
             case index.SymbolPackageValue:
@@ -572,7 +590,7 @@ resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expec
 
                     log.info(ptr);
 
-                    if symbol, ok := resolve_type_expression(ast_context, ptr.elem, false); ok {
+                    if symbol, ok := resolve_type_expression(ast_context, ptr.elem); ok {
 
                         #partial switch s2 in symbol.value {
                         case index.SymbolStructValue:
@@ -587,7 +605,7 @@ resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expec
 
                             for name, i in s2.names {
                                 if v.field != nil && strings.compare(name, v.field.name) == 0 {
-                                    return resolve_type_expression(ast_context, s2.types[i], false);
+                                    return resolve_type_expression(ast_context, s2.types[i]);
                                 }
                             }
 
@@ -617,7 +635,7 @@ resolve_type_expression :: proc(ast_context: ^AstContext, node: ^ast.Expr, expec
     Function recusively goes through the identifier until it hits a struct, enum, procedure literals, since you can
     have chained variable declarations. ie. a := foo { test =  2}; b := a; c := b;
  */
-resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expect_identifier := false) -> (index.Symbol, bool) {
+resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident) -> (index.Symbol, bool) {
 
     using ast;
 
@@ -628,14 +646,14 @@ resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expec
         case Ident:
             return resolve_type_identifier(ast_context, v);
         case Union_Type:
-            return make_symbol_union_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_union_from_ast(ast_context, v), true;
         case Enum_Type:
-            return make_symbol_enum_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_enum_from_ast(ast_context, v), true;
         case Struct_Type:
-            return make_symbol_struct_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_struct_from_ast(ast_context, v), true;
         case Proc_Lit:
             if !v.type.generic {
-                return make_symbol_procedure_from_ast(ast_context, v, node.name), !expect_identifier;
+                return make_symbol_procedure_from_ast(ast_context, v, node.name), true;
             }
             else {
                 return resolve_generic_function(ast_context, v);
@@ -643,15 +661,15 @@ resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expec
         case Proc_Group:
             return resolve_function_overload(ast_context, v);
         case Selector_Expr:
-            return resolve_type_expression(ast_context, local, false);
+            return resolve_type_expression(ast_context, local);
         case Array_Type:
             return make_symbol_generic_from_ast(ast_context, local), true;
         case Dynamic_Array_Type:
             return make_symbol_generic_from_ast(ast_context, local), true;
         case Index_Expr:
-            return resolve_type_expression(ast_context, local, false);
+            return resolve_type_expression(ast_context, local);
         case Pointer_Type:
-            return resolve_type_expression(ast_context, local, false);
+            return resolve_type_expression(ast_context, local);
         case:
             log.errorf("default type node kind: %T", v);
             return make_symbol_generic_from_ast(ast_context, local), true;
@@ -664,14 +682,14 @@ resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expec
         case Ident:
             return resolve_type_identifier(ast_context, v);
         case Struct_Type:
-            return make_symbol_struct_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_struct_from_ast(ast_context, v), true;
         case Union_Type:
-            return make_symbol_union_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_union_from_ast(ast_context, v), true;
         case Enum_Type:
-            return make_symbol_enum_from_ast(ast_context, v), !expect_identifier;
+            return make_symbol_enum_from_ast(ast_context, v), true;
         case Proc_Lit:
             if !v.type.generic {
-                return make_symbol_procedure_from_ast(ast_context, v, node.name), !expect_identifier;
+                return make_symbol_procedure_from_ast(ast_context, v, node.name), true;
             }
             else {
                 return resolve_generic_function(ast_context, v);
@@ -679,15 +697,15 @@ resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expec
         case Proc_Group:
             return resolve_function_overload(ast_context, v);
         case Selector_Expr:
-            return resolve_type_expression(ast_context, local, false);
+            return resolve_type_expression(ast_context, local);
         case Array_Type:
             return make_symbol_generic_from_ast(ast_context, global), true;
         case Dynamic_Array_Type:
             return make_symbol_generic_from_ast(ast_context, global), true;
         case Index_Expr:
-            return resolve_type_expression(ast_context, global, false);
+            return resolve_type_expression(ast_context, global);
         case Pointer_Type:
-            return resolve_type_expression(ast_context, local, false);
+            return resolve_type_expression(ast_context, local);
         case:
             log.errorf("default type node kind: %T", v);
             return make_symbol_generic_from_ast(ast_context, global), true;
@@ -776,6 +794,29 @@ resolve_type_identifier :: proc(ast_context: ^AstContext, node: ast.Ident, expec
     return index.Symbol {}, false;
 }
 
+resolve_ident_is_variable ::  proc(ast_context: ^AstContext, node: ast.Ident) -> bool {
+
+    if v, ok := ast_context.variables[node.name]; ok {
+        return v;
+    }
+
+    return false;
+}
+
+resolve_ident_is_package ::  proc(ast_context: ^AstContext, node: ast.Ident) -> bool {
+
+    for imp in ast_context.imports {
+
+        if strings.compare(imp.base, node.name) == 0 {
+            return true;
+        }
+
+    }
+
+    return false;
+}
+
+
 resolve_symbol_return :: proc(ast_context: ^AstContext, symbol: index.Symbol, ok := true) -> (index.Symbol, bool) {
 
     if !ok {
@@ -992,7 +1033,7 @@ get_generic_assignment :: proc(file: ast.File, value: ^ast.Expr, ast_context: ^A
 
         ast_context.call = value;
 
-        if symbol, ok := resolve_type_expression(ast_context, v.expr, false); ok {
+        if symbol, ok := resolve_type_expression(ast_context, v.expr); ok {
 
             if procedure, ok := symbol.value.(index.SymbolProcedureValue); ok {
 
@@ -1033,6 +1074,7 @@ get_locals_value_decl :: proc(file: ast.File, value_decl: ast.Value_Decl, ast_co
 
     if value_decl.type != nil {
         str := common.get_ast_node_string(value_decl.names[0], file.src);
+        ast_context.variables[str] = value_decl.is_mutable;
         ast_context.locals[str] = value_decl.type;
         return;
     }
@@ -1048,6 +1090,7 @@ get_locals_value_decl :: proc(file: ast.File, value_decl: ast.Value_Decl, ast_co
         for name, i in value_decl.names {
             str := common.get_ast_node_string(name, file.src);
             ast_context.locals[str] = results[i];
+            ast_context.variables[str] = value_decl.is_mutable;
         }
 
     }
@@ -1094,7 +1137,7 @@ get_locals_using_stmt :: proc(file: ast.File, stmt: ast.Using_Stmt, ast_context:
 
     for u in stmt.list {
 
-        if symbol, ok := resolve_type_expression(ast_context, u, false); ok {
+        if symbol, ok := resolve_type_expression(ast_context, u); ok {
 
             #partial switch v in symbol.value {
             case index.SymbolPackageValue:
@@ -1269,6 +1312,7 @@ get_locals :: proc(file: ast.File, function: ^ast.Node, ast_context: ^AstContext
                 if arg.type != nil {
                     str := common.get_ast_node_string(name, file.src);
                     ast_context.locals[str] = arg.type;
+                    ast_context.variables[str] = true;
                 }
             }
 
@@ -1288,8 +1332,10 @@ get_locals :: proc(file: ast.File, function: ^ast.Node, ast_context: ^AstContext
         get_locals_stmt(file, stmt, ast_context, document_position);
     }
 
-    log.info(ast_context.locals);
+}
 
+clear_locals :: proc(ast_context: ^AstContext) {
+    clear(&ast_context.locals);
 }
 
 get_definition_location :: proc(document: ^Document, position: common.Position) -> (common.Location, bool) {
@@ -1475,7 +1521,7 @@ get_completion_list :: proc(document: ^Document, position: common.Position) -> (
                     ast_context.current_package = ast_context.document_package;
                 }
 
-                if symbol, ok := resolve_type_expression(&ast_context, v.types[i], false); ok {
+                if symbol, ok := resolve_type_expression(&ast_context, v.types[i]); ok {
                     symbol.name = name;
                     symbol.type = .Field;
                     append(&symbols, symbol);
@@ -1515,7 +1561,7 @@ get_completion_list :: proc(document: ^Document, position: common.Position) -> (
 
             if ptr, ok := v.expr.derived.(ast.Pointer_Type); ok {
 
-                if symbol, ok := resolve_type_expression(&ast_context, ptr.elem, false); ok {
+                if symbol, ok := resolve_type_expression(&ast_context, ptr.elem); ok {
 
                     #partial switch s in symbol.value {
                     case index.SymbolStructValue:
@@ -1529,7 +1575,7 @@ get_completion_list :: proc(document: ^Document, position: common.Position) -> (
                                 ast_context.current_package = ast_context.document_package;
                             }
 
-                            if symbol, ok := resolve_type_expression(&ast_context, s.types[i], false); ok {
+                            if symbol, ok := resolve_type_expression(&ast_context, s.types[i]); ok {
                                 symbol.name = name;
                                 symbol.type = .Field;
                                 append(&symbols, symbol);
@@ -1556,11 +1602,6 @@ get_completion_list :: proc(document: ^Document, position: common.Position) -> (
             append(&items, item);
         }
 
-        //if there is no field we had to recover from bad expr and create a node (remove when parser can accept temp_allocator)
-        if position_context.field == nil {
-            common.free_ast(position_context.selector, context.allocator);
-        }
-
         list.items = items[:];
     }
 
@@ -1599,11 +1640,11 @@ get_completion_list :: proc(document: ^Document, position: common.Position) -> (
 
             ident.name = item.label;
 
-            if symbol, ok := resolve_type_identifier(&ast_context, ident^, true); ok {
+            if symbol, ok := resolve_type_identifier(&ast_context, ident^); ok {
                 items[i].kind = .Variable;
             }
 
-            else if symbol, ok := resolve_type_identifier(&ast_context, ident^, false); ok {
+            else if symbol, ok := resolve_type_identifier(&ast_context, ident^); ok {
                 items[i].kind = cast(CompletionItemKind)symbol.type;
             }
 
diff --git a/src/server/requests.odin b/src/server/requests.odin
index 259cafd..7b1eafb 100644
--- a/src/server/requests.odin
+++ b/src/server/requests.odin
@@ -261,12 +261,9 @@ request_initialize :: proc(params: json.Value, id: RequestId, config: ^common.Co
 
     config.signature_offset_support = initialize_params.capabilities.textDocument.signatureHelp.signatureInformation.parameterInformation.labelOffsetSupport;
 
-
     completionTriggerCharacters := [] string { "." };
     signatureTriggerCharacters := [] string { "(" };
 
-
-
     token_type := type_info_of(SemanticTokenTypes).variant.(runtime.Type_Info_Named).base.variant.(runtime.Type_Info_Enum);
     token_modifier := type_info_of(SemanticTokenModifiers).variant.(runtime.Type_Info_Named).base.variant.(runtime.Type_Info_Enum);
 
@@ -298,6 +295,7 @@ request_initialize :: proc(params: json.Value, id: RequestId, config: ^common.Co
                 },
                 semanticTokensProvider = SemanticTokensOptions {
                     //range = true,
+                    full = true,
                     legend = SemanticTokensLegend {
                         tokenTypes = token_types,
                         tokenModifiers = token_modifiers,
@@ -524,8 +522,30 @@ notification_did_save :: proc(params: json.Value, id: RequestId, config: ^common
 
 request_semantic_token_full :: proc(params: json.Value, id: RequestId, config: ^common.Config, writer: ^Writer) -> common.Error {
 
+    params_object, ok := params.value.(json.Object);
+
+    if !ok {
+        return .ParseError;
+    }
+
+    semantic_params: SemanticTokensParams;
+
+    if unmarshal(params, semantic_params, context.temp_allocator) != .None {
+        return .ParseError;
+    }
+
+    document := document_get(semantic_params.textDocument.uri);
+
+    if document == nil {
+        return .InternalError;
+    }
+
+    document_refresh(document, config, nil);
+
+    symbols := get_semantic_tokens(document);
+
     response := make_response_message(
-        params = nil,
+        params = symbols,
         id = id,
     );
 
diff --git a/src/server/semantic_tokens.odin b/src/server/semantic_tokens.odin
index 03f739b..8f6d66d 100644
--- a/src/server/semantic_tokens.odin
+++ b/src/server/semantic_tokens.odin
@@ -1,6 +1,11 @@
 package server
 
+import "core:odin/tokenizer"
+import "core:odin/ast"
+import "core:log"
+
 import "shared:common"
+import "shared:index"
 
 SemanticTokenTypes :: enum {
 	Namespace,
@@ -21,6 +26,7 @@ SemanticTokenTypes :: enum {
 };
 
 SemanticTokenModifiers :: enum {
+	None,
 	Declaration,
 	Definition,
 	Deprecated,
@@ -60,41 +66,348 @@ SemanticTokensRangeParams :: struct {
 };
 
 SemanticTokens :: struct {
-	data: [] uint,
+	data: [] u32,
 };
 
-SemanticTokenInternal :: struct {
-	line: uint,
-	column: uint,
-	length: uint,
+SemanticTokenBuilder :: struct {
+	current_function: ^ast.Node,
+	current_start: int,
+	tokens: [dynamic] u32,
 };
 
+make_token_builder :: proc(allocator := context.temp_allocator) -> SemanticTokenBuilder {
+
+	return {
+		tokens = make([dynamic]u32, context.temp_allocator),
+	};
+
+}
 
-convert_to_finished_tokens :: proc(tokens: [dynamic]SemanticTokenInternal) -> [] SemanticTokens {
-	return {};
+get_tokens :: proc(builder: SemanticTokenBuilder) -> SemanticTokens {
+	return {
+		data = builder.tokens[:],
+	};
 }
 
-get_semantic_tokens :: proc(document: ^Document) -> [] SemanticTokens {
+get_semantic_tokens :: proc(document: ^Document) -> SemanticTokens {
+
+	ast_context := make_ast_context(document.ast, document.imports, document.package_name, context.temp_allocator);
+	builder := make_token_builder();
+
+	get_globals(document.ast, &ast_context);
+
+	for decl in document.ast.decls {
+		write_semantic_tokens(cast(^ast.Node)decl, &builder, &ast_context);
+	}
+
+	return get_tokens(builder);
+}
+
+write_semantic_node :: proc(builder: ^SemanticTokenBuilder, node: ^ast.Node, src: []byte, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
+
+	position := common.get_relative_token_position(node.pos.offset, src, builder.current_start);
+
+	name := common.get_ast_node_string(node, src);
+
+	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(name), cast(u32)type, 0);
+
+	builder.current_start = node.pos.offset;
+}
 
-	tokens := make([dynamic]SemanticTokenInternal, context.temp_allocator);
+write_semantic_token :: proc(builder: ^SemanticTokenBuilder, token: tokenizer.Token, src: []byte, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
 
-	/*
-		Temp parse the document again, right now there is too many leaks that need to be fixued in the parser.
-	*/
+	position := common.get_relative_token_position(token.pos.offset, src, builder.current_start);
 
+	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(token.text), cast(u32)type, 0);
 
-	return {};
+	builder.current_start = token.pos.offset;
 }
 
 
-/*
-extract_semantic_tokens :: proc {
-	extract_semantic_tokens_node,
-	extract_semantic_tokens_dynamic_array,
-    extract_semantic_tokens_array,
+write_semantic_token_pos :: proc(builder: ^SemanticTokenBuilder, pos: tokenizer.Pos, name: string, src: []byte, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
+
+	position := common.get_relative_token_position(pos.offset, src, builder.current_start);
+
+	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(name), cast(u32)type, 0);
+
+	builder.current_start = pos.offset;
+}
+
+
+resolve_and_write_ident :: proc(node: ^ast.Node, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	n := node.derived.(ast.Ident);
+
+	ast_context.current_package = ast_context.document_package;
+
+	if resolve_ident_is_variable(ast_context, n) {
+		write_semantic_node(builder, node, ast_context.file.src, .Variable, .None);
+	}
+
+	else if symbol, ok := resolve_type_identifier(ast_context, n); ok {
+
+		#partial switch v in symbol.value {
+		case index.SymbolPackageValue:
+			write_semantic_node(builder, node, ast_context.file.src, .Namespace, .None);
+		case index.SymbolStructValue:
+			write_semantic_node(builder, node, ast_context.file.src, .Struct, .None);
+		case index.SymbolEnumValue:
+			write_semantic_node(builder, node, ast_context.file.src, .Enum, .None);
+		case index.SymbolUnionValue:
+			write_semantic_node(builder, node, ast_context.file.src, .Enum, .None);
+		case index.SymbolGenericValue:
+			#partial switch symbol.type {
+			case .Keyword:
+				write_semantic_node(builder, node, ast_context.file.src, .Keyword, .None);
+			}
+		}
+
+	}
+}
+
+resolve_and_write_expr :: proc(expr: ^ast.Expr, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	ast_context.current_package = ast_context.document_package;
+
+	if symbol, ok := resolve_type_expression(ast_context, expr); ok {
+
+		#partial switch v in symbol.value {
+		case index.SymbolPackageValue:
+			write_semantic_node(builder, expr, ast_context.file.src, .Namespace, .None);
+		case index.SymbolStructValue:
+			write_semantic_node(builder, expr, ast_context.file.src, .Struct, .None);
+		case index.SymbolEnumValue:
+			write_semantic_node(builder, expr, ast_context.file.src, .Enum, .None);
+		case index.SymbolUnionValue:
+			write_semantic_node(builder, expr, ast_context.file.src, .Enum, .None);
+		case index.SymbolGenericValue:
+			#partial switch symbol.type {
+			case .Keyword:
+				write_semantic_node(builder, expr, ast_context.file.src, .Keyword, .None);
+			}
+		}
+
+	}
+
+}
+
+write_semantic_tokens :: proc {
+	write_semantic_tokens_node,
+	write_semantic_tokens_dynamic_array,
+    write_semantic_tokens_array,
 };
 
-extract_semantic_tokens_node :: proc() {
+write_semantic_tokens_array :: proc(array: $A/[]^$T, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+    for elem, i in array {
+        write_semantic_tokens(elem, builder, ast_context);
+    }
+
+}
+
+write_semantic_tokens_dynamic_array :: proc(array: $A/[dynamic]^$T, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+    for elem, i in array {
+        write_semantic_tokens(elem, builder, ast_context);
+    }
+
+}
+
+write_semantic_tokens_node :: proc(node: ^ast.Node, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	using ast;
+
+    if node == nil {
+        return;
+    }
+
+	switch n in node.derived {
+	case Ident:
+		resolve_and_write_ident(node, builder, ast_context);
+	case Selector_Expr:
+		write_semantic_selector(cast(^Selector_Expr)node, builder, ast_context);
+	case Pointer_Type:
+		write_semantic_token_pos(builder, node.pos, "^", ast_context.file.src, .Operator, .None);
+		write_semantic_tokens(n.elem, builder, ast_context);
+	case Value_Decl:
+		write_semantic_tokens_value_decl(n, builder, ast_context);
+	case Block_Stmt:
+		write_semantic_tokens(n.stmts, builder, ast_context);
+	case Expr_Stmt:
+		write_semantic_tokens(n.expr, builder, ast_context);
+	case Range_Stmt:
+		get_locals_at(builder.current_function, node, ast_context);
+		write_semantic_token_pos(builder, n.for_pos, "for", ast_context.file.src, .Keyword, .None);
+		if n.val0 != nil {
+			if ident, ok := n.val0.derived.(Ident); ok {
+				write_semantic_node(builder, n.val0, ast_context.file.src, .Variable, .None);
+			}
+		}
+
+		if n.val1 != nil {
+			if ident, ok := n.val1.derived.(Ident); ok {
+				write_semantic_node(builder, n.val1, ast_context.file.src, .Variable, .None);
+			}
+		}
+
+		write_semantic_token_pos(builder, n.in_pos, "in", ast_context.file.src, .Keyword, .None);
+
+		write_semantic_tokens(n.expr, builder, ast_context);
+
+		write_semantic_tokens(n.body, builder, ast_context);
+	case:
+		//log.infof("unhandled write node %v", n);
+	}
+
+
+
+}
+
+write_semantic_tokens_value_decl :: proc(value_decl: ast.Value_Decl, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	using ast;
+
+	for name, i in value_decl.names {
+		if ident, ok := name.derived.(Ident); ok {
+
+			if value_decl.type != nil {
+
+			}
+
+			else {
+
+				if len(value_decl.values) == 1 {
+
+					switch v in value_decl.values[0].derived {
+					case ast.Struct_Type:
+						write_semantic_node(builder, name, ast_context.file.src, .Struct, .None);
+						write_semantic_token_pos(builder, v.pos, "struct", ast_context.file.src, .Keyword, .None);
+					case ast.Enum_Type:
+						write_semantic_node(builder, name, ast_context.file.src, .Enum, .None);
+						write_semantic_token_pos(builder, v.pos, "enum", ast_context.file.src, .Keyword, .None);
+						write_semantic_enum_fields(v, builder, ast_context);
+					case ast.Proc_Lit:
+						write_semantic_node(builder, name, ast_context.file.src, .Function, .None);
+						write_semantic_token_pos(builder, v.pos, "proc", ast_context.file.src, .Keyword, .None);
+						write_semantic_proc_type(v.type, builder, ast_context);
+
+						last_function := builder.current_function;
+						builder.current_function = value_decl.values[0];
+						get_locals_at(builder.current_function, builder.current_function, ast_context);
+						write_semantic_tokens(v.body, builder, ast_context);
+						builder.current_function = last_function;
+					}
+
+				}
+
+				else {
+
+				}
+
+			}
+
+		}
+	}
 
 }
-*/
\ No newline at end of file
+
+write_semantic_proc_type :: proc(node: ^ast.Proc_Type, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	using ast;
+
+	if node == nil {
+		return;
+	}
+
+	if node.params != nil {
+
+		for param in node.params.list {
+
+			for name in param.names {
+
+				if ident, ok := name.derived.(Ident); ok {
+					write_semantic_node(builder, name, ast_context.file.src, .Parameter, .None);
+				}
+
+			}
+
+			write_semantic_tokens(param.type, builder, ast_context);
+		}
+
+	}
+
+	if node.results != nil {
+
+		for result in node.results.list {
+
+
+			for name in result.names {
+
+				if ident, ok := name.derived.(Ident); ok {
+					//write_semantic_node(builder, name, ast_context.file.src, .Parameter, .None);
+				}
+
+			}
+
+			write_semantic_tokens(result.type, builder, ast_context);
+
+		}
+
+	}
+
+}
+
+write_semantic_enum_fields :: proc(node: ast.Enum_Type, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	using ast;
+
+	if node.fields == nil {
+		return;
+	}
+
+	for field in node.fields {
+
+		if ident, ok := field.derived.(Ident); ok {
+			write_semantic_node(builder, field, ast_context.file.src, .EnumMember, .None);
+		}
+
+	}
+
+}
+
+write_semantic_selector :: proc(selector: ^ast.Selector_Expr, builder: ^SemanticTokenBuilder, ast_context: ^AstContext) {
+
+	using ast;
+
+
+	if ident, ok := selector.expr.derived.(Ident); ok {
+		resolve_and_write_ident(selector.expr, builder, ast_context); //base
+		resolve_and_write_expr(selector, builder, ast_context); //field
+	}
+
+	else {
+
+	}
+
+
+
+}
+
+get_locals_at :: proc(function: ^ast.Node, position: ^ast.Node, ast_context: ^AstContext) {
+
+	clear_locals(ast_context);
+
+	if function == nil {
+		return;
+	}
+
+	if position == nil {
+		return;
+	}
+
+	document_position := DocumentPositionContext {
+		position = position.end.offset,
+	};
+
+	get_locals(ast_context.file, function, ast_context, &document_position);
+}
\ No newline at end of file
diff --git a/src/server/types.odin b/src/server/types.odin
index 40d7a5e..4d81447 100644
--- a/src/server/types.odin
+++ b/src/server/types.odin
@@ -22,6 +22,7 @@ ResponseParams :: union {
     CompletionList,
 	SignatureHelp,
 	[] DocumentSymbol,
+	SemanticTokens,
 };
 
 ResponseMessage :: struct {
@@ -187,7 +188,7 @@ DidOpenTextDocumentParams :: struct {
 
 DocumentSymbolParams :: struct  {
 	textDocument: TextDocumentIdentifier,
-}
+};
 
 DidChangeTextDocumentParams :: struct {
 	textDocument: VersionedTextDocumentIdentifier,