work on the new semantic tokens.

2 files changed, 364 insertions, 22 deletions

diff --git a/src/server/requests.odin b/src/server/requests.odin
index 8d965c7..c4bd7b9 100644
--- a/src/server/requests.odin
+++ b/src/server/requests.odin
@@ -589,7 +589,6 @@ request_initialized :: proc (task: ^common.Task) {
 }
 
 request_shutdown :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -607,7 +606,6 @@ request_shutdown :: proc (task: ^common.Task) {
 }
 
 request_definition :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -648,7 +646,6 @@ request_definition :: proc (task: ^common.Task) {
 }
 
 request_completion :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -692,7 +689,6 @@ request_completion :: proc (task: ^common.Task) {
 }
 
 request_signature_help :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -733,7 +729,6 @@ request_signature_help :: proc (task: ^common.Task) {
 }
 
 request_format_document :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -786,7 +781,6 @@ notification_exit :: proc (task: ^common.Task) {
 }
 
 notification_did_open :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -818,7 +812,6 @@ notification_did_open :: proc (task: ^common.Task) {
 }
 
 notification_did_change :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -846,7 +839,6 @@ notification_did_change :: proc (task: ^common.Task) {
 }
 
 notification_did_close :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -952,7 +944,6 @@ notification_did_save :: proc (task: ^common.Task) {
 }
 
 request_semantic_token_full :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -1001,7 +992,6 @@ request_semantic_token_full :: proc (task: ^common.Task) {
 }
 
 request_semantic_token_range :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -1040,7 +1030,6 @@ request_semantic_token_range :: proc (task: ^common.Task) {
 }
 
 request_document_symbols :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
@@ -1075,7 +1064,6 @@ request_document_symbols :: proc (task: ^common.Task) {
 }
 
 request_hover :: proc (task: ^common.Task) {
-
 	info := get_request_info(task);
 
 	using info;
diff --git a/src/server/semantic_tokens.odin b/src/server/semantic_tokens.odin
index f7d01dd..095bb91 100644
--- a/src/server/semantic_tokens.odin
+++ b/src/server/semantic_tokens.odin
@@ -81,7 +81,7 @@ SemanticTokenBuilder :: struct {
 
 make_token_builder :: proc(allocator := context.temp_allocator) -> SemanticTokenBuilder {
 	return {
-		tokens = make([dynamic]u32, context.temp_allocator),
+		tokens = make([dynamic]u32, 1000, context.temp_allocator),
 	};
 }
 
@@ -100,11 +100,15 @@ get_semantic_tokens :: proc(document: ^common.Document, range: common.Range) ->
 		write_semantic_token(&builder, document.ast.pkg_token, document.ast.src, .Keyword, .None);
 	}
 
-	resolve_entire_file(document, context.temp_allocator);
+	ast_context := make_ast_context(document.ast, document.imports, document.package_name, document.uri.uri, &document.symbol_cache);
+
+	//resolve_entire_file(document, &ast_context, context.temp_allocator);
+
+	ast_context.current_package = ast_context.document_package;
 
 	for decl in document.ast.decls {
 		if range.start.line <= decl.pos.line && decl.end.line <= range.end.line {
-			
+			visit(decl, &builder, &ast_context);
 		}
 	}
 
@@ -113,27 +117,377 @@ get_semantic_tokens :: proc(document: ^common.Document, range: common.Range) ->
 
 write_semantic_node :: proc(builder: ^SemanticTokenBuilder, node: ^ast.Node, src: string, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
 	position := common.get_relative_token_position(node.pos.offset, transmute([]u8)src, builder.current_start);
-
 	name := common.get_ast_node_string(node, src);
-
 	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(name), cast(u32)type, 0);
-
 	builder.current_start = node.pos.offset;
 }
 
 write_semantic_token :: proc(builder: ^SemanticTokenBuilder, token: tokenizer.Token, src: string, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
 	position := common.get_relative_token_position(token.pos.offset, transmute([]u8)src, builder.current_start);
-
 	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(token.text), cast(u32)type, 0);
-
 	builder.current_start = token.pos.offset;
 }
 
 write_semantic_string :: proc(builder: ^SemanticTokenBuilder, pos: tokenizer.Pos, name: string, src: string, type: SemanticTokenTypes, modifier: SemanticTokenModifiers) {
 	position := common.get_relative_token_position(pos.offset, transmute([]u8)src, builder.current_start);
-
 	append(&builder.tokens, cast(u32)position.line, cast(u32)position.character, cast(u32)len(name), cast(u32)type, 0);
-
 	builder.current_start = pos.offset;
 }
 
+visit :: proc {
+	visit_node,
+	visit_dynamic_array,
+	visit_array,
+	visit_stmt,
+};
+
+visit_array :: proc(array: $A/[]^$T, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	for elem, i in array {
+		visit(elem, builder, ast_context);
+	}
+}
+
+visit_dynamic_array :: proc(array: $A/[dynamic]^$T, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	for elem, i in array {
+		visit(elem, builder, ast_context);
+	}
+}
+
+visit_stmt :: proc(node: ^ast.Stmt, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	visit_node(node, builder, ast_context);
+}
+
+visit_node :: proc(node: ^ast.Node, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using ast;
+
+	if node == nil {
+		return;
+	}
+
+	switch n in node.derived {
+	case Ellipsis:
+		write_semantic_string(builder, node.pos, "..", ast_context.file.src, .Operator, .None);
+		visit(n.expr, builder, ast_context);
+	case Ident:
+		if true {
+			write_semantic_node(builder, node, ast_context.file.src, .Variable, .None);
+			return;
+		}
+		if symbol, ok := analysis.lookup_symbol_cache(ast_context, n); ok {
+			if symbol.type == .Variable {
+				write_semantic_node(builder, node, ast_context.file.src, .Variable, .None);
+			}
+			
+			#partial switch v in symbol.value { 
+			case index.SymbolPackageValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Namespace, .None);
+			case index.SymbolStructValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Struct, .None);
+			case index.SymbolEnumValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Enum, .None);
+			case index.SymbolUnionValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Enum, .None);
+			case index.SymbolProcedureValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Function, .None);
+			case index.SymbolProcedureGroupValue:
+				write_semantic_node(builder, node, ast_context.file.src, .Function, .None);
+			}
+		}
+	case Selector_Expr:
+		visit_selector(cast(^Selector_Expr)node, builder, ast_context);
+	case Pointer_Type:
+		write_semantic_string(builder, node.pos, "^", ast_context.file.src, .Operator, .None);
+		visit(n.elem, builder, ast_context);
+	case Value_Decl:
+		visit_value_decl(n, builder, ast_context);
+	case Block_Stmt:
+		visit(n.stmts, builder, ast_context);
+	case Expr_Stmt:
+		visit(n.expr, builder, ast_context);
+	case Range_Stmt:
+		write_semantic_string(builder, n.for_pos, "for", ast_context.file.src, .Keyword, .None);
+
+		for val in n.vals {
+			if ident, ok := val.derived.(Ident); ok {
+				write_semantic_node(builder, val, ast_context.file.src, .Variable, .None);
+			}
+		}
+
+		write_semantic_string(builder, n.in_pos, "in", ast_context.file.src, .Keyword, .None);
+		visit(n.expr, builder, ast_context);
+		visit(n.body, builder, ast_context);
+	case If_Stmt:
+		write_semantic_string(builder, n.if_pos, "if", ast_context.file.src, .Keyword, .None);
+		visit(n.init, builder, ast_context);
+		visit(n.cond, builder, ast_context);
+		visit(n.body, builder, ast_context);
+		if n.else_stmt != nil {
+			write_semantic_string(builder, n.else_pos, "else", ast_context.file.src, .Keyword, .None);
+			visit(n.else_stmt, builder, ast_context);
+		}
+	case For_Stmt:
+		write_semantic_string(builder, n.for_pos, "for", ast_context.file.src, .Keyword, .None);
+		visit(n.init, builder, ast_context);
+		visit(n.cond, builder, ast_context);
+		visit(n.post, builder, ast_context);
+		visit(n.body, builder, ast_context);
+	case Switch_Stmt:
+		write_semantic_string(builder, n.switch_pos, "switch", ast_context.file.src, .Keyword, .None);
+		visit(n.init, builder, ast_context);
+		visit(n.cond, builder, ast_context);
+		visit(n.body, builder, ast_context);
+	case Type_Switch_Stmt:
+		write_semantic_string(builder, n.switch_pos, "switch", ast_context.file.src, .Keyword, .None);
+		visit(n.tag, builder, ast_context);
+		visit(n.expr, builder, ast_context);
+		visit(n.body, builder, ast_context);
+	case Assign_Stmt:
+		for l in n.lhs {
+			if ident, ok := l.derived.(Ident); ok {
+				write_semantic_node(builder, l, ast_context.file.src, .Variable, .None);
+			} else {
+				visit(l, builder, ast_context);
+			}
+		}
+
+		visit_token_op(builder, n.op, ast_context.file.src);
+		visit(n.rhs, builder, ast_context);
+	case Case_Clause:
+		write_semantic_string(builder, n.case_pos, "case", ast_context.file.src, .Keyword, .None);
+		visit(n.list, builder, ast_context);
+		visit(n.body, builder, ast_context);
+	case Call_Expr:
+		if ident, ok := n.expr.derived.(Ident); ok {
+			write_semantic_node(builder, n.expr, ast_context.file.src, .Function, .None);
+		} else {
+			visit(n.expr, builder, ast_context);
+		}
+		visit(n.args, builder, ast_context);
+	case Implicit_Selector_Expr:
+		write_semantic_node(builder, n.field, ast_context.file.src, .Enum, .None);
+	case Array_Type:
+		visit(n.elem, builder, ast_context);
+	case Binary_Expr:
+		visit(n.left, builder, ast_context);
+		visit_token_op(builder, n.op, ast_context.file.src);
+		visit(n.right, builder, ast_context);
+	case Comp_Lit:
+		visit(n.type, builder, ast_context);
+		visit(n.elems, builder, ast_context);
+	case Struct_Type:
+		write_semantic_string(builder, n.pos, "struct", ast_context.file.src, .Keyword, .None);
+		visit_struct_fields(n, builder, ast_context);
+	case Type_Assertion:
+		visit(n.expr, builder, ast_context);
+		visit(n.type, builder, ast_context);
+	case Type_Cast:
+		write_semantic_string(builder, n.pos, "cast", ast_context.file.src, .Keyword, .None);
+		visit(n.type, builder, ast_context);
+		visit(n.expr, builder, ast_context);
+	case Paren_Expr:
+		visit(n.expr, builder, ast_context);
+	case Deref_Expr:
+		visit(n.expr, builder, ast_context);
+	case Return_Stmt:
+		write_semantic_string(builder, n.pos, "return", ast_context.file.src, .Keyword, .None);
+		visit(n.results, builder, ast_context);
+	case Dynamic_Array_Type:
+		write_semantic_string(builder, n.dynamic_pos, "dynamic", ast_context.file.src, .Keyword, .None);
+		visit(n.elem, builder, ast_context);
+	case Field_Value:
+		if ident, ok := n.field.derived.(Ident); ok {
+			write_semantic_node(builder, n.field, ast_context.file.src, .Property, .None);
+		}
+
+		visit(n.value, builder, ast_context);
+	case Index_Expr:
+		visit(n.expr, builder, ast_context);
+		visit(n.index, builder, ast_context);
+	case Basic_Lit:
+		visit_basic_lit(n, builder, ast_context);
+	case Unary_Expr:
+		visit(n.expr, builder, ast_context);
+	case Implicit:
+	case Slice_Expr:
+		visit(n.expr, builder, ast_context);
+	case Using_Stmt:
+		write_semantic_string(builder, n.pos, "using", ast_context.file.src, .Keyword, .None);
+		visit(n.list, builder, ast_context);
+	case Map_Type:
+		write_semantic_string(builder, n.tok_pos, "map", ast_context.file.src, .Keyword, .None);
+		visit(n.key, builder, ast_context);
+		visit(n.value, builder, ast_context);
+	case Defer_Stmt:
+		write_semantic_string(builder, n.pos, "defer", ast_context.file.src, .Keyword, .None);
+		visit(n.stmt, builder, ast_context);
+	case Import_Decl:
+		write_semantic_token(builder, n.import_tok, ast_context.file.src, .Keyword, .None);
+
+		if n.name.text != "" {
+			write_semantic_token(builder, n.name, ast_context.file.src, .Namespace, .None);
+		}
+
+		write_semantic_token(builder, n.relpath, ast_context.file.src, .String, .None);
+	case:
+		log.warnf("unhandled write node %v", n);
+	}
+}
+
+visit_basic_lit :: proc(basic_lit: ast.Basic_Lit, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using analysis;
+
+	if symbol, ok := resolve_basic_lit(ast_context, basic_lit); ok {
+
+		if generic, ok := symbol.value.(index.SymbolGenericValue); ok {
+
+			ident := generic.expr.derived.(ast.Ident);
+
+			if ident.name == "string" {
+				write_semantic_node(builder, generic.expr, ast_context.file.src, .String, .None);
+			} else if ident.name == "int" {
+				write_semantic_node(builder, generic.expr, ast_context.file.src, .Number, .None);
+			} else {
+			}
+		}
+	}
+}
+
+visit_value_decl :: proc(value_decl: ast.Value_Decl, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using ast;
+
+	if value_decl.type != nil {
+		for name in value_decl.names {
+			write_semantic_node(builder, name, ast_context.file.src, .Variable, .None);
+		}
+
+		visit(value_decl.type, builder, ast_context);
+
+		return;
+	}
+
+	if len(value_decl.values) == 1 {
+		switch v in value_decl.values[0].derived {
+		case Struct_Type:
+			write_semantic_node(builder, value_decl.names[0], ast_context.file.src, .Struct, .None);
+			write_semantic_string(builder, v.pos, "struct", ast_context.file.src, .Keyword, .None);
+			visit_struct_fields(v, builder, ast_context);
+		case Enum_Type:
+			write_semantic_node(builder, value_decl.names[0], ast_context.file.src, .Enum, .None);
+			write_semantic_string(builder, v.pos, "enum", ast_context.file.src, .Keyword, .None);
+			visit_enum_fields(v, builder, ast_context);
+		case Proc_Group:
+			write_semantic_node(builder, value_decl.names[0], ast_context.file.src, .Function, .None);
+			write_semantic_string(builder, v.pos, "proc", ast_context.file.src, .Keyword, .None);
+			for arg in v.args {
+				if ident, ok := arg.derived.(Ident); ok {
+					write_semantic_node(builder, arg, ast_context.file.src, .Function, .None);
+				}
+			}
+		case Proc_Lit:
+			write_semantic_node(builder, value_decl.names[0], ast_context.file.src, .Function, .None);
+			write_semantic_string(builder, v.pos, "proc", ast_context.file.src, .Keyword, .None);
+			visit_proc_type(v.type, builder, ast_context);
+
+			visit(v.body, builder, ast_context);
+		case:
+			for name in value_decl.names {
+				write_semantic_node(builder, name, ast_context.file.src, .Variable, .None);
+			}
+
+			visit(value_decl.values[0], builder, ast_context);
+		}
+	} else {
+		for name in value_decl.names {
+			write_semantic_node(builder, name, ast_context.file.src, .Variable, .None);
+		}
+
+		for value in value_decl.values {
+			visit(value, builder, ast_context);
+		}
+	}
+}
+
+visit_token_op :: proc(builder: ^SemanticTokenBuilder, token: tokenizer.Token, src: string) {
+	if token.text == "in" {
+		write_semantic_string(builder, token.pos, token.text, src, .Keyword, .None);
+	} else {
+		write_semantic_string(builder, token.pos, token.text, src, .Operator, .None);
+	}
+}
+
+visit_proc_type :: proc(node: ^ast.Proc_Type, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using ast;
+
+	if node == nil {
+		return;
+	}
+
+	if node.params != nil {
+		for param in node.params.list {
+			for name in param.names {
+				if ident, ok := name.derived.(Ident); ok {
+					write_semantic_node(builder, name, ast_context.file.src, .Parameter, .None);
+				}
+			}
+
+			visit(param.type, builder, ast_context);
+		}
+	}
+
+	if node.results != nil {
+		for result in node.results.list {
+			for name in result.names {
+				if ident, ok := name.derived.(Ident); ok {
+					//write_semantic_node(builder, name, ast_context.file.src, .Parameter, .None);
+				}
+			}
+
+			visit(result.type, builder, ast_context);
+		}
+	}
+}
+
+visit_enum_fields :: proc(node: ast.Enum_Type, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using ast;
+
+	if node.fields == nil {
+		return;
+	}
+
+	for field in node.fields {
+
+		if ident, ok := field.derived.(Ident); ok {
+			write_semantic_node(builder, field, ast_context.file.src, .EnumMember, .None);
+		}
+		else if f, ok := field.derived.(Field_Value); ok {
+			if ident, ok := f.field.derived.(Ident); ok {
+				write_semantic_node(builder, f.field, ast_context.file.src, .EnumMember, .None);
+			}
+			visit(f.value, builder, ast_context);
+		}
+	}
+}
+
+visit_struct_fields :: proc(node: ast.Struct_Type, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using ast;
+
+	if node.fields == nil {
+		return;
+	}
+
+	for field in node.fields.list {
+
+		for name in field.names {
+			if ident, ok := name.derived.(Ident); ok {
+				write_semantic_node(builder, name, ast_context.file.src, .Property, .None);
+			}
+		}
+
+		visit(field.type, builder, ast_context);
+	}
+}
+
+visit_selector :: proc(selector: ^ast.Selector_Expr, builder: ^SemanticTokenBuilder, ast_context: ^analysis.AstContext) {
+	using analysis;
+	using ast;
+}
\ No newline at end of file