Setup tests for semantic tokens and add one simple test

4 files changed, 73 insertions, 21 deletions

diff --git a/src/server/requests.odin b/src/server/requests.odin
index 3ca38d6..6216ecb 100644
--- a/src/server/requests.odin
+++ b/src/server/requests.odin
@@ -1272,17 +1272,18 @@ request_semantic_token_full :: proc(
 		end = common.Position{line = 9000000}, //should be enough
 	}
 
-	symbols: SemanticTokens
+	tokens_params: SemanticTokensResponseParams
 
 	if config.enable_semantic_tokens {
 		resolve_entire_file_cached(document)
 
 		if file, ok := file_resolve_cache.files[document.uri.uri]; ok {
-			symbols = get_semantic_tokens(document, range, file.symbols)
+			tokens := get_semantic_tokens(document, range, file.symbols)
+			tokens_params = semantic_tokens_to_response_params(tokens)
 		}
 	}
 
-	response := make_response_message(params = symbols, id = id)
+	response := make_response_message(params = tokens_params, id = id)
 
 	send_response(response, writer)
 
@@ -1313,21 +1314,22 @@ request_semantic_token_range :: proc(
 		return .InternalError
 	}
 
-	symbols: SemanticTokens
+	tokens_params: SemanticTokensResponseParams
 
 	if config.enable_semantic_tokens {
 		resolve_entire_file_cached(document)
 
 		if file, ok := file_resolve_cache.files[document.uri.uri]; ok {
-			symbols = get_semantic_tokens(
+			tokens := get_semantic_tokens(
 				document,
 				semantic_params.range,
 				file.symbols,
 			)
+			tokens_params = semantic_tokens_to_response_params(tokens)
 		}
 	}
 
-	response := make_response_message(params = symbols, id = id)
+	response := make_response_message(params = tokens_params, id = id)
 
 	send_response(response, writer)
 
diff --git a/src/server/semantic_tokens.odin b/src/server/semantic_tokens.odin
index 507cd18..ae8918a 100644
--- a/src/server/semantic_tokens.odin
+++ b/src/server/semantic_tokens.odin
@@ -15,7 +15,7 @@ import "core:unicode/utf8"
 
 import "src:common"
 
-SemanticTokenTypes :: enum u8 {
+SemanticTokenTypes :: enum u32 {
 	Namespace,
 	Type,
 	Enum,
@@ -101,22 +101,40 @@ SemanticTokensRangeParams :: struct {
 	range:        common.Range,
 }
 
-SemanticTokens :: struct {
+SemanticToken :: struct {
+	// token line number, relative to the previous token
+	delta_line: u32,
+	// token start character, relative to the previous token
+	// (relative to 0 or the previous token’s start if they are on the same line)
+	delta_char: u32,
+	len:        u32,
+	type:       SemanticTokenTypes,
+	modifiers:  SemanticTokenModifiers,
+}
+#assert(size_of(SemanticToken) == 5 * size_of(u32))
+
+SemanticTokensResponseParams :: struct {
 	data: []u32,
 }
 
 SemanticTokenBuilder :: struct {
 	current_start: int,
-	tokens:        [dynamic]u32,
+	tokens:        [dynamic]SemanticToken,
 	symbols:       map[uintptr]SymbolAndNode,
 	src:           string,
 }
 
+semantic_tokens_to_response_params :: proc (
+	tokens: []SemanticToken,
+) -> SemanticTokensResponseParams {
+	return {data = (cast([^]u32)raw_data(tokens))[:len(tokens) * 5]}
+}
+
 get_semantic_tokens :: proc(
 	document: ^Document,
 	range: common.Range,
 	symbols: map[uintptr]SymbolAndNode,
-) -> SemanticTokens {
+) -> []SemanticToken {
 	ast_context := make_ast_context(
 		document.ast,
 		document.imports,
@@ -127,7 +145,7 @@ get_semantic_tokens :: proc(
 	ast_context.current_package = ast_context.document_package
 
 	builder: SemanticTokenBuilder = {
-		tokens  = make([dynamic]u32, 10000, context.temp_allocator),
+		tokens  = make([dynamic]SemanticToken, 0, 2000, context.temp_allocator),
 		symbols = symbols,
 		src     = ast_context.file.src,
 	}
@@ -139,7 +157,7 @@ get_semantic_tokens :: proc(
 		}
 	}
 
-	return {data = builder.tokens[:]}
+	return builder.tokens[:]
 }
 
 write_semantic_at_pos :: proc(
@@ -154,14 +172,13 @@ write_semantic_at_pos :: proc(
 		transmute([]u8)builder.src,
 		builder.current_start,
 	)
-	append(
-		&builder.tokens,
-		cast(u32)position.line,
-		cast(u32)position.character,
-		cast(u32)len,
-		cast(u32)type,
-		transmute(u32)modifiers,
-	)
+	append(&builder.tokens, SemanticToken{
+		delta_line = cast(u32)position.line,
+		delta_char = cast(u32)position.character,
+		len        = cast(u32)len,
+		type       = type,
+		modifiers  = modifiers,
+	})
 	builder.current_start = pos
 }
 
diff --git a/src/server/types.odin b/src/server/types.odin
index bb6d00f..d082309 100644
--- a/src/server/types.odin
+++ b/src/server/types.odin
@@ -23,7 +23,7 @@ ResponseParams :: union {
 	CompletionList,
 	SignatureHelp,
 	[]DocumentSymbol,
-	SemanticTokens,
+	SemanticTokensResponseParams,
 	Hover,
 	[]TextEdit,
 	[]InlayHint,
diff --git a/src/testing/testing.odin b/src/testing/testing.odin
index 210db59..0d9c028 100644
--- a/src/testing/testing.odin
+++ b/src/testing/testing.odin
@@ -414,3 +414,36 @@ expect_reference_locations :: proc(
 		}
 	}
 }
+
+expect_semantic_tokens :: proc(
+	t: ^testing.T,
+	src: ^Source,
+	expected: []server.SemanticToken,
+) {
+	setup(src)
+	defer teardown(src)
+
+	
+	resolve_flag: server.ResolveReferenceFlag
+	symbols_and_nodes := server.resolve_entire_file(
+		src.document,
+		resolve_flag,
+		context.temp_allocator,
+	)
+
+	range := common.Range{end = {line = 9000000}} //should be enough
+	tokens := server.get_semantic_tokens(src.document, range, symbols_and_nodes)
+
+	testing.expectf(t, len(expected) == len(tokens), "\nExpected %d tokens, but received %d", len(expected), len(tokens))
+
+	for i in 0..<min(len(expected), len(tokens)) {
+		e, a := expected[i], tokens[i]
+		testing.expectf(t,
+			e == a,
+			"\n[%d]: Expected \n(%d, %d, %d, %v, %w)\nbut received\n(%d, %d, %d, %v, %w)",
+			i,
+			e.delta_line, e.delta_char, e.len, e.type, e.modifiers,
+			a.delta_line, a.delta_char, a.len, a.type, a.modifiers,
+		)
+	}
+}