refractoring and more semantic tokens

2 files changed, 42 insertions, 2 deletions

diff --git a/src/common/ast.odin b/src/common/ast.odin
index ca26726..ef98acb 100644
--- a/src/common/ast.odin
+++ b/src/common/ast.odin
@@ -15,7 +15,9 @@ keyword_map : map [string] bool =
          "bool" = true,
          "rawptr" = true,
          "any" = true,
-         "u32" = true};
+         "u32" = true,
+         "true" = true,
+         "false" = true};
 
 get_ast_node_string :: proc(node: ^ast.Node, src: [] byte) -> string {
     return string(src[node.pos.offset:node.end.offset]);
diff --git a/src/common/position.odin b/src/common/position.odin
index 7eb4173..220f347 100644
--- a/src/common/position.odin
+++ b/src/common/position.odin
@@ -7,7 +7,9 @@ import "core:odin/ast"
 
 /*
     This file handles the conversion between utf-16 and utf-8 offsets in the text document
- */
+*/
+
+//TODO(Optimize by calculating all the newlines at parse instead of calculating them)
 
 Position :: struct {
 	line: int,
@@ -53,6 +55,42 @@ get_absolute_position :: proc(position: Position, document_text: [] u8) -> (Abso
     return absolute, true;
 }
 
+get_relative_token_position :: proc(offset: int, document_text: [] u8, current_start: int) -> Position {
+
+    start_index := current_start;
+
+    data := document_text[start_index:];
+
+    i: int;
+
+    position: Position;
+
+    for i + start_index < offset {
+
+        r, w := utf8.decode_rune(data[i:]);
+
+        if r == '\n' { //\r?
+            position.character = 0;
+            position.line += 1;
+            i += 1;
+        }
+
+        else {
+            if r < 0x10000 {
+                position.character += 1;
+            }
+
+            else {
+                position.character += 2;
+            }
+
+            i += w;
+        }
+    }
+
+    return position;
+}
+
 /*
     Get the range of a token in utf16 space
  */