Full Unicode Support

1 files changed, 0 insertions, 27 deletions

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index ee8fa7ef5..d4cfac52c 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -1,30 +1,3 @@
-// TODO(bill): Unicode support
-b32 rune_is_letter(Rune r) {
-	if (r < 0x80 && gb_char_is_alpha(cast(char)r) || r == '_') {
-		return true;
-	}
-	return false;
-}
-
-b32 rune_is_digit(Rune r) {
-	if (r < 0x80 && gb_is_between(r, '0', '9'))
-		return true;
-	return false;
-}
-
-b32 rune_is_whitespace(Rune r) {
-	switch (r) {
-	case ' ':
-	case '\t':
-	case '\n':
-	case '\r':
-	case '\f':
-	case '\v':
-		return true;
-	}
-	return false;
-}
-
 #define TOKEN_KINDS \
 	TOKEN_KIND(Token_Invalid, "Invalid"), \
 	TOKEN_KIND(Token_EOF, "EOF"), \