diff options
Diffstat (limited to 'src/tokenizer.cpp')
| -rw-r--r-- | src/tokenizer.cpp | 27 |
1 files changed, 0 insertions, 27 deletions
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index ee8fa7ef5..d4cfac52c 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1,30 +1,3 @@ -// TODO(bill): Unicode support -b32 rune_is_letter(Rune r) { - if (r < 0x80 && gb_char_is_alpha(cast(char)r) || r == '_') { - return true; - } - return false; -} - -b32 rune_is_digit(Rune r) { - if (r < 0x80 && gb_is_between(r, '0', '9')) - return true; - return false; -} - -b32 rune_is_whitespace(Rune r) { - switch (r) { - case ' ': - case '\t': - case '\n': - case '\r': - case '\f': - case '\v': - return true; - } - return false; -} - #define TOKEN_KINDS \ TOKEN_KIND(Token_Invalid, "Invalid"), \ TOKEN_KIND(Token_EOF, "EOF"), \ |