From 9cadd5846582dcaf0c9c5309eaa35cf2e08ec94f Mon Sep 17 00:00:00 2001
From: gingerBill <bill@gingerbill.org>
Date: Tue, 28 May 2019 14:44:32 +0100
Subject: Improve tokenizer slightly

---
 src/tokenizer.cpp | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

(limited to 'src/tokenizer.cpp')

diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index dd9aa109c..a551f0545 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -625,6 +625,13 @@ gb_inline void scan_mantissa(Tokenizer *t, i32 base) {
 	}
 }
 
+u8 peek_byte(Tokenizer *t, isize offset=0) {
+	if (t->read_curr+offset < t->end) {
+		return t->read_curr[offset];
+	}
+	return 0;
+}
+
 Token scan_number_to_token(Tokenizer *t, bool seen_decimal_point) {
 	Token token = {};
 	token.kind = Token_Integer;
@@ -718,14 +725,12 @@ Token scan_number_to_token(Tokenizer *t, bool seen_decimal_point) {
 
 fraction:
 	if (t->curr_rune == '.') {
-		// HACK(bill): This may be inefficient
-		TokenizerState state = save_tokenizer_state(t);
-		advance_to_next_rune(t);
-		if (t->curr_rune == '.') {
-			// TODO(bill): Clean up this shit
-			restore_tokenizer_state(t, &state);
+		if (peek_byte(t) == '.') {
+			// NOTE(bill): this is kind of ellipsis
 			goto end;
 		}
+		advance_to_next_rune(t);
+
 		token.kind = Token_Float;
 		scan_mantissa(t, 10);
 	}
@@ -983,7 +988,7 @@ Token tokenizer_get_token(Tokenizer *t) {
 		} break;
 
 		case '.':
-			if (t->curr_rune == '.') { // Could be an ellipsis
+			if (t->curr_rune == '.') {
 				advance_to_next_rune(t);
 				token.kind = Token_Ellipsis;
 				if (t->curr_rune == '<') {
@@ -1039,7 +1044,7 @@ Token tokenizer_get_token(Tokenizer *t) {
 			if (t->curr_rune == '=') {
 				advance_to_next_rune(t);
 				token.kind = Token_SubEq;
-			} else if (t->curr_rune == '-' && t->read_curr[0] == '-') {
+			} else if (t->curr_rune == '-' && peek_byte(t) == '-') {
 				advance_to_next_rune(t);
 				advance_to_next_rune(t);
 				token.kind = Token_Undef;
-- 
cgit v1.2.3