Semicolons mandatory again (and probably forever now...)

1 files changed, 20 insertions, 83 deletions

diff --git a/src/tokenizer.c b/src/tokenizer.c
index 5d1d63a1b..1839abdc2 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -304,8 +304,6 @@ typedef struct Tokenizer {
 	u8 *  line;        // current line pos
 	isize line_count;
 
-	bool insert_semicolon; // Inserts a semicolon before the next newline
-
 	isize error_count;
 	Array(String) allocated_strings;
 } Tokenizer;
@@ -418,7 +416,7 @@ gb_inline void destroy_tokenizer(Tokenizer *t) {
 void tokenizer_skip_whitespace(Tokenizer *t) {
 	while (t->curr_rune == ' ' ||
 	       t->curr_rune == '\t' ||
-	       (t->curr_rune == '\n' && !t->insert_semicolon) ||
+	       t->curr_rune == '\n' ||
 	       t->curr_rune == '\r') {
 		advance_to_next_rune(t);
 	}
@@ -677,8 +675,6 @@ Token tokenizer_get_token(Tokenizer *t) {
 	token.pos.line = t->line_count;
 	token.pos.column = t->curr - t->line + 1;
 
-	bool insert_semicolon = false;
-
 	Rune curr_rune = t->curr_rune;
 	if (rune_is_letter(curr_rune)) {
 		token.kind = Token_Ident;
@@ -706,48 +702,19 @@ Token tokenizer_get_token(Tokenizer *t) {
 					}
 				}
 			}
-
-			switch (token.kind) {
-			case Token_Ident:
-			case Token_return:
-			case Token_break:
-			case Token_continue:
-			case Token_fallthrough:
-				insert_semicolon = true;
-				break;
-			}
-		} else {
-			insert_semicolon = true;
 		}
 
 	} else if (gb_is_between(curr_rune, '0', '9')) {
-		insert_semicolon = true;
 		token = scan_number_to_token(t, false);
 	} else {
 		advance_to_next_rune(t);
 		switch (curr_rune) {
 		case GB_RUNE_EOF:
-			if (t->insert_semicolon) {
-				t->insert_semicolon = false;
-				token.string = str_lit("\n");
-				token.kind = Token_Semicolon;
-				return token;
-			}
 			token.kind = Token_EOF;
 			break;
 
-		case '\n':
-			// NOTE(bill): This will be only be reached if t->insert_semicolom was set
-			// earlier and exited early from tokenizer_skip_whitespace()
-			t->insert_semicolon = false;
-			token.string = str_lit("\n");
-			token.kind = Token_Semicolon;
-			return token;
-
 		case '\'': // Rune Literal
 		{
-			insert_semicolon = true;
-
 			token.kind = Token_Rune;
 			Rune quote = curr_rune;
 			bool valid = true;
@@ -780,7 +747,6 @@ Token tokenizer_get_token(Tokenizer *t) {
 				if (success == 2) {
 					array_add(&t->allocated_strings, token.string);
 				}
-				t->insert_semicolon = true;
 				return token;
 			} else {
 				tokenizer_err(t, "Invalid rune literal");
@@ -790,8 +756,6 @@ Token tokenizer_get_token(Tokenizer *t) {
 		case '`': // Raw String Literal
 		case '"': // String Literal
 		{
-			insert_semicolon = true;
-
 			i32 success;
 			Rune quote = curr_rune;
 			token.kind = Token_String;
@@ -829,7 +793,6 @@ Token tokenizer_get_token(Tokenizer *t) {
 				if (success == 2) {
 					array_add(&t->allocated_strings, token.string);
 				}
-				t->insert_semicolon = true;
 				return token;
 			} else {
 				tokenizer_err(t, "Invalid string literal");
@@ -839,7 +802,6 @@ Token tokenizer_get_token(Tokenizer *t) {
 		case '.':
 			token.kind = Token_Period; // Default
 			if (gb_is_between(t->curr_rune, '0', '9')) { // Might be a number
-				insert_semicolon = true;
 				token = scan_number_to_token(t, true);
 			} else if (t->curr_rune == '.') { // Could be an ellipsis
 				advance_to_next_rune(t);
@@ -881,21 +843,18 @@ Token tokenizer_get_token(Tokenizer *t) {
 			token.kind = Token_OpenParen;
 			break;
 		case ')':
-			insert_semicolon = true;
 			token.kind = Token_CloseParen;
 			break;
 		case '[':
 			token.kind = Token_OpenBracket;
 			break;
 		case ']':
-			insert_semicolon = true;
 			token.kind = Token_CloseBracket;
 			break;
 		case '{':
 			token.kind = Token_OpenBrace;
 			break;
 		case '}':
-			insert_semicolon = true;
 			token.kind = Token_CloseBrace;
 			break;
 
@@ -906,56 +865,37 @@ Token tokenizer_get_token(Tokenizer *t) {
 		case '!': token.kind = token_kind_variant2(t, Token_Not,   Token_NotEq);     break;
 		case '+':
 			token.kind = token_kind_variant3(t, Token_Add, Token_AddEq, '+', Token_Increment);
-			if (token.kind == Token_Increment) {
-				insert_semicolon = true;
-			}
 			break;
 		case '-':
 			token.kind = token_kind_variant4(t, Token_Sub,   Token_SubEq, '-', Token_Decrement, '>', Token_ArrowRight);
-			if (token.kind == Token_Decrement) {
-				insert_semicolon = true;
-			}
 			break;
 		case '/': {
-			if (t->curr_rune == '/' || t->curr_rune == '*') {
-				if (t->insert_semicolon && tokenizer_find_line_end(t)) {
-					t->curr_rune = '/';
-					t->curr = token.string.text;
-					t->read_curr = t->curr+1;
-					t->insert_semicolon = false;
-
-					token.kind = Token_Semicolon;
-					token.string = str_lit("\n");
-					return token;
+			if (t->curr_rune == '/') {
+				while (t->curr_rune != '\n') {
+					advance_to_next_rune(t);
 				}
-
-				if (t->curr_rune == '/') {
-					while (t->curr_rune != '\n') {
+				token.kind = Token_Comment;
+			} else if (t->curr_rune == '*') {
+				isize comment_scope = 1;
+				advance_to_next_rune(t);
+				while (comment_scope > 0) {
+					if (t->curr_rune == '/') {
 						advance_to_next_rune(t);
-					}
-					token.kind = Token_Comment;
-				} else if (t->curr_rune == '*') {
-					isize comment_scope = 1;
-					advance_to_next_rune(t);
-					while (comment_scope > 0) {
-						if (t->curr_rune == '/') {
+						if (t->curr_rune == '*') {
 							advance_to_next_rune(t);
-							if (t->curr_rune == '*') {
-								advance_to_next_rune(t);
-								comment_scope++;
-							}
-						} else if (t->curr_rune == '*') {
-							advance_to_next_rune(t);
-							if (t->curr_rune == '/') {
-								advance_to_next_rune(t);
-								comment_scope--;
-							}
-						} else {
+							comment_scope++;
+						}
+					} else if (t->curr_rune == '*') {
+						advance_to_next_rune(t);
+						if (t->curr_rune == '/') {
 							advance_to_next_rune(t);
+							comment_scope--;
 						}
+					} else {
+						advance_to_next_rune(t);
 					}
-					token.kind = Token_Comment;
 				}
+				token.kind = Token_Comment;
 			} else {
 				token.kind = token_kind_variant2(t, Token_Quo, Token_QuoEq);
 			}
@@ -994,14 +934,11 @@ Token tokenizer_get_token(Tokenizer *t) {
 				int len = cast(int)gb_utf8_encode_rune(str, curr_rune);
 				tokenizer_err(t, "Illegal character: %.*s (%d) ", len, str, curr_rune);
 			}
-			insert_semicolon = t->insert_semicolon;
 			token.kind = Token_Invalid;
 			break;
 		}
 	}
 
-	t->insert_semicolon = insert_semicolon;
-
 	token.string.len = t->curr - token.string.text;
 	return token;
 }