Make `-insert-semicolon` the default now

4 files changed, 129 insertions, 156 deletions

diff --git a/src/build_settings.cpp b/src/build_settings.cpp
index 5d93ab1ca..c614a6dc8 100644
--- a/src/build_settings.cpp
+++ b/src/build_settings.cpp
@@ -120,8 +120,10 @@ enum CommandKind : u32 {
 	Command_doc     = 1<<5,
 	Command_version = 1<<6,
 	Command_test    = 1<<7,
+	
+	Command_strip_semicolon = 1<<8,
 
-	Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test,
+	Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test|Command_strip_semicolon,
 	Command__does_build = Command_run|Command_build|Command_test,
 	Command_all = ~(u32)0,
 };
@@ -134,6 +136,7 @@ char const *odin_command_strings[32] = {
 	"doc",
 	"version",
 	"test",
+	"strip-semicolon",
 };
 
 
@@ -201,7 +204,6 @@ struct BuildContext {
 	bool   different_os;
 	bool   keep_object_files;
 	bool   disallow_do;
-	bool   insert_semicolon;
 
 
 	bool   ignore_warnings;
diff --git a/src/main.cpp b/src/main.cpp
index 4edbc0420..23c770e37 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1307,7 +1307,7 @@ bool parse_build_flags(Array<String> args) {
 							break;
 
 						case BuildFlag_InsertSemicolon:
-							build_context.insert_semicolon = true;
+							gb_printf_err("-insert-semicolon flag is not required any more\n");
 							break;
 
 						case BuildFlag_StrictStyle:
@@ -1644,7 +1644,7 @@ void print_show_help(String const arg0, String const &command) {
 	} else if (command == "run") {
 		print_usage_line(1, "run       same as 'build', but also then runs the newly compiled executable.");
 	} else if (command == "check") {
-		print_usage_line(1, "check     parse and type check .odin file");
+		print_usage_line(1, "check     parse and type check .odin file(s)");
 	} else if (command == "test") {
 		print_usage_line(1, "test      build ands runs procedures with the attribute @(test) in the initial package");
 	} else if (command == "query") {
@@ -1656,14 +1656,18 @@ void print_show_help(String const arg0, String const &command) {
 		print_usage_line(3, "odin doc core/path core/path/filepath");
 	} else if (command == "version") {
 		print_usage_line(1, "version   print version");
-	}
+	} else if (command == "strip-semicolon") {
+		print_usage_line(1, "strip-semicolon");
+		print_usage_line(2, "parse and type check .odin file(s) and then remove unneeded semicolons from the entire project");
+	} 
 
 	bool doc = command == "doc";
 	bool build = command == "build";
 	bool run_or_build = command == "run" || command == "build" || command == "test";
 	bool test_only = command == "test";
-	bool check_only = command == "check";
-	bool check = run_or_build || command == "check";
+	bool strip_semicolon = command == "strip-semicolon";
+	bool check_only = command == "check" || strip_semicolon;
+	bool check = run_or_build || check_only;
 
 	print_usage_line(0, "");
 	print_usage_line(1, "Flags");
@@ -1730,6 +1734,10 @@ void print_show_help(String const arg0, String const &command) {
 		print_usage_line(1, "-keep-temp-files");
 		print_usage_line(2, "Keeps the temporary files generated during compilation");
 		print_usage_line(0, "");
+	} else if (strip_semicolon) {
+		print_usage_line(1, "-keep-temp-files");
+		print_usage_line(2, "Keeps the temporary files generated during stripping the unneeded semicolons from files");
+		print_usage_line(0, "");
 	}
 
 	if (check) {
@@ -2002,6 +2010,36 @@ bool check_env(void) {
 	return true;
 }
 
+struct StripSemicolonFile {
+	String old_fullpath;
+	String new_fullpath;
+	AstFile *file;
+};
+
+int strip_semicolons(Parser *parser) {
+	#if 0
+	isize file_count = 0;
+	for_array(i, parser->packages) {
+		AstPackage *pkg = parser->packages[i];
+		file_count += pkg->files.count;
+	}
+	gb_printf_err("File count: %td\n", file_count);
+	
+	auto generated_files = array_make<StripSemicolonFile>(permanent_allocator(), 0, file_count);
+	
+	for_array(i, parser->packages) {
+		AstPackage *pkg = parser->packages[i];
+		for_array(j, pkg->files) {
+			AstFile *file = pkg->files[j];
+			gb_printf_err("%.*s\n", LIT(file->fullpath));
+		}
+	}
+	
+	#endif
+	return 0;
+}
+
+
 
 int main(int arg_count, char const **arg_ptr) {
 #define TIME_SECTION(str) do { debugf("[Section] %s\n", str); timings_start_section(&global_timings, str_lit(str)); } while (0)
@@ -2090,6 +2128,14 @@ int main(int arg_count, char const **arg_ptr) {
 		build_context.command_kind = Command_check;
 		build_context.no_output_files = true;
 		init_filename = args[2];
+	} else if (command == "strip-semicolon") {
+		if (args.count < 3) {
+			usage(args[0]);
+			return 1;
+		}
+		build_context.command_kind = Command_strip_semicolon;
+		build_context.no_output_files = true;
+		init_filename = args[2];
 	} else if (command == "query") {
 		if (args.count < 3) {
 			usage(args[0]);
@@ -2209,6 +2255,10 @@ int main(int arg_count, char const **arg_ptr) {
 	if (any_errors()) {
 		return 1;
 	}
+	
+	if (build_context.command_kind == Command_strip_semicolon) {
+		return strip_semicolons(parser);
+	}
 
 	if (build_context.generate_docs) {
 		if (global_error_collector.count != 0) {
diff --git a/src/parser.cpp b/src/parser.cpp
index 98064d63b..ec1a5776b 100644
--- a/src/parser.cpp
+++ b/src/parser.cpp
@@ -1281,9 +1281,6 @@ Token peek_token(AstFile *f) {
 }
 
 bool skip_possible_newline(AstFile *f) {
-	if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
-		return false;
-	}
 	if (token_is_newline(f->curr_token)) {
 		advance_token(f);
 		return true;
@@ -1292,9 +1289,6 @@ bool skip_possible_newline(AstFile *f) {
 }
 
 bool skip_possible_newline_for_literal(AstFile *f) {
-	if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
-		return false;
-	}
 	Token curr = f->curr_token;
 	if (token_is_newline(curr)) {
 		Token next = peek_token(f);
@@ -1500,62 +1494,11 @@ bool is_semicolon_optional_for_node(AstFile *f, Ast *s) {
 	if (s == nullptr) {
 		return false;
 	}
-
-	if (build_context.insert_semicolon) {
-		return true;
-	}
-
-	switch (s->kind) {
-	case Ast_EmptyStmt:
-	case Ast_BlockStmt:
-		return true;
-
-	case Ast_IfStmt:
-	case Ast_WhenStmt:
-	case Ast_ForStmt:
-	case Ast_RangeStmt:
-	case Ast_SwitchStmt:
-	case Ast_TypeSwitchStmt:
-		return true;
-
-	case Ast_HelperType:
-		return is_semicolon_optional_for_node(f, s->HelperType.type);
-	case Ast_DistinctType:
-		return is_semicolon_optional_for_node(f, s->DistinctType.type);
-
-	case Ast_PointerType:
-		return is_semicolon_optional_for_node(f, s->PointerType.type);
-
-	case Ast_StructType:
-	case Ast_UnionType:
-	case Ast_EnumType:
-		// Require semicolon within a procedure body
-		return f->curr_proc == nullptr;
-	case Ast_ProcLit:
-		return true;
-
-	case Ast_PackageDecl:
-	case Ast_ImportDecl:
-	case Ast_ForeignImportDecl:
-		return true;
-
-	case Ast_ValueDecl:
-		if (s->ValueDecl.is_mutable) {
-			return false;
-		}
-		if (s->ValueDecl.values.count > 0) {
-			return is_semicolon_optional_for_node(f, s->ValueDecl.values[s->ValueDecl.values.count-1]);
-		}
-		break;
-
-	case Ast_ForeignBlockDecl:
-		return is_semicolon_optional_for_node(f, s->ForeignBlockDecl.body);
-	}
-
-	return false;
+	return true;
 }
 
 void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) {
+	#if 0
 	if (!build_context.insert_semicolon && token.string == "\n") {
 		switch (token.kind) {
 		case Token_CloseBrace:
@@ -1571,6 +1514,14 @@ void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) {
 		tok.pos.column -= 1;
 		syntax_error(tok, "Expected ';', got newline");
 	}
+	#endif
+}
+
+void assign_removal_flag_to_semicolon(Token *token) {
+	// NOTE(bill): this is used for rewriting files to strip unneeded semicolons
+	if (token->kind == Token_Semicolon && token->string == ";") {
+		token->flags |= TokenFlag_Remove;
+	}
 }
 
 void expect_semicolon(AstFile *f, Ast *s) {
@@ -1578,6 +1529,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
 
 	if (allow_token(f, Token_Semicolon)) {
 		expect_semicolon_newline_error(f, f->prev_token, s);
+		assign_removal_flag_to_semicolon(&f->prev_token);
 		return;
 	}
 	switch (f->curr_token.kind) {
@@ -1592,6 +1544,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
 	prev_token = f->prev_token;
 	if (prev_token.kind == Token_Semicolon) {
 		expect_semicolon_newline_error(f, f->prev_token, s);
+		assign_removal_flag_to_semicolon(&f->prev_token);
 		return;
 	}
 
@@ -1599,40 +1552,19 @@ void expect_semicolon(AstFile *f, Ast *s) {
 		return;
 	}
 
-
-
 	if (s != nullptr) {
-		bool insert_semi = (f->tokenizer.flags & TokenizerFlag_InsertSemicolon) != 0;
-		if (insert_semi) {
-			switch (f->curr_token.kind) {
-			case Token_CloseBrace:
-			case Token_CloseParen:
-			case Token_else:
-			case Token_EOF:
-				return;
+		switch (f->curr_token.kind) {
+		case Token_CloseBrace:
+		case Token_CloseParen:
+		case Token_else:
+		case Token_EOF:
+			return;
 
-			default:
-				if (is_semicolon_optional_for_node(f, s)) {
-					return;
-				}
-				break;
-			}
-		} else if (prev_token.pos.line != f->curr_token.pos.line) {
+		default:
 			if (is_semicolon_optional_for_node(f, s)) {
 				return;
 			}
-		} else {
-			switch (f->curr_token.kind) {
-			case Token_CloseBrace:
-			case Token_CloseParen:
-			case Token_else:
-				return;
-			case Token_EOF:
-				if (is_semicolon_optional_for_node(f, s)) {
-					return;
-				}
-				break;
-			}
+			break;
 		}
 		String node_string = ast_strings[s->kind];
 		String p = token_to_string(f->curr_token);
@@ -4144,7 +4076,7 @@ Ast *parse_for_stmt(AstFile *f) {
 				if (f->curr_token.string != ";") {
 					syntax_error(f->curr_token, "Expected ';', got %.*s", LIT(token_to_string(f->curr_token)));
 				} else {
-					expect_semicolon(f, nullptr);
+					expect_token(f, Token_Semicolon);
 				}
 
 				if (f->curr_token.kind != Token_OpenBrace &&
@@ -4717,12 +4649,10 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
 	if (!string_ends_with(f->fullpath, str_lit(".odin"))) {
 		return ParseFile_WrongExtension;
 	}
-	TokenizerFlags tokenizer_flags = TokenizerFlag_InsertSemicolon;
-
 	zero_item(&f->tokenizer);
 	f->tokenizer.curr_file_id = f->id;
 
-	TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath, tokenizer_flags);
+	TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath);
 	if (err != TokenizerInit_None) {
 		switch (err) {
 		case TokenizerInit_Empty:
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 1e2a3caf3..3a3c40a7d 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -120,7 +120,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \
 TOKEN_KIND(Token__KeywordEnd, ""), \
 	TOKEN_KIND(Token_Count, "")
 
-enum TokenKind {
+enum TokenKind : u8 {
 #define TOKEN_KIND(e, s) e
 	TOKEN_KINDS
 #undef TOKEN_KIND
@@ -235,21 +235,26 @@ TokenPos token_pos_add_column(TokenPos pos) {
 	return pos;
 }
 
+enum TokenFlag : u8 {
+	TokenFlag_Remove = 1<<1,
+};
+
 struct Token {
 	TokenKind kind;
+	u8        flags;
 	String    string;
 	TokenPos  pos;
 };
 
 Token empty_token = {Token_Invalid};
-Token blank_token = {Token_Ident, {cast(u8 *)"_", 1}};
+Token blank_token = {Token_Ident, 0, {cast(u8 *)"_", 1}};
 
 Token make_token_ident(String s) {
-	Token t = {Token_Ident, s};
+	Token t = {Token_Ident, 0, s};
 	return t;
 }
 Token make_token_ident(char const *s) {
-	Token t = {Token_Ident, make_string_c(s)};
+	Token t = {Token_Ident, 0, make_string_c(s)};
 	return t;
 }
 
@@ -701,12 +706,6 @@ enum TokenizerInitError {
 	TokenizerInit_Count,
 };
 
-
-enum TokenizerFlags {
-	TokenizerFlag_None = 0,
-	TokenizerFlag_InsertSemicolon = 1<<0,
-};
-
 struct Tokenizer {
 	i32 curr_file_id;
 	String fullpath;
@@ -721,7 +720,6 @@ struct Tokenizer {
 
 	i32 error_count;
 
-	TokenizerFlags flags;
 	bool insert_semicolon;
 };
 
@@ -789,8 +787,7 @@ void advance_to_next_rune(Tokenizer *t) {
 	}
 }
 
-void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size, TokenizerFlags flags) {
-	t->flags = flags;
+void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size) {
 	t->fullpath = fullpath;
 	t->line_count = 1;
 
@@ -804,7 +801,7 @@ void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data,
 	}
 }
 
-TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath, TokenizerFlags flags = TokenizerFlag_None) {
+TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) {
 	TokenizerInitError err = TokenizerInit_None;
 
 	char *c_str = alloc_cstring(temporary_allocator(), fullpath);
@@ -813,15 +810,13 @@ TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &full
 	gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str);
 
 	if (fc.size > I32_MAX) {
-		t->flags = flags;
 		t->fullpath = fullpath;
 		t->line_count = 1;
 		err = TokenizerInit_FileTooLarge;
 		gb_file_free_contents(&fc);
 	} else if (fc.data != nullptr) {
-		init_tokenizer_with_data(t, fullpath, fc.data, fc.size, flags);
+		init_tokenizer_with_data(t, fullpath, fc.data, fc.size);
 	} else {
-		t->flags = flags;
 		t->fullpath = fullpath;
 		t->line_count = 1;
 		gbFile f = {};
@@ -1163,9 +1158,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
 			return;
 
 		case '\\':
-			if (t->flags & TokenizerFlag_InsertSemicolon) {
-				t->insert_semicolon = false;
-			}
+			t->insert_semicolon = false;
 			tokenizer_get_token(t, token);
 			if (token->pos.line == current_pos.line) {
 				tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\");
@@ -1493,43 +1486,41 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
 	token->string.len = t->curr - token->string.text;
 
 semicolon_check:;
-	if (t->flags & TokenizerFlag_InsertSemicolon) {
-		switch (token->kind) {
-		case Token_Invalid:
-		case Token_Comment:
-			// Preserve insert_semicolon info
-			break;
-		case Token_Ident:
-		case Token_context:
-		case Token_typeid:
-		case Token_break:
-		case Token_continue:
-		case Token_fallthrough:
-		case Token_return:
-		case Token_or_return:
-			/*fallthrough*/
-		case Token_Integer:
-		case Token_Float:
-		case Token_Imag:
-		case Token_Rune:
-		case Token_String:
-		case Token_Undef:
-			/*fallthrough*/
-		case Token_Question:
-		case Token_Pointer:
-		case Token_CloseParen:
-		case Token_CloseBracket:
-		case Token_CloseBrace:
-			/*fallthrough*/
-		case Token_Increment:
-		case Token_Decrement:
-			/*fallthrough*/
-			t->insert_semicolon = true;
-			break;
-		default:
-			t->insert_semicolon = false;
-			break;
-		}
+	switch (token->kind) {
+	case Token_Invalid:
+	case Token_Comment:
+		// Preserve insert_semicolon info
+		break;
+	case Token_Ident:
+	case Token_context:
+	case Token_typeid:
+	case Token_break:
+	case Token_continue:
+	case Token_fallthrough:
+	case Token_return:
+	case Token_or_return:
+		/*fallthrough*/
+	case Token_Integer:
+	case Token_Float:
+	case Token_Imag:
+	case Token_Rune:
+	case Token_String:
+	case Token_Undef:
+		/*fallthrough*/
+	case Token_Question:
+	case Token_Pointer:
+	case Token_CloseParen:
+	case Token_CloseBracket:
+	case Token_CloseBrace:
+		/*fallthrough*/
+	case Token_Increment:
+	case Token_Decrement:
+		/*fallthrough*/
+		t->insert_semicolon = true;
+		break;
+	default:
+		t->insert_semicolon = false;
+		break;
 	}
 
 	return;