Improve a tokenizer error message; Add extra tokens for `in` and `notin` for fun!

2 files changed, 15 insertions, 15 deletions

diff --git a/src/parser.cpp b/src/parser.cpp
index 84c03587d..2eee82e22 100644
--- a/src/parser.cpp
+++ b/src/parser.cpp
@@ -4539,6 +4539,7 @@ ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) {
 
 	TokenPos err_pos = {0};
 	ParseFileError err = init_ast_file(file, fi->fullpath, &err_pos);
+	err_pos.file = fi->fullpath;
 
 	if (err != ParseFile_None) {
 		if (err == ParseFile_EmptyFile) {
@@ -4563,7 +4564,7 @@ ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) {
 			error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi->name), LIT(fi->fullpath));
 			break;
 		case ParseFile_InvalidToken:
-			error(pos, "Failed to parse file: %.*s; invalid token found in file at (%td:%td)", LIT(fi->name), err_pos.line, err_pos.column);
+			error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi->name));
 			break;
 		case ParseFile_EmptyFile:
 			error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi->name));
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 48f7af19c..1537b8139 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -927,13 +927,15 @@ Token tokenizer_get_token(Tokenizer *t) {
 		case '}':  token.kind = Token_CloseBrace;   break;
 		case '\\': token.kind = Token_BackSlash;    break;
 
-		case 0x2260:  token.kind = Token_NotEq; break; // '≠'
-		case 0x2264:  token.kind = Token_LtEq;  break; // '≤'
-		case 0x2265:  token.kind = Token_GtEq;  break; // '≥'
+		case 0x2260: token.kind = Token_NotEq; break; // '≠'
+		case 0x2264: token.kind = Token_LtEq;  break; // '≤'
+		case 0x2265: token.kind = Token_GtEq;  break; // '≥'
+		case 0x2208: token.kind = Token_in;    break; // '∈'
+		case 0x2209: token.kind = Token_notin; break; // '∉'
 
-		case '%': token.kind = token_kind_dub_eq(t, '%', Token_Mod, Token_ModEq, Token_ModMod, Token_ModModEq);      break;
+		case '%': token.kind = token_kind_dub_eq(t, '%', Token_Mod, Token_ModEq, Token_ModMod, Token_ModModEq);  break;
 
-		case '*': token.kind = token_kind_variant2(t, Token_Mul, Token_MulEq);                                        break;
+		case '*': token.kind = token_kind_variant2(t, Token_Mul, Token_MulEq); break;
 		case '=':
 			token.kind = Token_Eq;
 			if (t->curr_rune == '>') {
@@ -944,21 +946,18 @@ Token tokenizer_get_token(Tokenizer *t) {
 				token.kind = Token_CmpEq;
 			}
 			break;
-		case '~': token.kind = token_kind_variant2(t, Token_Xor, Token_XorEq);                                        break;
-		case '!': token.kind = token_kind_variant2(t, Token_Not, Token_NotEq);                                        break;
-		case '+': token.kind = token_kind_variant2(t, Token_Add, Token_AddEq);                                        break;
+		case '~': token.kind = token_kind_variant2(t, Token_Xor, Token_XorEq);  break;
+		case '!': token.kind = token_kind_variant2(t, Token_Not, Token_NotEq);  break;
+		case '+': token.kind = token_kind_variant2(t, Token_Add, Token_AddEq);  break;
 		case '-':
 			token.kind = Token_Sub;
 			if (t->curr_rune == '=') {
 				advance_to_next_rune(t);
 				token.kind = Token_SubEq;
-			} else if (t->curr_rune == '-') {
+			} else if (t->curr_rune == '-' && t->read_curr[0] == '-') {
 				advance_to_next_rune(t);
-				token.kind = Token_Invalid;
-				if (t->curr_rune == '-') {
-					advance_to_next_rune(t);
-					token.kind = Token_Undef;
-				}
+				advance_to_next_rune(t);
+				token.kind = Token_Undef;
 			} else if (t->curr_rune == '>') {
 				advance_to_next_rune(t);
 				token.kind = Token_ArrowRight;