From a31bab5aae10757f5029b00e39beb0e3815b92b1 Mon Sep 17 00:00:00 2001 From: Ginger Bill Date: Fri, 23 Sep 2016 14:59:58 +0100 Subject: Unicode file loading; push_allocator & push_context --- src/tokenizer.cpp | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) (limited to 'src/tokenizer.cpp') diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 88fec813f..487ab004c 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -82,29 +82,31 @@ TOKEN_KIND(Token__ComparisonEnd, "_ComparisonEnd"), \ TOKEN_KIND(Token__OperatorEnd, "_OperatorEnd"), \ \ TOKEN_KIND(Token__KeywordBegin, "_KeywordBegin"), \ - TOKEN_KIND(Token_type, "type"), \ - TOKEN_KIND(Token_proc, "proc"), \ - TOKEN_KIND(Token_match, "match"), \ - TOKEN_KIND(Token_break, "break"), \ - TOKEN_KIND(Token_continue, "continue"), \ - TOKEN_KIND(Token_fallthrough, "fallthrough"), \ - TOKEN_KIND(Token_case, "case"), \ - TOKEN_KIND(Token_default, "default"), \ - TOKEN_KIND(Token_then, "then"), \ - TOKEN_KIND(Token_if, "if"), \ - TOKEN_KIND(Token_else, "else"), \ - TOKEN_KIND(Token_for, "for"), \ - TOKEN_KIND(Token_range, "range"), \ - TOKEN_KIND(Token_defer, "defer"), \ - TOKEN_KIND(Token_return, "return"), \ - TOKEN_KIND(Token_struct, "struct"), \ - TOKEN_KIND(Token_union, "union"), \ - TOKEN_KIND(Token_raw_union, "raw_union"), \ - TOKEN_KIND(Token_enum, "enum"), \ - TOKEN_KIND(Token_using, "using"), \ - TOKEN_KIND(Token_asm, "asm"), \ - TOKEN_KIND(Token_volatile, "volatile"), \ - TOKEN_KIND(Token_atomic, "atomic"), \ + TOKEN_KIND(Token_type, "type"), \ + TOKEN_KIND(Token_proc, "proc"), \ + TOKEN_KIND(Token_match, "match"), \ + TOKEN_KIND(Token_break, "break"), \ + TOKEN_KIND(Token_continue, "continue"), \ + TOKEN_KIND(Token_fallthrough, "fallthrough"), \ + TOKEN_KIND(Token_case, "case"), \ + TOKEN_KIND(Token_default, "default"), \ + TOKEN_KIND(Token_then, "then"), \ + TOKEN_KIND(Token_if, "if"), \ + TOKEN_KIND(Token_else, "else"), \ + TOKEN_KIND(Token_for, "for"), \ + TOKEN_KIND(Token_range, "range"), \ + TOKEN_KIND(Token_defer, "defer"), \ + TOKEN_KIND(Token_return, "return"), \ + TOKEN_KIND(Token_struct, "struct"), \ + TOKEN_KIND(Token_union, "union"), \ + TOKEN_KIND(Token_raw_union, "raw_union"), \ + TOKEN_KIND(Token_enum, "enum"), \ + TOKEN_KIND(Token_using, "using"), \ + TOKEN_KIND(Token_asm, "asm"), \ + TOKEN_KIND(Token_volatile, "volatile"), \ + TOKEN_KIND(Token_atomic, "atomic"), \ + TOKEN_KIND(Token_push_allocator, "push_allocator"), \ + TOKEN_KIND(Token_push_context, "push_context"), \ TOKEN_KIND(Token__KeywordEnd, "_KeywordEnd"), \ TOKEN_KIND(Token_Count, "") -- cgit v1.2.3