From 062a3c2fae3712c60af00798a0815509a732790b Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sat, 19 Apr 2025 20:25:44 +0200 Subject: Fix parsing of CDATA tags (#5059) Fixes #5054 --- core/encoding/xml/tokenizer.odin | 38 +++++++-------- core/encoding/xml/xml_reader.odin | 99 ++++++++++++++++++++++----------------- 2 files changed, 73 insertions(+), 64 deletions(-) (limited to 'core/encoding/xml') diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index a2bbaf28e..3ef9a6388 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -16,6 +16,7 @@ package encoding_xml import "core:fmt" import "core:unicode" import "core:unicode/utf8" +import "core:strings" Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any) @@ -121,7 +122,7 @@ default_error_handler :: proc(pos: Pos, msg: string, args: ..any) { error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) { pos := offset_to_pos(t, offset) if t.err != nil { - t.err(pos, msg, ..args) + t.err(pos=pos, fmt=msg, args=args) } t.error_count += 1 } @@ -268,32 +269,27 @@ scan_comment :: proc(t: ^Tokenizer) -> (comment: string, err: Error) { // Skip CDATA skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) { - if t.read_offset + len(CDATA_START) >= len(t.src) { - // Can't be the start of a CDATA tag. + if s := string(t.src[t.offset:]); !strings.has_prefix(s, CDATA_START) { return .None } - if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { - t.read_offset += len(CDATA_START) - offset := t.offset + t.read_offset += len(CDATA_START) + offset := t.offset - cdata_scan: for { - advance_rune(t) - if t.ch < 0 { - error(t, offset, "[scan_string] CDATA was not terminated\n") - return .Premature_EOF - } + cdata_scan: for { + advance_rune(t) + if t.ch < 0 { + error(t, offset, "[scan_string] CDATA was not terminated\n") + return .Premature_EOF + } - // Scan until the end of a CDATA tag. - if t.read_offset + len(CDATA_END) < len(t.src) { - if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { - t.read_offset += len(CDATA_END) - break cdata_scan - } - } + // Scan until the end of a CDATA tag. + if s := string(t.src[t.read_offset:]); strings.has_prefix(s, CDATA_END) { + t.read_offset += len(CDATA_END) + break cdata_scan } } - return + return .None } @(optimization_mode="favor_size") @@ -393,6 +389,8 @@ scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token { case '/': kind = .Slash case '-': kind = .Dash case ':': kind = .Colon + case '[': kind = .Open_Bracket + case ']': kind = .Close_Bracket case '"', '\'': kind = .Invalid diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index b8c8b13a4..60744357c 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -56,7 +56,7 @@ Option_Flag :: enum { Option_Flags :: bit_set[Option_Flag; u16] Document :: struct { - elements: [dynamic]Element, + elements: [dynamic]Element `fmt:"v,element_count"`, element_count: Element_ID, prologue: Attributes, @@ -70,15 +70,15 @@ Document :: struct { // If we encounter comments before the root node, and the option to intern comments is given, this is where they'll live. // Otherwise they'll be in the element tree. - comments: [dynamic]string, + comments: [dynamic]string `fmt:"-"`, // Internal - tokenizer: ^Tokenizer, - allocator: mem.Allocator, + tokenizer: ^Tokenizer `fmt:"-"`, + allocator: mem.Allocator `fmt:"-"`, // Input. Either the original buffer, or a copy if `.Input_May_Be_Modified` isn't specified. - input: []u8, - strings_to_free: [dynamic]string, + input: []u8 `fmt:"-"`, + strings_to_free: [dynamic]string `fmt:"-"`, } Element :: struct { @@ -195,7 +195,6 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha loop: for { skip_whitespace(t) - // NOTE(Jeroen): This is faster as a switch. switch t.ch { case '<': // Consume peeked `<` @@ -306,9 +305,13 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha } } + case .Open_Bracket: + // This could be a CDATA tag part of a tag's body. Unread the ` (attr: Attribute, offset: int, err: E t := doc.tokenizer key := expect(t, .Ident) or_return - offset = t.offset - len(key.text) - _ = expect(t, .Eq) or_return value := expect(t, .String, multiline_string=true) or_return @@ -591,6 +561,47 @@ parse_doctype :: proc(doc: ^Document) -> (err: Error) { return .None } +parse_body :: proc(doc: ^Document, element: Element_ID, opts: Options) -> (err: Error) { + assert(doc != nil) + context.allocator = doc.allocator + t := doc.tokenizer + + body_text := scan_string(t, t.offset) or_return + needs_processing := .Unbox_CDATA in opts.flags + needs_processing |= .Decode_SGML_Entities in opts.flags + + if !needs_processing { + append(&doc.elements[element].value, body_text) + return + } + + decode_opts := entity.XML_Decode_Options{} + if .Keep_Tag_Body_Comments not_in opts.flags { + decode_opts += { .Comment_Strip } + } + + if .Decode_SGML_Entities not_in opts.flags { + decode_opts += { .No_Entity_Decode } + } + + if .Unbox_CDATA in opts.flags { + decode_opts += { .Unbox_CDATA } + if .Decode_SGML_Entities in opts.flags { + decode_opts += { .Decode_CDATA } + } + } + + decoded, decode_err := entity.decode_xml(body_text, decode_opts) + if decode_err == .None { + append(&doc.elements[element].value, decoded) + append(&doc.strings_to_free, decoded) + } else { + append(&doc.elements[element].value, body_text) + } + + return +} + Element_ID :: u32 new_element :: proc(doc: ^Document) -> (id: Element_ID) { @@ -609,4 +620,4 @@ new_element :: proc(doc: ^Document) -> (id: Element_ID) { cur := doc.element_count doc.element_count += 1 return cur -} +} \ No newline at end of file -- cgit v1.2.3 From ab5ca087a7a64e2085edd955b403da0e8bc2c4f0 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sat, 19 Apr 2025 23:44:02 +0200 Subject: Add comment --- core/encoding/xml/xml_reader.odin | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'core/encoding/xml') diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 60744357c..d616be9dc 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -308,6 +308,10 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha case .Open_Bracket: // This could be a CDATA tag part of a tag's body. Unread the ` Date: Mon, 9 Jun 2025 15:09:04 +0200 Subject: Fix #4705 Allocate `doc.tokenizer`, and free it in `destroy`. --- core/encoding/xml/xml_reader.odin | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'core/encoding/xml') diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index d616be9dc..707d2b3f3 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -175,7 +175,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha data = bytes.clone(data) } - t := &Tokenizer{} + t := new(Tokenizer) init(t, string(data), path, error_handler) doc = new(Document) @@ -403,6 +403,7 @@ destroy :: proc(doc: ^Document) { } delete(doc.strings_to_free) + free(doc.tokenizer) free(doc) } -- cgit v1.2.3