[xml] Initial implementation of `core:encoding/xml`.

A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). Features: - Supports enough of the XML 1.0/1.1 spec to handle the 99.9% of XML documents in common current usage. - Simple to understand and use. Small. Caveats: - We do NOT support HTML in this package, as that may or may not be valid XML. If it works, great. If it doesn't, that's not considered a bug. - We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences. - <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options. TODO: - Optional CDATA unboxing. - Optional `>`, ` `, ` ` and other escape substitution in tag bodies. - Test suite MAYBE: - XML writer? - Serialize/deserialize Odin types?
author: Jeroen van Rijn <Kelimion@users.noreply.github.com> 2021-11-30 23:01:22 +0100
committer: Jeroen van Rijn <Kelimion@users.noreply.github.com> 2021-12-05 02:52:22 +0100
commit: b5c828fe4ee3f0942b2eda1dc5753e4ad6d38ea9 (patch)
tree: ffbd45adb60e3de951dc2948801d5a57b21dd2c9 /tests
parent: 6ce5608003e630bc0de1c591fd4cbea3fe59e1d3 (diff)
8 files changed, 435 insertions, 30 deletions
diff --git a/tests/core/Makefile b/tests/core/Makefile
index 0f0ffe4d6..e17dede90 100644
--- a/tests/core/Makefile
+++ b/tests/core/Makefile
@@ -1,22 +1,29 @@
 ODIN=../../odin
 PYTHON=$(shell which python3)
 
-all: download_test_assets image_test compress_test strings_test hash_test crypto_test
+all: download_test_assets image_test compress_test strings_test hash_test crypto_test encoding_test
 
 download_test_assets:
 	$(PYTHON) download_assets.py
 
 image_test:
-	$(ODIN) run image/test_core_image.odin
+	$(ODIN) run image/test_core_image.odin -out=test_image -o:speed -no-bounds-check
 
 compress_test:
-	$(ODIN) run compress/test_core_compress.odin
+	$(ODIN) run compress/test_core_compress.odin -out=test_compress -o:speed -no-bounds-check
 
 strings_test:
-	$(ODIN) run strings/test_core_strings.odin
+	$(ODIN) run strings/test_core_strings.odin -out=test_strings -o:speed -no-bounds-check
+
+odin_test:
+	$(ODIN) run odin -out=test_odin -o:speed -no-bounds-check
 
 hash_test:
 	$(ODIN) run hash -out=test_hash -o:speed -no-bounds-check
 
 crypto_test:
-	$(ODIN) run crypto -out=crypto_hash -o:speed -no-bounds-check
-\ No newline at end of file
+	$(ODIN) run crypto -out=crypto_hash -o:speed -no-bounds-check
+
+encoding_test:
+	$(ODIN) run encoding/json -out=test_encoding_json -o:speed -no-bounds-check
+	$(ODIN) run encoding/xml  -out=test_encoding_xml  -o:speed -no-bounds-check
diff --git a/tests/core/assets/xml/nl_NL-qt-ts.ts b/tests/core/assets/xml/nl_NL-qt-ts.ts
new file mode 100644
index 000000000..6ec3f2f47
--- /dev/null
+++ b/tests/core/assets/xml/nl_NL-qt-ts.ts
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.1" language="nl" sourcelanguage="en">
+<context>
+  <name>Page</name>
+  <message>
+    <source>Text for translation</source>
+    <comment>commenting</comment>
+    <translation type="obsolete">Tekst om te vertalen</translation>
+  </message>
+  <message>
+     <source>Also text to translate</source>
+     <extracomment>some text</extracomment>
+    <translation>Ook tekst om te vertalen</translation>
+  </message>
+</context>
+<context>
+  <name>installscript</name>
+  <message>
+    <source>99 bottles of beer on the wall</source>
+    <oldcomment>some new comments here</oldcomment>
+    <translation>99 flessen bier op de muur</translation>
+  </message>
+</context>
+<context>
+    <name>apple_count</name>
+    <message numerus="yes">
+      <source>%d apple(s)</source>
+      <translation>
+        <numerusform>%d appel</numerusform>
+        <numerusform>%d appels</numerusform>
+      </translation>
+    </message>
+  </context>
+</TS>
diff --git a/tests/core/assets/xml/nl_NL-xliff-1.0.xliff b/tests/core/assets/xml/nl_NL-xliff-1.0.xliff
new file mode 100644
index 000000000..7a1abcd66
--- /dev/null
+++ b/tests/core/assets/xml/nl_NL-xliff-1.0.xliff
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
+	<file id="42" original="Foozle.xml" source-language="en" target-language="nl-NL" datatype="plaintext">
+		<body>
+			<trans-unit id="874396" maxwidth="20" size-unit="char">
+				<source>text</source>
+				<target state="translated">tekst</target>
+				<note>Context</note>
+			</trans-unit>
+			<trans-unit id="874397" approved="yes">
+				<source>text 1</source>
+				<target state="translated">tekst 1</target>
+				<note>Context 1</note>
+			</trans-unit>
+			<trans-unit id="874398">
+				<source>text 2</source>
+				<target state="needs-translation"/>
+				<context context-type="context">Context of the segment 2</context>
+			</trans-unit>
+			<trans-unit id="874399" translate="no">
+				<source>text 3</source>
+				<target state="final">translation 3</target>
+				<note>Context 3</note>
+			</trans-unit>
+			<group restype="x-gettext-plurals">
+				<note>Plurals</note>
+				<trans-unit id="14343743[0]">
+					<source>%d month</source>
+					<target xml:lang="nl" state="translated">%d maand</target>
+				</trans-unit>
+				<trans-unit id="14343743[1]">
+					<source>%d months</source>
+					<target xml:lang="nl" state="translated">%d maanden</target>
+				</trans-unit>
+			</group>
+		</body>
+	</file>
+</xliff>
diff --git a/tests/core/assets/xml/nl_NL-xliff-2.0.xliff b/tests/core/assets/xml/nl_NL-xliff-2.0.xliff
new file mode 100644
index 000000000..611ac80c4
--- /dev/null
+++ b/tests/core/assets/xml/nl_NL-xliff-2.0.xliff
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:2.0" version="2.0" srcLang="en" trgLang="nl">
+	<file id="f1">
+		<notes>
+			<note id="n1">Note for file</note>
+		</notes>
+		<unit id="u1">
+			<notes>
+				<note id="n1">Note for unit</note>
+			</notes>
+			<segment id="s1" state="initial">
+				<source>text</source>
+				<target></target>
+			</segment>
+		</unit>
+		<unit id="u2">
+			<notes>
+				<note id="n2">Note for unit 2</note>
+			</notes>
+			<segment id="s2" state="translated">
+				<source>text 2</source>
+				<target>translation 2</target>
+			</segment>
+		</unit>
+		<unit id="u3">
+			<notes>
+				<note id="n3">Note for unit 3</note>
+			</notes>
+			<segment id="s3" state="final">
+				<source>text 3</source>
+				<target>approved translation 3</target>
+			</segment>
+		</unit>
+		<group id="90290" type="x-gettext:plurals">
+			<unit id="90291" name="90290[0]">
+				<notes>
+					<note category="context">Plurals</note>
+				</notes>
+				<segment>
+					<source>%d month</source>
+					<target xml:lang="nl">%d maand</target>
+				</segment>
+			</unit>
+			<unit id="90292" name="90290[1]">
+				<segment>
+					<source>%d months</source>
+					<target xml:lang="nl">%d maanden</target>
+				</segment>
+			</unit>
+		</group>
+	</file>
+</xliff>
+\ No newline at end of file
diff --git a/tests/core/assets/xml/utf8.xml b/tests/core/assets/xml/utf8.xml
new file mode 100644
index 000000000..c9ed3bf69
--- /dev/null
+++ b/tests/core/assets/xml/utf8.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE 恥ずべきフクロウ>
+<恥ずべきフクロウ 올빼미_id="Foozle&#32;<![CDATA[<greeting>Hello, world!"</greeting>]]>Barzle">
+<부끄러운:barzle>
+	<name foo:bar="birmese">ရှက်စရာ ဇီးကွက်</name>
+	<nickname>Owl of Shame</nickname>
+	<data>More CDATA <![CDATA[<greeting>Hello, world!</greeting><![CDATA] < ]]> Nonsense.</data>
+</부끄러운:barzle>
+\ No newline at end of file
diff --git a/tests/core/build.bat b/tests/core/build.bat
index 176b7f175..7a214acc9 100644
--- a/tests/core/build.bat
+++ b/tests/core/build.bat
@@ -5,34 +5,35 @@ python3 download_assets.py
 echo ---
 echo Running core:image tests
 echo ---
-%PATH_TO_ODIN% run image    %COMMON%
+%PATH_TO_ODIN% run image    %COMMON% -out:test_image.exe
 
 echo ---
 echo Running core:compress tests
 echo ---
-%PATH_TO_ODIN% run compress %COMMON%
+%PATH_TO_ODIN% run compress %COMMON% -out:test_compress.exe
 
 echo ---
 echo Running core:strings tests
 echo ---
-%PATH_TO_ODIN% run strings %COMMON%
+%PATH_TO_ODIN% run strings %COMMON% -out:test_strings.exe
 
 echo ---
 echo Running core:hash tests
 echo ---
-%PATH_TO_ODIN% run hash %COMMON% -o:size
+%PATH_TO_ODIN% run hash %COMMON% -o:size -out:test_hash.exe
 
 echo ---
 echo Running core:odin tests
 echo ---
-%PATH_TO_ODIN% run odin %COMMON% -o:size
+%PATH_TO_ODIN% run odin %COMMON% -o:size -out:test_odin.exe
 
 echo ---
 echo Running core:crypto hash tests
 echo ---
-%PATH_TO_ODIN% run crypto %COMMON%
+%PATH_TO_ODIN% run crypto %COMMON% -o:speed -out:test_crypto.exe
 
 echo ---
 echo Running core:encoding tests
 echo ---
-%PATH_TO_ODIN% run encoding %COMMON%
-\ No newline at end of file
+%PATH_TO_ODIN% run encoding\json %COMMON% -out:test_json.exe
+%PATH_TO_ODIN% run encoding\xml %COMMON% -out:test_xml.exe
+\ No newline at end of file
diff --git a/tests/core/encoding/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin
index f536eb4c6..4f415c008 100644
--- a/tests/core/encoding/test_core_json.odin
+++ b/tests/core/encoding/json/test_core_json.odin
@@ -8,32 +8,32 @@ TEST_count := 0
 TEST_fail  := 0
 
 when ODIN_TEST {
-    expect  :: testing.expect
-    log     :: testing.log
+	expect  :: testing.expect
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
-        TEST_count += 1
-        if !condition {
-            TEST_fail += 1
-            fmt.println(message)
-            return
-        }
-        fmt.println(" PASS")
-    }
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
-        fmt.printf("log: %v\n", v)
-    }
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+		fmt.printf("[%v] ", loc)
+		TEST_count += 1
+		if !condition {
+			TEST_fail += 1
+			fmt.println(message)
+			return
+		}
+		fmt.println(" PASS")
+	}
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("[%v] ", loc)
+		fmt.printf("log: %v\n", v)
+	}
 }
 
 main :: proc() {
-    t := testing.T{}
+	t := testing.T{}
 	
 	parse_json(&t)
 	marshal_json(&t)
 
-    fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
 }
 
 @test
diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin
new file mode 100644
index 000000000..7eefac212
--- /dev/null
+++ b/tests/core/encoding/xml/test_core_xml.odin
@@ -0,0 +1,264 @@
+package test_core_xml
+
+import "core:encoding/xml"
+import "core:testing"
+import "core:mem"
+import "core:fmt"
+
+Silent :: proc(pos: xml.Pos, fmt: string, args: ..any) {
+	// Custom (silent) error handler.
+}
+
+OPTIONS :: xml.Options{
+	flags            = {
+		.Ignore_Unsupported, .Intern_Comments,
+	},
+	expected_doctype = "",
+}
+
+TEST_count := 0
+TEST_fail  := 0
+
+TEST :: struct {
+	filename: string,
+	options:  xml.Options,
+	expected: struct {
+		error:        xml.Error,
+		xml_version:  string,
+		xml_encoding: string,
+		doctype:      string,
+	},
+}
+
+TESTS :: []TEST{
+	/*
+		First we test that certain files parse without error.
+	*/
+	{
+		filename  = "assets/xml/utf8.xml",
+		options   = OPTIONS,
+		expected  = {
+			error        = .None,
+			xml_version  = "1.0",
+			xml_encoding = "utf-8",
+			doctype      = "恥ずべきフクロウ",
+		},
+	},
+	{
+		filename  = "assets/xml/nl_NL-qt-ts.ts",
+		options   = OPTIONS,
+		expected  = {
+			error        = .None,
+			xml_version  = "1.0",
+			xml_encoding = "utf-8",
+			doctype      = "TS",
+		},
+	},
+	{
+		filename  = "assets/xml/nl_NL-xliff-1.0.xliff",
+		options   = OPTIONS,
+		expected  = {
+			error        = .None,
+			xml_version  = "1.0",
+			xml_encoding = "UTF-8",
+			doctype      = "",
+		},
+	},
+	{
+		filename  = "assets/xml/nl_NL-xliff-2.0.xliff",
+		options   = OPTIONS,
+		expected  = {
+			error        = .None,
+			xml_version  = "1.0",
+			xml_encoding = "utf-8",
+			doctype      = "",
+		},
+	},
+
+	/*
+		Then we test that certain errors are returned as expected.
+	*/
+	{
+		filename  = "assets/xml/utf8.xml",
+		options   = {
+			flags            = {
+				.Ignore_Unsupported, .Intern_Comments,
+			},
+			expected_doctype = "Odin",
+		},
+		expected  = {
+			error        = .Invalid_DocType,
+			xml_version  = "1.0",
+			xml_encoding = "utf-8",
+			doctype      = "恥ずべきフクロウ",
+		},
+	},
+}
+
+when ODIN_TEST {
+    expect  :: testing.expect
+    log     :: testing.log
+} else {
+    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+        fmt.printf("[%v] ", loc)
+        TEST_count += 1
+        if !condition {
+            TEST_fail += 1
+            fmt.println(message)
+            return
+        }
+        fmt.println(" PASS")
+    }
+    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+        fmt.printf("[%v] ", loc)
+        fmt.printf("log: %v\n", v)
+    }
+}
+
+main :: proc() {
+    t := testing.T{}
+
+	track: mem.Tracking_Allocator
+	mem.tracking_allocator_init(&track, context.allocator)
+	context.allocator = mem.tracking_allocator(&track)
+
+	run_tests(&t)
+
+	if len(track.allocation_map) > 0 {
+		for _, v in track.allocation_map {
+			err_msg := fmt.tprintf("%v Leaked %v bytes.", v.location, v.size)
+			expect(&t, false, err_msg)
+		}
+	}	
+
+    fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+}
+
+@test
+run_tests :: proc(t: ^testing.T) {
+	using fmt
+
+	count := 0
+
+	for test in TESTS {
+		printf("Trying to parse %v\n\n", test.filename)
+
+		doc, err := xml.parse(test.filename, test.options, Silent)
+		defer xml.destroy(doc)
+
+		err_msg := tprintf("Expected return value %v, got %v", test.expected.error, err)
+		expect(t, err == test.expected.error, err_msg)
+
+		if len(test.expected.xml_version) > 0 {
+			xml_version := ""
+			for attr in doc.prolog {
+				if attr.key == "version" {
+					xml_version = attr.val
+				}
+			}
+
+			err_msg  = tprintf("Expected XML version %v, got %v", test.expected.xml_version, xml_version)
+			expect(t, xml_version == test.expected.xml_version, err_msg)
+		}
+
+		if len(test.expected.xml_encoding) > 0 {
+			xml_encoding := ""
+			for attr in doc.prolog {
+				if attr.key == "encoding" {
+					xml_encoding = attr.val
+				}
+			}
+
+			err_msg  = tprintf("Expected XML encoding %v, got %v", test.expected.xml_encoding, xml_encoding)
+			expect(t, xml_encoding == test.expected.xml_encoding, err_msg)
+		}
+
+		err_msg  = tprintf("Expected DOCTYPE %v, got %v", test.expected.doctype, doc.doctype.ident)
+		expect(t, doc.doctype.ident == test.expected.doctype, err_msg)
+
+		/*
+			File-specific tests.
+		*/
+		switch count {
+		case 0:
+			expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.")
+			attr := doc.root.attribs[0]
+
+			attr_key_expected := "올빼미_id"
+			attr_val_expected := "Foozle&#32;<![CDATA[<greeting>Hello, world!\"</greeting>]]>Barzle"
+
+			attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key)
+			expect(t, attr.key == attr_key_expected, attr_err)
+
+			attr_err  = tprintf("Expected %v, got %v", attr_val_expected, attr.val)
+			expect(t, attr.val == attr_val_expected, attr_err)
+
+			expect(t, len(doc.root.children) > 0, "Expected the root tag to have children.")
+			child := doc.root.children[0]
+
+			first_child_ident := "부끄러운:barzle"
+			attr_err  = tprintf("Expected first child tag's ident to be %v, got %v", first_child_ident, child.ident)
+			expect(t, child.ident == first_child_ident, attr_err)
+
+		case 2:
+			expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.")
+
+			{
+				attr := doc.root.attribs[0]
+
+				attr_key_expected := "version"
+				attr_val_expected := "1.2"
+
+				attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key)
+				expect(t, attr.key == attr_key_expected, attr_err)
+
+				attr_err  = tprintf("Expected %v, got %v", attr_val_expected, attr.val)
+				expect(t, attr.val == attr_val_expected, attr_err)
+			}
+
+			{
+				attr := doc.root.attribs[1]
+
+				attr_key_expected := "xmlns"
+				attr_val_expected := "urn:oasis:names:tc:xliff:document:1.2"
+
+				attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key)
+				expect(t, attr.key == attr_key_expected, attr_err)
+
+				attr_err  = tprintf("Expected %v, got %v", attr_val_expected, attr.val)
+				expect(t, attr.val == attr_val_expected, attr_err)
+			}
+
+		case 3:
+			expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.")
+
+			{
+				attr := doc.root.attribs[0]
+
+				attr_key_expected := "xmlns"
+				attr_val_expected := "urn:oasis:names:tc:xliff:document:2.0"
+
+				attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key)
+				expect(t, attr.key == attr_key_expected, attr_err)
+
+				attr_err  = tprintf("Expected %v, got %v", attr_val_expected, attr.val)
+				expect(t, attr.val == attr_val_expected, attr_err)
+			}
+
+			{
+				attr := doc.root.attribs[1]
+
+				attr_key_expected := "version"
+				attr_val_expected := "2.0"
+
+				attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key)
+				expect(t, attr.key == attr_key_expected, attr_err)
+
+				attr_err  = tprintf("Expected %v, got %v", attr_val_expected, attr.val)
+				expect(t, attr.val == attr_val_expected, attr_err)
+			}
+		}
+
+		count += 1
+	}
+}
+\ No newline at end of file
author	Jeroen van Rijn <Kelimion@users.noreply.github.com>	2021-11-30 23:01:22 +0100
committer	Jeroen van Rijn <Kelimion@users.noreply.github.com>	2021-12-05 02:52:22 +0100
commit	b5c828fe4ee3f0942b2eda1dc5753e4ad6d38ea9 (patch)
tree	ffbd45adb60e3de951dc2948801d5a57b21dd2c9 /tests
parent	6ce5608003e630bc0de1c591fd4cbea3fe59e1d3 (diff)