From d0f87913e2133b8101faef6ea76e0853d4da524b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 2 Dec 2024 10:49:49 +0000 Subject: Fix #4549 --- core/encoding/json/tokenizer.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/json/tokenizer.odin b/core/encoding/json/tokenizer.odin index 5c20a2cc3..1a57ba6f0 100644 --- a/core/encoding/json/tokenizer.odin +++ b/core/encoding/json/tokenizer.odin @@ -485,7 +485,7 @@ is_valid_string_literal :: proc(str: string, spec: Specification) -> bool { case '"': // okay case '\'': - if spec != .JSON { + if spec == .JSON { return false } // okay -- cgit v1.2.3 From 37fb2754a1d98cd3b8738717183543924c66524b Mon Sep 17 00:00:00 2001 From: dozn <16659513+dozn@users.noreply.github.com> Date: Thu, 5 Dec 2024 07:23:34 -0800 Subject: Move Struct Field Zipping Outside of JSON Token Loop --- core/encoding/json/unmarshal.odin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 447397de4..33fd104b7 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -417,15 +417,15 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm if .raw_union in t.flags { return UNSUPPORTED_TYPE } - + + fields := reflect.struct_fields_zipped(ti.id) + struct_loop: for p.curr_token.kind != end_token { key := parse_object_key(p, p.allocator) or_return defer delete(key, p.allocator) unmarshal_expect_token(p, .Colon) - fields := reflect.struct_fields_zipped(ti.id) - field_test :: #force_inline proc "contextless" (field_used: [^]byte, offset: uintptr) -> bool { prev_set := field_used[offset/8] & byte(offset&7) != 0 field_used[offset/8] |= byte(offset&7) -- cgit v1.2.3 From d452d37b93cb3318f995adc74f84076c1b3103a5 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 5 Dec 2024 15:51:05 +0000 Subject: Fix #4537 --- core/encoding/json/tokenizer.odin | 1 + 1 file changed, 1 insertion(+) (limited to 'core/encoding') diff --git a/core/encoding/json/tokenizer.odin b/core/encoding/json/tokenizer.odin index 1a57ba6f0..e46d879a7 100644 --- a/core/encoding/json/tokenizer.odin +++ b/core/encoding/json/tokenizer.odin @@ -259,6 +259,7 @@ get_token :: proc(t: ^Tokenizer) -> (token: Token, err: Error) { skip_digits(t) } if t.r == 'e' || t.r == 'E' { + token.kind = .Float switch r := next_rune(t); r { case '+', '-': next_rune(t) -- cgit v1.2.3 From 7edd332993b485ebdd81ccce5af5dacb16e72156 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 6 Dec 2024 22:12:52 +0100 Subject: fix #4536 - stack buffer overflow for size_of 0 types in struct unmarshal --- core/encoding/json/unmarshal.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 33fd104b7..c70b8d39a 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -433,7 +433,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm } field_used_bytes := (reflect.size_of_typeid(ti.id)+7)/8 - field_used := intrinsics.alloca(field_used_bytes, 1) + field_used := intrinsics.alloca(field_used_bytes + 1, 1) // + 1 to not overflow on size_of 0 types. intrinsics.mem_zero(field_used, field_used_bytes) use_field_idx := -1 -- cgit v1.2.3 From 8c761627c84847bae3ec5e77a1408c542d9460b7 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 02:17:57 +0100 Subject: encoding/base32: Replace assertions with error returns Replace assertions with proper error handling in base32.decode() to allow programs to handle invalid input gracefully rather than crashing. The function now returns ([]byte, Error) instead of just []byte. --- core/encoding/base32/base32.odin | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index f3320428d..688b27544 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -7,6 +7,15 @@ package encoding_base32 // Incase your specific version does not use padding, you may // truncate it from the encoded output. +// Error represents errors that can occur during base32 decoding operations. +// See RFC 4648 sections 3.2, 4 and 6. +Error :: enum { + None, + Invalid_Character, // Input contains characters outside of base32 alphabet (A-Z, 2-7) + Invalid_Length, // Input length is not valid for base32 (must be a multiple of 8 with proper padding) + Malformed_Input, // Input has improper structure (wrong padding position or incomplete groups) +} + ENC_TABLE := [32]byte { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', @@ -91,9 +100,9 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al } } -decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> []byte #no_bounds_check{ +decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> ([]byte, Error) #no_bounds_check { if len(data) == 0 { - return nil + return nil, .None } outi := 0 @@ -113,16 +122,29 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato input := data[0] data = data[1:] if input == byte(PADDING) && j >= 2 && len(data) < 8 { - assert(!(len(data) + j < 8 - 1), "Corrupted input") - for k := 0; k < 8-1-j; k +=1 { - assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input") + // assert(!(len(data) + j < 8 - 1), "Corrupted input") + if len(data) + j < 8 - 1 { + return nil, .Malformed_Input + } + // assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input") + for k := 0; k < 8-1-j; k += 1 { + if len(data) < k || data[k] != byte(PADDING) { + return nil, .Malformed_Input + } } dlen, end = j, true - assert(dlen != 1 && dlen != 3 && dlen != 6, "Corrupted input") + // assert(dlen != 1 && dlen != 3 && dlen != 6, "Corrupted input") + if dlen == 1 || dlen == 3 || dlen == 6 { + return nil, .Invalid_Length + } break } - dbuf[j] = DEC_TABLE[input] - assert(dbuf[j] != 0xff, "Corrupted input") + decoded := DEC_TBL[input] + // assert(dbuf[j] != 0xff, "Corrupted input") + if decoded == 0 && input != byte(ENC_TABLE[0]) { + return nil, .Invalid_Character + } + dbuf[j] = decoded j += 1 } @@ -144,5 +166,5 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato } outi += 5 } - return out + return out, .None } -- cgit v1.2.3 From b9338777e34006b40b9315e795232e0608caa499 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 02:20:32 +0100 Subject: encoding/base32: Fix buffer allocation and bounds checking Fix buffer allocation size calculation and add proper bounds checking to ensure output buffer has sufficient space. This fixes crashes that could occur with inputs like "AA" and other edge cases where the output buffer was too small. Remove #no_bounds_check as proper bounds checking is necessary for safe error handling. The small performance trade-off is worth the improved robustness. --- core/encoding/base32/base32.odin | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 688b27544..8e3499dce 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -100,15 +100,18 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al } } -decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> ([]byte, Error) #no_bounds_check { +decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> ([]byte, Error) { if len(data) == 0 { return nil, .None } + // Calculate maximum possible output size and allocate buffer + out_len := (len(data) * 5 + 7) / 8 // Ceiling division to ensure enough space + out := make([]byte, out_len, allocator) + outi := 0 data := data - out := make([]byte, len(data) / 8 * 5, allocator) end := false for len(data) > 0 && !end { dbuf : [8]byte @@ -122,25 +125,22 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato input := data[0] data = data[1:] if input == byte(PADDING) && j >= 2 && len(data) < 8 { - // assert(!(len(data) + j < 8 - 1), "Corrupted input") if len(data) + j < 8 - 1 { return nil, .Malformed_Input } - // assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input") for k := 0; k < 8-1-j; k += 1 { if len(data) < k || data[k] != byte(PADDING) { return nil, .Malformed_Input } } dlen, end = j, true - // assert(dlen != 1 && dlen != 3 && dlen != 6, "Corrupted input") if dlen == 1 || dlen == 3 || dlen == 6 { return nil, .Invalid_Length } break } + decoded := DEC_TBL[input] - // assert(dbuf[j] != 0xff, "Corrupted input") if decoded == 0 && input != byte(ENC_TABLE[0]) { return nil, .Invalid_Character } @@ -148,23 +148,41 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato j += 1 } + // Ensure we have enough space in output buffer + needed := 5 // Each full 8-char block produces 5 bytes + if outi + needed > len(out) { + return nil, .Invalid_Length + } + + // Process complete input blocks switch dlen { case 8: + if len(dbuf) < 8 { return nil, .Invalid_Length } out[outi + 4] = dbuf[6] << 5 | dbuf[7] fallthrough case 7: + if len(dbuf) < 7 { return nil, .Invalid_Length } out[outi + 3] = dbuf[4] << 7 | dbuf[5] << 2 | dbuf[6] >> 3 fallthrough case 5: + if len(dbuf) < 5 { return nil, .Invalid_Length } out[outi + 2] = dbuf[3] << 4 | dbuf[4] >> 1 fallthrough case 4: + if len(dbuf) < 4 { return nil, .Invalid_Length } out[outi + 1] = dbuf[1] << 6 | dbuf[2] << 1 | dbuf[3] >> 4 fallthrough case 2: + if len(dbuf) < 2 { return nil, .Invalid_Length } out[outi + 0] = dbuf[0] << 3 | dbuf[1] >> 2 } outi += 5 } + + // Trim output buffer to actual size + if outi < len(out) { + out = out[:outi] + } + return out, .None } -- cgit v1.2.3 From 7672ac945a7b8e35b600a9d2a2421caf16a5a364 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 15:28:34 +0100 Subject: encoding/base32: Add RFC 4648 test suite Add test suite based on RFC 4648 test vectors and validation rules: - Add section 10 test vectors for valid encoding/decoding - Add test cases for invalid character handling (section 3.2) - Add test cases for padding validation (section 4) - Add test cases for length requirements (section 6) The test vectors verify that: - Empty string encodes/decodes correctly - Standard cases like "foo" -> "MZXW6===" work - Invalid characters are rejected - Missing or malformed padding is detected - Invalid lengths are caught --- core/encoding/base32/base32.odin | 104 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 8e3499dce..54737c9ce 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -1,5 +1,8 @@ package encoding_base32 +import "core:testing" +import "core:bytes" + // @note(zh): Encoding utility for Base32 // A secondary param can be used to supply a custom alphabet to // @link(encode) and a matching decoding table to @link(decode). @@ -186,3 +189,104 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato return out, .None } + +@(test) +test_base32_decode_valid :: proc(t: ^testing.T) { + // RFC 4648 Section 10 - Test vectors + cases := [?]struct { + input, expected: string, + }{ + {"", ""}, + {"MY======", "f"}, + {"MZXQ====", "fo"}, + {"MZXW6===", "foo"}, + {"MZXW6YQ=", "foob"}, + {"MZXW6YTB", "fooba"}, + {"MZXW6YTBOI======", "foobar"}, + } + + + for c in cases { + output, err := decode(c.input) + testing.expect_value(t, err, Error.None) + expected := transmute([]u8)c.expected + + if output != nil { + testing.expect(t, bytes.equal(output, expected)) + } else { + testing.expect(t, len(c.expected) == 0) + } + } +} + +@(test) +test_base32_encode :: proc(t: ^testing.T) { + // RFC 4648 Section 10 - Test vectors + cases := [?]struct { + input, expected: string, + }{ + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + } + + for c in cases { + output := encode(transmute([]byte)c.input) + testing.expect(t, output == c.expected) + } +} + +@(test) +test_base32_decode_invalid :: proc(t: ^testing.T) { + // Section 3.2 - Alphabet check + { + // Characters outside alphabet + input := "MZ1W6YTB" // '1' not in alphabet (A-Z, 2-7) + _, err := decode(input) + testing.expect_value(t, err, Error.Invalid_Character) + } + { + // Lowercase not allowed + input := "mzxq====" + _, err := decode(input) + testing.expect_value(t, err, Error.Invalid_Character) + } + + // Section 4 - Padding requirements + { + // Padding must only be at end + input := "MZ=Q====" + _, err := decode(input) + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Missing padding + input := "MZXQ" // Should be MZXQ==== + _, err := decode(input) + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Incorrect padding length + input := "MZXQ=" // Needs 4 padding chars + _, err := decode(input) + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Too much padding + input := "MY=========" // Extra padding chars + _, err := decode(input) + testing.expect_value(t, err, Error.Malformed_Input) + } + + // Section 6 - Block size requirements + { + // Single character (invalid block) + input := "M" + _, err := decode(input) + testing.expect_value(t, err, Error.Invalid_Length) + } +} -- cgit v1.2.3 From f1f2ed31940d02728ce55e591743834968dd5994 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 15:52:33 +0100 Subject: encoding/base32: Fix decode implementation per RFC 4648 Rework base32.decode() to properly handle all cases per RFC 4648: - Fix error detection order: - Check minimum length first (Invalid_Length) - Check character validity (Invalid_Character) - Check padding and structure (Malformed_Input) - Fix padding validation: - Add required padding length checks (2=6, 4=4, 5=3, 7=1 chars) - Ensure padding only appears at end - Fix handling of unpadded inputs - Fix buffer handling: - Proper output buffer size calculation - Add bounds checking for buffer access - Add proper buffer validation For example: - "M" correctly returns Invalid_Length (too short) - "mzxq====" correctly returns Invalid_Character (lowercase) - "MZXQ=" correctly returns Malformed_Input (wrong padding) - Unpadded input lengths must be multiples of 8 These changes make the decode function fully compliant with RFC 4648 requirements while providing proper error handling. --- core/encoding/base32/base32.odin | 137 ++++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 58 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 54737c9ce..d940c856b 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -103,88 +103,109 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al } } -decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> ([]byte, Error) { +decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> (out: []byte, err: Error) { if len(data) == 0 { return nil, .None } - // Calculate maximum possible output size and allocate buffer - out_len := (len(data) * 5 + 7) / 8 // Ceiling division to ensure enough space - out := make([]byte, out_len, allocator) + // Check minimum length requirement first + if len(data) < 2 { + return nil, .Invalid_Length + } - outi := 0 - data := data + // Validate characters - only A-Z and 2-7 allowed before padding + for i := 0; i < len(data); i += 1 { + c := data[i] + if c == byte(PADDING) { + break + } + if !((c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7')) { + return nil, .Invalid_Character + } + } - end := false - for len(data) > 0 && !end { - dbuf : [8]byte - dlen := 8 + // Validate padding and length + data_len := len(data) + padding_count := 0 + for i := data_len - 1; i >= 0; i -= 1 { + if data[i] != byte(PADDING) { + break + } + padding_count += 1 + } - for j := 0; j < 8; { - if len(data) == 0 { - dlen, end = j, true - break - } - input := data[0] - data = data[1:] - if input == byte(PADDING) && j >= 2 && len(data) < 8 { - if len(data) + j < 8 - 1 { - return nil, .Malformed_Input - } - for k := 0; k < 8-1-j; k += 1 { - if len(data) < k || data[k] != byte(PADDING) { - return nil, .Malformed_Input - } - } - dlen, end = j, true - if dlen == 1 || dlen == 3 || dlen == 6 { - return nil, .Invalid_Length - } - break + // Check for proper padding and length combinations + if padding_count > 0 { + // Verify no padding in the middle + for i := 0; i < data_len - padding_count; i += 1 { + if data[i] == byte(PADDING) { + return nil, .Malformed_Input } + } - decoded := DEC_TBL[input] - if decoded == 0 && input != byte(ENC_TABLE[0]) { - return nil, .Invalid_Character + // Required padding for each content length mod 8 + content_len := data_len - padding_count + required_padding := map[int]int{ + 2 = 6, // 2 chars need 6 padding chars + 4 = 4, // 4 chars need 4 padding chars + 5 = 3, // 5 chars need 3 padding chars + 7 = 1, // 7 chars need 1 padding char + } + + mod8 := content_len % 8 + if req_pad, ok := required_padding[mod8]; ok { + if padding_count != req_pad { + return nil, .Malformed_Input } - dbuf[j] = decoded - j += 1 + } else if mod8 != 0 { + // If not in the map and not a multiple of 8, it's invalid + return nil, .Malformed_Input + } + } else { + // No padding - must be multiple of 8 + if data_len % 8 != 0 { + return nil, .Malformed_Input } + } - // Ensure we have enough space in output buffer - needed := 5 // Each full 8-char block produces 5 bytes - if outi + needed > len(out) { - return nil, .Invalid_Length + // Calculate decoded length: 5 bytes for every 8 input chars + input_chars := data_len - padding_count + out_len := input_chars * 5 / 8 + out = make([]byte, out_len, allocator) + defer if err != .None { + delete(out) + } + + // Process input in 8-byte blocks + outi := 0 + for i := 0; i < input_chars; i += 8 { + buf: [8]byte + block_size := min(8, input_chars - i) + + // Decode block + for j := 0; j < block_size; j += 1 { + buf[j] = DEC_TBL[data[i + j]] } - // Process complete input blocks - switch dlen { + // Convert to output bytes based on block size + bytes_to_write := block_size * 5 / 8 + switch block_size { case 8: - if len(dbuf) < 8 { return nil, .Invalid_Length } - out[outi + 4] = dbuf[6] << 5 | dbuf[7] + out[outi + 4] = (buf[6] << 5) | buf[7] fallthrough case 7: - if len(dbuf) < 7 { return nil, .Invalid_Length } - out[outi + 3] = dbuf[4] << 7 | dbuf[5] << 2 | dbuf[6] >> 3 + out[outi + 3] = (buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3) fallthrough case 5: - if len(dbuf) < 5 { return nil, .Invalid_Length } - out[outi + 2] = dbuf[3] << 4 | dbuf[4] >> 1 + out[outi + 2] = (buf[3] << 4) | (buf[4] >> 1) fallthrough case 4: - if len(dbuf) < 4 { return nil, .Invalid_Length } - out[outi + 1] = dbuf[1] << 6 | dbuf[2] << 1 | dbuf[3] >> 4 + out[outi + 1] = (buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4) fallthrough case 2: - if len(dbuf) < 2 { return nil, .Invalid_Length } - out[outi + 0] = dbuf[0] << 3 | dbuf[1] >> 2 + out[outi] = (buf[0] << 3) | (buf[1] >> 2) } - outi += 5 - } - - // Trim output buffer to actual size - if outi < len(out) { - out = out[:outi] + outi += bytes_to_write } return out, .None -- cgit v1.2.3 From 93238db202c6648d11d4a78a83b1a29751ac77a3 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 16:00:00 +0100 Subject: encoding/base32: Use consistent allocator and add proper cleanup Fix memory handling throughout base32 package: - Make padding map package-level constant (to avoid repeated allocs) - Use passed allocator in encode's make() call - Add defer delete for allocated memory in encode - Add proper cleanup in test cases - Fix memory cleanup of output buffers The changes ensure consistent allocator usage and cleanup in both implementation and tests. --- core/encoding/base32/base32.odin | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index d940c856b..72e20e427 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -45,11 +45,19 @@ DEC_TABLE := [?]u8 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } +REQUIRED_PADDING := map[int]int{ + 2 = 6, // 2 chars need 6 padding chars + 4 = 4, // 4 chars need 4 padding chars + 5 = 3, // 5 chars need 3 padding chars + 7 = 1, // 7 chars need 1 padding char +} + encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { out_length := (len(data) + 4) / 5 * 8 - out := make([]byte, out_length) + out := make([]byte, out_length, allocator) + defer delete(out) _encode(out, data) - return string(out) + return string(out[:]) } @private @@ -143,22 +151,13 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato } } - // Required padding for each content length mod 8 content_len := data_len - padding_count - required_padding := map[int]int{ - 2 = 6, // 2 chars need 6 padding chars - 4 = 4, // 4 chars need 4 padding chars - 5 = 3, // 5 chars need 3 padding chars - 7 = 1, // 7 chars need 1 padding char - } - mod8 := content_len % 8 - if req_pad, ok := required_padding[mod8]; ok { + if req_pad, ok := REQUIRED_PADDING[mod8]; ok { if padding_count != req_pad { return nil, .Malformed_Input } } else if mod8 != 0 { - // If not in the map and not a multiple of 8, it's invalid return nil, .Malformed_Input } } else { @@ -208,7 +207,7 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato outi += bytes_to_write } - return out, .None + return } @(test) @@ -226,12 +225,13 @@ test_base32_decode_valid :: proc(t: ^testing.T) { {"MZXW6YTBOI======", "foobar"}, } - for c in cases { output, err := decode(c.input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.None) expected := transmute([]u8)c.expected - if output != nil { testing.expect(t, bytes.equal(output, expected)) } else { @@ -267,13 +267,19 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { { // Characters outside alphabet input := "MZ1W6YTB" // '1' not in alphabet (A-Z, 2-7) - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Invalid_Character) } { // Lowercase not allowed input := "mzxq====" - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Invalid_Character) } @@ -281,25 +287,37 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { { // Padding must only be at end input := "MZ=Q====" - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Malformed_Input) } { // Missing padding input := "MZXQ" // Should be MZXQ==== - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Malformed_Input) } { // Incorrect padding length input := "MZXQ=" // Needs 4 padding chars - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Malformed_Input) } { // Too much padding input := "MY=========" // Extra padding chars - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Malformed_Input) } @@ -307,7 +325,10 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { { // Single character (invalid block) input := "M" - _, err := decode(input) + output, err := decode(input) + if output != nil { + defer delete(output) + } testing.expect_value(t, err, Error.Invalid_Length) } } -- cgit v1.2.3 From e75a49f095ed7ecd72b4caccf550545a980163ab Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 16:07:01 +0100 Subject: encoding/base32: Set optimization mode for decode() --- core/encoding/base32/base32.odin | 1 + 1 file changed, 1 insertion(+) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 72e20e427..53d31fb30 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -111,6 +111,7 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al } } +@(optimization_mode="favor_size") decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> (out: []byte, err: Error) { if len(data) == 0 { return nil, .None -- cgit v1.2.3 From 8211a911dbf75b683da4f5c1c3f2970d56103497 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 24 Dec 2024 20:46:38 +0100 Subject: encoding/base32: Replace padding map with switch statement Replace package-level map with a simple switch statement for padding validation. This eliminates allocations we can't properly free while maintaining the same validation logic. --- core/encoding/base32/base32.odin | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 53d31fb30..68b8b7a5e 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -45,13 +45,6 @@ DEC_TABLE := [?]u8 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } -REQUIRED_PADDING := map[int]int{ - 2 = 6, // 2 chars need 6 padding chars - 4 = 4, // 4 chars need 4 padding chars - 5 = 3, // 5 chars need 3 padding chars - 7 = 1, // 7 chars need 1 padding char -} - encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { out_length := (len(data) + 4) / 5 * 8 out := make([]byte, out_length, allocator) @@ -154,8 +147,17 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato content_len := data_len - padding_count mod8 := content_len % 8 - if req_pad, ok := REQUIRED_PADDING[mod8]; ok { - if padding_count != req_pad { + required_padding: int + switch mod8 { + case 2: required_padding = 6 // 2 chars need 6 padding chars + case 4: required_padding = 4 // 4 chars need 4 padding chars + case 5: required_padding = 3 // 5 chars need 3 padding chars + case 7: required_padding = 1 // 7 chars need 1 padding char + case: required_padding = 0 + } + + if required_padding > 0 { + if padding_count != required_padding { return nil, .Malformed_Input } } else if mod8 != 0 { -- cgit v1.2.3 From e7fb02a84a24f5430199249934a3dc37b10a4d39 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Wed, 25 Dec 2024 16:15:41 +0100 Subject: encoding/base32: Add custom validation support Add support for custom alphabet validation through an optional validation function parameter. The default validation follows RFC 4648 base32 alphabet rules (A-Z, 2-7). This properly supports the documented ability to use custom alphabets. --- core/encoding/base32/base32.odin | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 68b8b7a5e..7c70b7e9a 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -7,18 +7,25 @@ import "core:bytes" // A secondary param can be used to supply a custom alphabet to // @link(encode) and a matching decoding table to @link(decode). // If none is supplied it just uses the standard Base32 alphabet. -// Incase your specific version does not use padding, you may +// In case your specific version does not use padding, you may // truncate it from the encoded output. // Error represents errors that can occur during base32 decoding operations. // See RFC 4648 sections 3.2, 4 and 6. Error :: enum { None, - Invalid_Character, // Input contains characters outside of base32 alphabet (A-Z, 2-7) + Invalid_Character, // Input contains characters outside the specified alphabet Invalid_Length, // Input length is not valid for base32 (must be a multiple of 8 with proper padding) Malformed_Input, // Input has improper structure (wrong padding position or incomplete groups) } +Validate_Proc :: #type proc(c: byte) -> bool + +@private +_validate_default :: proc(c: byte) -> bool { + return (c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7') +} + ENC_TABLE := [32]byte { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', @@ -105,7 +112,11 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al } @(optimization_mode="favor_size") -decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> (out: []byte, err: Error) { +decode :: proc( + data: string, + DEC_TBL := DEC_TABLE, + validate: Validate_Proc = _validate_default, + allocator := context.allocator) -> (out: []byte, err: Error) { if len(data) == 0 { return nil, .None } @@ -115,13 +126,13 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato return nil, .Invalid_Length } - // Validate characters - only A-Z and 2-7 allowed before padding + // Validate characters using provided validation function for i := 0; i < len(data); i += 1 { c := data[i] if c == byte(PADDING) { break } - if !((c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7')) { + if !validate(c) { return nil, .Invalid_Character } } -- cgit v1.2.3 From 88c0e62095730354d139af3b90b7d91fa19f5e1a Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Thu, 26 Dec 2024 14:48:02 +0100 Subject: encoding/base32: Use `ENC_TBL` parameter consistently in encode() Fix encoding to properly use provided encoding table parameter instead of hardcoded `ENC_TABLE`. This makes encode properly support custom alphabets as documented. --- core/encoding/base32/base32.odin | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 7c70b7e9a..ea529ed63 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -56,7 +56,7 @@ encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocato out_length := (len(data) + 4) / 5 * 8 out := make([]byte, out_length, allocator) defer delete(out) - _encode(out, data) + _encode(out, data, ENC_TBL) return string(out[:]) } @@ -69,26 +69,26 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al carry: byte switch len(data) { case: - out[7] = ENC_TABLE[data[4] & 0x1f] + out[7] = ENC_TBL[data[4] & 0x1f] carry = data[4] >> 5 fallthrough case 4: - out[6] = ENC_TABLE[carry | (data[3] << 3) & 0x1f] - out[5] = ENC_TABLE[(data[3] >> 2) & 0x1f] + out[6] = ENC_TBL[carry | (data[3] << 3) & 0x1f] + out[5] = ENC_TBL[(data[3] >> 2) & 0x1f] carry = data[3] >> 7 fallthrough case 3: - out[4] = ENC_TABLE[carry | (data[2] << 1) & 0x1f] + out[4] = ENC_TBL[carry | (data[2] << 1) & 0x1f] carry = (data[2] >> 4) & 0x1f fallthrough case 2: - out[3] = ENC_TABLE[carry | (data[1] << 4) & 0x1f] - out[2] = ENC_TABLE[(data[1] >> 1) & 0x1f] + out[3] = ENC_TBL[carry | (data[1] << 4) & 0x1f] + out[2] = ENC_TBL[(data[1] >> 1) & 0x1f] carry = (data[1] >> 6) & 0x1f fallthrough case 1: - out[1] = ENC_TABLE[carry | (data[0] << 2) & 0x1f] - out[0] = ENC_TABLE[data[0] >> 3] + out[1] = ENC_TBL[carry | (data[0] << 2) & 0x1f] + out[0] = ENC_TBL[data[0] >> 3] } if len(data) < 5 { -- cgit v1.2.3 From 490f52700533bdda725fd90fc19cb248d38b2ff5 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Thu, 26 Dec 2024 19:20:46 +0100 Subject: encoding/base32: Expand `DEC_TABLE` to full 256 bytes The decoding table was only 224 bytes which caused type mismatches when using custom alphabets, so expand with zeroes to cover full byte range while maintaining the same decoding logic. --- core/encoding/base32/base32.odin | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index ea529ed63..12b6a426b 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -35,21 +35,23 @@ ENC_TABLE := [32]byte { PADDING :: '=' -DEC_TABLE := [?]u8 { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, +DEC_TABLE := [256]u8 { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { -- cgit v1.2.3 From d22cb20d85e0afeb61f6ad26434dd92b9d9b2ff7 Mon Sep 17 00:00:00 2001 From: dozn <16659513+dozn@users.noreply.github.com> Date: Sat, 28 Dec 2024 07:48:09 -0800 Subject: Use Struct Tags For Embedded (with `using`) Structs When Unmarshalling JSON A fix for https://github.com/odin-lang/Odin/issues/4539 --- core/encoding/json/unmarshal.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index c70b8d39a..e76de2747 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -470,7 +470,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm } } - if field.name == key { + if field.name == key || (field.tag != "" && reflect.struct_tag_get(field.tag, "json") == key) { offset = field.offset type = field.type found = true -- cgit v1.2.3 From 87c159c69fd699312fa014700c03f43188dd0728 Mon Sep 17 00:00:00 2001 From: dozn <16659513+dozn@users.noreply.github.com> Date: Sat, 28 Dec 2024 08:13:38 -0800 Subject: Remove unnecessary string() conversion. --- core/encoding/json/unmarshal.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index e76de2747..57371e360 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -439,7 +439,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm use_field_idx := -1 for field, field_idx in fields { - tag_value := string(reflect.struct_tag_get(field.tag, "json")) + tag_value := reflect.struct_tag_get(field.tag, "json") json_name, _ := json_name_from_tag_value(tag_value) if key == json_name { use_field_idx = field_idx -- cgit v1.2.3 From c9c59edc646082ad9a687a80ccfd4e421d7e15d4 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Sun, 29 Dec 2024 23:35:01 +0100 Subject: encoding/base32: Move tests to base32_test.odin Move existing test procedures to a dedicated test file for better code organization and maintainability. --- core/encoding/base32/base32.odin | 126 --------------------------------- core/encoding/base32/base32_test.odin | 127 ++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 126 deletions(-) create mode 100644 core/encoding/base32/base32_test.odin (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 12b6a426b..60ece7b26 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -1,8 +1,5 @@ package encoding_base32 -import "core:testing" -import "core:bytes" - // @note(zh): Encoding utility for Base32 // A secondary param can be used to supply a custom alphabet to // @link(encode) and a matching decoding table to @link(decode). @@ -225,126 +222,3 @@ decode :: proc( return } - -@(test) -test_base32_decode_valid :: proc(t: ^testing.T) { - // RFC 4648 Section 10 - Test vectors - cases := [?]struct { - input, expected: string, - }{ - {"", ""}, - {"MY======", "f"}, - {"MZXQ====", "fo"}, - {"MZXW6===", "foo"}, - {"MZXW6YQ=", "foob"}, - {"MZXW6YTB", "fooba"}, - {"MZXW6YTBOI======", "foobar"}, - } - - for c in cases { - output, err := decode(c.input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.None) - expected := transmute([]u8)c.expected - if output != nil { - testing.expect(t, bytes.equal(output, expected)) - } else { - testing.expect(t, len(c.expected) == 0) - } - } -} - -@(test) -test_base32_encode :: proc(t: ^testing.T) { - // RFC 4648 Section 10 - Test vectors - cases := [?]struct { - input, expected: string, - }{ - {"", ""}, - {"f", "MY======"}, - {"fo", "MZXQ===="}, - {"foo", "MZXW6==="}, - {"foob", "MZXW6YQ="}, - {"fooba", "MZXW6YTB"}, - {"foobar", "MZXW6YTBOI======"}, - } - - for c in cases { - output := encode(transmute([]byte)c.input) - testing.expect(t, output == c.expected) - } -} - -@(test) -test_base32_decode_invalid :: proc(t: ^testing.T) { - // Section 3.2 - Alphabet check - { - // Characters outside alphabet - input := "MZ1W6YTB" // '1' not in alphabet (A-Z, 2-7) - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Invalid_Character) - } - { - // Lowercase not allowed - input := "mzxq====" - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Invalid_Character) - } - - // Section 4 - Padding requirements - { - // Padding must only be at end - input := "MZ=Q====" - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Malformed_Input) - } - { - // Missing padding - input := "MZXQ" // Should be MZXQ==== - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Malformed_Input) - } - { - // Incorrect padding length - input := "MZXQ=" // Needs 4 padding chars - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Malformed_Input) - } - { - // Too much padding - input := "MY=========" // Extra padding chars - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Malformed_Input) - } - - // Section 6 - Block size requirements - { - // Single character (invalid block) - input := "M" - output, err := decode(input) - if output != nil { - defer delete(output) - } - testing.expect_value(t, err, Error.Invalid_Length) - } -} diff --git a/core/encoding/base32/base32_test.odin b/core/encoding/base32/base32_test.odin new file mode 100644 index 000000000..e492f9a85 --- /dev/null +++ b/core/encoding/base32/base32_test.odin @@ -0,0 +1,127 @@ +package encoding_base32 + +import "core:testing" +import "core:bytes" + +@(test) +test_base32_decode_valid :: proc(t: ^testing.T) { + // RFC 4648 Section 10 - Test vectors + cases := [?]struct { + input, expected: string, + }{ + {"", ""}, + {"MY======", "f"}, + {"MZXQ====", "fo"}, + {"MZXW6===", "foo"}, + {"MZXW6YQ=", "foob"}, + {"MZXW6YTB", "fooba"}, + {"MZXW6YTBOI======", "foobar"}, + } + + for c in cases { + output, err := decode(c.input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.None) + expected := transmute([]u8)c.expected + if output != nil { + testing.expect(t, bytes.equal(output, expected)) + } else { + testing.expect(t, len(c.expected) == 0) + } + } +} + +@(test) +test_base32_encode :: proc(t: ^testing.T) { + // RFC 4648 Section 10 - Test vectors + cases := [?]struct { + input, expected: string, + }{ + {"", ""}, + {"f", "MY======"}, + {"fo", "MZXQ===="}, + {"foo", "MZXW6==="}, + {"foob", "MZXW6YQ="}, + {"fooba", "MZXW6YTB"}, + {"foobar", "MZXW6YTBOI======"}, + } + + for c in cases { + output := encode(transmute([]byte)c.input) + testing.expect(t, output == c.expected) + } +} + +@(test) +test_base32_decode_invalid :: proc(t: ^testing.T) { + // Section 3.2 - Alphabet check + { + // Characters outside alphabet + input := "MZ1W6YTB" // '1' not in alphabet (A-Z, 2-7) + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Invalid_Character) + } + { + // Lowercase not allowed + input := "mzxq====" + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Invalid_Character) + } + + // Section 4 - Padding requirements + { + // Padding must only be at end + input := "MZ=Q====" + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Missing padding + input := "MZXQ" // Should be MZXQ==== + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Incorrect padding length + input := "MZXQ=" // Needs 4 padding chars + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Malformed_Input) + } + { + // Too much padding + input := "MY=========" // Extra padding chars + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Malformed_Input) + } + + // Section 6 - Block size requirements + { + // Single character (invalid block) + input := "M" + output, err := decode(input) + if output != nil { + defer delete(output) + } + testing.expect_value(t, err, Error.Invalid_Length) + } +} -- cgit v1.2.3 From 0d4c0064d961d8a50901afd8c359962b687d1cbd Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Mon, 30 Dec 2024 03:03:50 +0100 Subject: encoding/base32: Add encode->decode roundtrip test Add test_base32_roundtrip() to verify the encode->decode roundtrip preserves data integrity. This test helps ensure our base32 implementation correctly handles the full encode->decode cycle without data loss or corruption. --- core/encoding/base32/base32_test.odin | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32_test.odin b/core/encoding/base32/base32_test.odin index e492f9a85..6589abc42 100644 --- a/core/encoding/base32/base32_test.odin +++ b/core/encoding/base32/base32_test.odin @@ -125,3 +125,26 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { testing.expect_value(t, err, Error.Invalid_Length) } } + +@(test) +test_base32_roundtrip :: proc(t: ^testing.T) { + cases := [?]string{ + "", + "f", + "fo", + "foo", + "foob", + "fooba", + "foobar", + } + + for input in cases { + encoded := encode(transmute([]byte)input) + decoded, err := decode(encoded) + if decoded != nil { + defer delete(decoded) + } + testing.expect_value(t, err, Error.None) + testing.expect(t, bytes.equal(decoded, transmute([]byte)input)) + } +} -- cgit v1.2.3 From 591dd8765adb04ff5e0ba66c4c583ea49dbbcc87 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Mon, 30 Dec 2024 12:00:38 +0100 Subject: encoding/base32: Remove incorrect defer delete in encode() Remove premature deallocation of the output buffer which was causing use-after-free behavior. The returned string needs to take ownership of this memory, but the defer delete was freeing it before the string could be used. This fixes issues with encoding that were introduced by overly aggressive memory cleanup in 93238db2. --- core/encoding/base32/base32.odin | 1 - 1 file changed, 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 60ece7b26..c46d4a323 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -54,7 +54,6 @@ DEC_TABLE := [256]u8 { encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { out_length := (len(data) + 4) / 5 * 8 out := make([]byte, out_length, allocator) - defer delete(out) _encode(out, data, ENC_TBL) return string(out[:]) } -- cgit v1.2.3 From 82925097699c389475c5e2d12286a447165ffa65 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Mon, 30 Dec 2024 15:18:38 +0100 Subject: encoding/base32: Add custom alphabet test case Add test case to verify custom alphabet support. The test uses a decimal-uppercase alphabet (0-9, A-V) to test both encoding and decoding with custom tables, including validation. This ensures the encode and decode functions work correctly with custom encoding tables and validation functions as documented. --- core/encoding/base32/base32_test.odin | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32_test.odin b/core/encoding/base32/base32_test.odin index 6589abc42..968c6c4df 100644 --- a/core/encoding/base32/base32_test.odin +++ b/core/encoding/base32/base32_test.odin @@ -148,3 +148,78 @@ test_base32_roundtrip :: proc(t: ^testing.T) { testing.expect(t, bytes.equal(decoded, transmute([]byte)input)) } } + +@(test) + +test_base32_custom_alphabet :: proc(t: ^testing.T) { + custom_enc_table := [32]byte{ + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + } + + custom_dec_table: [256]u8 + for i := 0; i < len(custom_enc_table); i += 1 { + custom_dec_table[custom_enc_table[i]] = u8(i) + } + + /* + custom_dec_table := [256]u8{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x00-0x0f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x10-0x1f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x20-0x2f + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, // 0x30-0x3f ('0'-'9') + 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, // 0x40-0x4f ('A'-'O') + 25, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x50-0x5f ('P'-'V') + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60-0x6f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x70-0x7f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x80-0x8f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x90-0x9f + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xa0-0xaf + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xb0-0xbf + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xc0-0xcf + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xd0-0xdf + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xe0-0xef + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xf0-0xff + } + */ + + custom_validate :: proc(c: byte) -> bool { + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'V') || c == byte(PADDING) + } + + cases := [?]struct { + input: string, + enc_expected: string, + }{ + {"f", "CO======"}, + {"fo", "CPNG===="}, + {"foo", "CPNMU==="}, + } + + for c in cases { + // Test encoding + encoded := encode(transmute([]byte)c.input, custom_enc_table) + testing.expect(t, encoded == c.enc_expected) + + // Test decoding + decoded, err := decode(encoded, custom_dec_table, custom_validate) + defer if decoded != nil { + delete(decoded) + } + + testing.expect_value(t, err, Error.None) + testing.expect(t, bytes.equal(decoded, transmute([]byte)c.input)) + } + + // Test invalid character detection + { + input := "WXY=====" // Contains chars not in our alphabet + output, err := decode(input, custom_dec_table, custom_validate) + if output != nil { + delete(output) + } + testing.expect_value(t, err, Error.Invalid_Character) + } +} -- cgit v1.2.3 From 5ce6990077bf4d88a2e9617969e610b65b223f89 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Mon, 30 Dec 2024 15:26:42 +0100 Subject: encoding/base32: Add proper cleanup for encoded strings in tests Add defer delete for encoded strings across all test procedures to ensure proper cleanup and prevent memory leaks. This completes the memory management improvements started in 591dd876. --- core/encoding/base32/base32_test.odin | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32_test.odin b/core/encoding/base32/base32_test.odin index 968c6c4df..b032aa122 100644 --- a/core/encoding/base32/base32_test.odin +++ b/core/encoding/base32/base32_test.odin @@ -50,6 +50,7 @@ test_base32_encode :: proc(t: ^testing.T) { for c in cases { output := encode(transmute([]byte)c.input) + defer delete(output) testing.expect(t, output == c.expected) } } @@ -140,6 +141,7 @@ test_base32_roundtrip :: proc(t: ^testing.T) { for input in cases { encoded := encode(transmute([]byte)input) + defer delete(encoded) decoded, err := decode(encoded) if decoded != nil { defer delete(decoded) @@ -150,7 +152,6 @@ test_base32_roundtrip :: proc(t: ^testing.T) { } @(test) - test_base32_custom_alphabet :: proc(t: ^testing.T) { custom_enc_table := [32]byte{ '0', '1', '2', '3', '4', '5', '6', '7', @@ -201,6 +202,7 @@ test_base32_custom_alphabet :: proc(t: ^testing.T) { for c in cases { // Test encoding encoded := encode(transmute([]byte)c.input, custom_enc_table) + defer delete(encoded) testing.expect(t, encoded == c.enc_expected) // Test decoding -- cgit v1.2.3 From 3d25128520c558a5703c478eef61bb23326afe18 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Mon, 30 Dec 2024 15:31:57 +0100 Subject: encoding/base32: Convert files to UTF-8 with Unix line endings --- core/encoding/base32/base32.odin | 447 ++++++++++++++++++++------------------- 1 file changed, 224 insertions(+), 223 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index c46d4a323..d4e4cbe03 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -1,223 +1,224 @@ -package encoding_base32 - -// @note(zh): Encoding utility for Base32 -// A secondary param can be used to supply a custom alphabet to -// @link(encode) and a matching decoding table to @link(decode). -// If none is supplied it just uses the standard Base32 alphabet. -// In case your specific version does not use padding, you may -// truncate it from the encoded output. - -// Error represents errors that can occur during base32 decoding operations. -// See RFC 4648 sections 3.2, 4 and 6. -Error :: enum { - None, - Invalid_Character, // Input contains characters outside the specified alphabet - Invalid_Length, // Input length is not valid for base32 (must be a multiple of 8 with proper padding) - Malformed_Input, // Input has improper structure (wrong padding position or incomplete groups) -} - -Validate_Proc :: #type proc(c: byte) -> bool - -@private -_validate_default :: proc(c: byte) -> bool { - return (c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7') -} - -ENC_TABLE := [32]byte { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', - 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', - 'Y', 'Z', '2', '3', '4', '5', '6', '7', -} - -PADDING :: '=' - -DEC_TABLE := [256]u8 { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -} - -encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { - out_length := (len(data) + 4) / 5 * 8 - out := make([]byte, out_length, allocator) - _encode(out, data, ENC_TBL) - return string(out[:]) -} - -@private -_encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) { - out := out - data := data - - for len(data) > 0 { - carry: byte - switch len(data) { - case: - out[7] = ENC_TBL[data[4] & 0x1f] - carry = data[4] >> 5 - fallthrough - case 4: - out[6] = ENC_TBL[carry | (data[3] << 3) & 0x1f] - out[5] = ENC_TBL[(data[3] >> 2) & 0x1f] - carry = data[3] >> 7 - fallthrough - case 3: - out[4] = ENC_TBL[carry | (data[2] << 1) & 0x1f] - carry = (data[2] >> 4) & 0x1f - fallthrough - case 2: - out[3] = ENC_TBL[carry | (data[1] << 4) & 0x1f] - out[2] = ENC_TBL[(data[1] >> 1) & 0x1f] - carry = (data[1] >> 6) & 0x1f - fallthrough - case 1: - out[1] = ENC_TBL[carry | (data[0] << 2) & 0x1f] - out[0] = ENC_TBL[data[0] >> 3] - } - - if len(data) < 5 { - out[7] = byte(PADDING) - if len(data) < 4 { - out[6] = byte(PADDING) - out[5] = byte(PADDING) - if len(data) < 3 { - out[4] = byte(PADDING) - if len(data) < 2 { - out[3] = byte(PADDING) - out[2] = byte(PADDING) - } - } - } - break - } - data = data[5:] - out = out[8:] - } -} - -@(optimization_mode="favor_size") -decode :: proc( - data: string, - DEC_TBL := DEC_TABLE, - validate: Validate_Proc = _validate_default, - allocator := context.allocator) -> (out: []byte, err: Error) { - if len(data) == 0 { - return nil, .None - } - - // Check minimum length requirement first - if len(data) < 2 { - return nil, .Invalid_Length - } - - // Validate characters using provided validation function - for i := 0; i < len(data); i += 1 { - c := data[i] - if c == byte(PADDING) { - break - } - if !validate(c) { - return nil, .Invalid_Character - } - } - - // Validate padding and length - data_len := len(data) - padding_count := 0 - for i := data_len - 1; i >= 0; i -= 1 { - if data[i] != byte(PADDING) { - break - } - padding_count += 1 - } - - // Check for proper padding and length combinations - if padding_count > 0 { - // Verify no padding in the middle - for i := 0; i < data_len - padding_count; i += 1 { - if data[i] == byte(PADDING) { - return nil, .Malformed_Input - } - } - - content_len := data_len - padding_count - mod8 := content_len % 8 - required_padding: int - switch mod8 { - case 2: required_padding = 6 // 2 chars need 6 padding chars - case 4: required_padding = 4 // 4 chars need 4 padding chars - case 5: required_padding = 3 // 5 chars need 3 padding chars - case 7: required_padding = 1 // 7 chars need 1 padding char - case: required_padding = 0 - } - - if required_padding > 0 { - if padding_count != required_padding { - return nil, .Malformed_Input - } - } else if mod8 != 0 { - return nil, .Malformed_Input - } - } else { - // No padding - must be multiple of 8 - if data_len % 8 != 0 { - return nil, .Malformed_Input - } - } - - // Calculate decoded length: 5 bytes for every 8 input chars - input_chars := data_len - padding_count - out_len := input_chars * 5 / 8 - out = make([]byte, out_len, allocator) - defer if err != .None { - delete(out) - } - - // Process input in 8-byte blocks - outi := 0 - for i := 0; i < input_chars; i += 8 { - buf: [8]byte - block_size := min(8, input_chars - i) - - // Decode block - for j := 0; j < block_size; j += 1 { - buf[j] = DEC_TBL[data[i + j]] - } - - // Convert to output bytes based on block size - bytes_to_write := block_size * 5 / 8 - switch block_size { - case 8: - out[outi + 4] = (buf[6] << 5) | buf[7] - fallthrough - case 7: - out[outi + 3] = (buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3) - fallthrough - case 5: - out[outi + 2] = (buf[3] << 4) | (buf[4] >> 1) - fallthrough - case 4: - out[outi + 1] = (buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4) - fallthrough - case 2: - out[outi] = (buf[0] << 3) | (buf[1] >> 2) - } - outi += bytes_to_write - } - - return -} +package encoding_base32 + +// @note(zh): Encoding utility for Base32 +// A secondary param can be used to supply a custom alphabet to +// @link(encode) and a matching decoding table to @link(decode). +// If none is supplied it just uses the standard Base32 alphabet. +// In case your specific version does not use padding, you may +// truncate it from the encoded output. + +// Error represents errors that can occur during base32 decoding operations. +// See RFC 4648 sections 3.2, 4 and 6. +Error :: enum { + None, + Invalid_Character, // Input contains characters outside the specified alphabet + Invalid_Length, // Input length is not valid for base32 (must be a multiple of 8 with proper padding) + Malformed_Input, // Input has improper structure (wrong padding position or incomplete groups) +} + +Validate_Proc :: #type proc(c: byte) -> bool + +@private +_validate_default :: proc(c: byte) -> bool { + return (c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7') +} + +ENC_TABLE := [32]byte { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', '2', '3', '4', '5', '6', '7', +} + +PADDING :: '=' + +DEC_TABLE := [256]u8 { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +} + +encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { + out_length := (len(data) + 4) / 5 * 8 + out := make([]byte, out_length, allocator) + _encode(out, data, ENC_TBL) + return string(out[:]) +} + +@private +_encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) { + out := out + data := data + + for len(data) > 0 { + carry: byte + switch len(data) { + case: + out[7] = ENC_TBL[data[4] & 0x1f] + carry = data[4] >> 5 + fallthrough + case 4: + out[6] = ENC_TBL[carry | (data[3] << 3) & 0x1f] + out[5] = ENC_TBL[(data[3] >> 2) & 0x1f] + carry = data[3] >> 7 + fallthrough + case 3: + out[4] = ENC_TBL[carry | (data[2] << 1) & 0x1f] + carry = (data[2] >> 4) & 0x1f + fallthrough + case 2: + out[3] = ENC_TBL[carry | (data[1] << 4) & 0x1f] + out[2] = ENC_TBL[(data[1] >> 1) & 0x1f] + carry = (data[1] >> 6) & 0x1f + fallthrough + case 1: + out[1] = ENC_TBL[carry | (data[0] << 2) & 0x1f] + out[0] = ENC_TBL[data[0] >> 3] + } + + if len(data) < 5 { + out[7] = byte(PADDING) + if len(data) < 4 { + out[6] = byte(PADDING) + out[5] = byte(PADDING) + if len(data) < 3 { + out[4] = byte(PADDING) + if len(data) < 2 { + out[3] = byte(PADDING) + out[2] = byte(PADDING) + } + } + } + break + } + data = data[5:] + out = out[8:] + } +} + +@(optimization_mode="favor_size") +decode :: proc( + data: string, + DEC_TBL := DEC_TABLE, + validate: Validate_Proc = _validate_default, + allocator := context.allocator) -> (out: []byte, err: Error) +{ + if len(data) == 0 { + return nil, .None + } + + // Check minimum length requirement first + if len(data) < 2 { + return nil, .Invalid_Length + } + + // Validate characters using provided validation function + for i := 0; i < len(data); i += 1 { + c := data[i] + if c == byte(PADDING) { + break + } + if !validate(c) { + return nil, .Invalid_Character + } + } + + // Validate padding and length + data_len := len(data) + padding_count := 0 + for i := data_len - 1; i >= 0; i -= 1 { + if data[i] != byte(PADDING) { + break + } + padding_count += 1 + } + + // Check for proper padding and length combinations + if padding_count > 0 { + // Verify no padding in the middle + for i := 0; i < data_len - padding_count; i += 1 { + if data[i] == byte(PADDING) { + return nil, .Malformed_Input + } + } + + content_len := data_len - padding_count + mod8 := content_len % 8 + required_padding: int + switch mod8 { + case 2: required_padding = 6 // 2 chars need 6 padding chars + case 4: required_padding = 4 // 4 chars need 4 padding chars + case 5: required_padding = 3 // 5 chars need 3 padding chars + case 7: required_padding = 1 // 7 chars need 1 padding char + case: required_padding = 0 + } + + if required_padding > 0 { + if padding_count != required_padding { + return nil, .Malformed_Input + } + } else if mod8 != 0 { + return nil, .Malformed_Input + } + } else { + // No padding - must be multiple of 8 + if data_len % 8 != 0 { + return nil, .Malformed_Input + } + } + + // Calculate decoded length: 5 bytes for every 8 input chars + input_chars := data_len - padding_count + out_len := input_chars * 5 / 8 + out = make([]byte, out_len, allocator) + defer if err != .None { + delete(out) + } + + // Process input in 8-byte blocks + outi := 0 + for i := 0; i < input_chars; i += 8 { + buf: [8]byte + block_size := min(8, input_chars - i) + + // Decode block + for j := 0; j < block_size; j += 1 { + buf[j] = DEC_TBL[data[i + j]] + } + + // Convert to output bytes based on block size + bytes_to_write := block_size * 5 / 8 + switch block_size { + case 8: + out[outi + 4] = (buf[6] << 5) | buf[7] + fallthrough + case 7: + out[outi + 3] = (buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3) + fallthrough + case 5: + out[outi + 2] = (buf[3] << 4) | (buf[4] >> 1) + fallthrough + case 4: + out[outi + 1] = (buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4) + fallthrough + case 2: + out[outi] = (buf[0] << 3) | (buf[1] >> 2) + } + outi += bytes_to_write + } + + return +} -- cgit v1.2.3 From d6f4412dc33bf21607fe97ebfcbd2ba2c81368f1 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 31 Dec 2024 18:18:23 +0100 Subject: encoding/base32: Fix style issues for CI --- core/encoding/base32/base32.odin | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index d4e4cbe03..b6af279ec 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -48,7 +48,7 @@ DEC_TABLE := [256]u8 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { @@ -66,7 +66,7 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al for len(data) > 0 { carry: byte switch len(data) { - case: + case: out[7] = ENC_TBL[data[4] & 0x1f] carry = data[4] >> 5 fallthrough @@ -114,8 +114,7 @@ decode :: proc( data: string, DEC_TBL := DEC_TABLE, validate: Validate_Proc = _validate_default, - allocator := context.allocator) -> (out: []byte, err: Error) -{ + allocator := context.allocator) -> (out: []byte, err: Error) { if len(data) == 0 { return nil, .None } @@ -163,7 +162,7 @@ decode :: proc( case 4: required_padding = 4 // 4 chars need 4 padding chars case 5: required_padding = 3 // 5 chars need 3 padding chars case 7: required_padding = 1 // 7 chars need 1 padding char - case: required_padding = 0 + case: required_padding = 0 } if required_padding > 0 { -- cgit v1.2.3 From fe88c22b1fa3b5090ae74dc358896ef784446469 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Tue, 31 Dec 2024 23:47:33 +0100 Subject: encoding/base32: Fix RFC 4648 references and add RFC reference URL Fix incorrect RFC 4648 section references: - Add RFC URL reference at package level - Update Error enum documentation to reference correct sections: - Invalid_Character: Section 3.3 (non-alphabet characters) - Invalid_Length: Section 6 (base32 block size requirements) - Malformed_Input: Section 3.2 (padding) - Fix test file section references to match correct sections This ensures all RFC references are accurate and adds a link to the source RFC for reference. --- core/encoding/base32/base32.odin | 7 ++++++- core/encoding/base32/base32_test.odin | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index b6af279ec..0b8ec95c4 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -1,3 +1,5 @@ +// Base32 encoding/decoding implementation as specified in RFC 4648. +// [[ More; https://www.rfc-editor.org/rfc/rfc4648.html ]] package encoding_base32 // @note(zh): Encoding utility for Base32 @@ -8,7 +10,10 @@ package encoding_base32 // truncate it from the encoded output. // Error represents errors that can occur during base32 decoding operations. -// See RFC 4648 sections 3.2, 4 and 6. +// As per RFC 4648: +// - Section 3.3: Invalid character handling +// - Section 3.2: Padding requirements +// - Section 6: Base32 encoding specifics (including block size requirements) Error :: enum { None, Invalid_Character, // Input contains characters outside the specified alphabet diff --git a/core/encoding/base32/base32_test.odin b/core/encoding/base32/base32_test.odin index b032aa122..ea41ae36f 100644 --- a/core/encoding/base32/base32_test.odin +++ b/core/encoding/base32/base32_test.odin @@ -57,7 +57,7 @@ test_base32_encode :: proc(t: ^testing.T) { @(test) test_base32_decode_invalid :: proc(t: ^testing.T) { - // Section 3.2 - Alphabet check + // Section 3.3 - Non-alphabet characters { // Characters outside alphabet input := "MZ1W6YTB" // '1' not in alphabet (A-Z, 2-7) @@ -77,7 +77,7 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { testing.expect_value(t, err, Error.Invalid_Character) } - // Section 4 - Padding requirements + // Section 3.2 - Padding requirements { // Padding must only be at end input := "MZ=Q====" @@ -115,7 +115,7 @@ test_base32_decode_invalid :: proc(t: ^testing.T) { testing.expect_value(t, err, Error.Malformed_Input) } - // Section 6 - Block size requirements + // Section 6 - Base32 block size requirements { // Single character (invalid block) input := "M" -- cgit v1.2.3 From a4a156290589751a741060c4a2e33e72f0ef21a8 Mon Sep 17 00:00:00 2001 From: Zoltán Kéri Date: Fri, 3 Jan 2025 19:16:56 +0100 Subject: encoding/base32: Add `@(rodata)` attribute to default tables Add `@(rodata)` attribute to `ENC_TABLE` and `DEC_TABLE` to mark them as read-only data. This places these tables in the read-only section of the executable, protecting them from modification during program execution. --- core/encoding/base32/base32.odin | 2 ++ 1 file changed, 2 insertions(+) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 0b8ec95c4..8629491b1 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -28,6 +28,7 @@ _validate_default :: proc(c: byte) -> bool { return (c >= 'A' && c <= 'Z') || (c >= '2' && c <= '7') } +@(rodata) ENC_TABLE := [32]byte { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', @@ -37,6 +38,7 @@ ENC_TABLE := [32]byte { PADDING :: '=' +@(rodata) DEC_TABLE := [256]u8 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -- cgit v1.2.3 From 600e0ebed0c8d1b27de266edcee5cc392cfc306a Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 12 Jan 2025 12:13:29 +0100 Subject: Fix stray space vs. tab --- core/encoding/base32/base32.odin | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 8629491b1..2267a872b 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -118,10 +118,10 @@ _encode :: proc(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.al @(optimization_mode="favor_size") decode :: proc( - data: string, - DEC_TBL := DEC_TABLE, - validate: Validate_Proc = _validate_default, - allocator := context.allocator) -> (out: []byte, err: Error) { + data: string, + DEC_TBL := DEC_TABLE, + validate: Validate_Proc = _validate_default, + allocator := context.allocator) -> (out: []byte, err: Error) { if len(data) == 0 { return nil, .None } -- cgit v1.2.3 From 51b80c5a2027270c634a6c4a41e8fed0216ae4c1 Mon Sep 17 00:00:00 2001 From: jkenda Date: Sat, 1 Feb 2025 10:32:07 +0100 Subject: encoding/json: marshal enumerated arrays to objects with key-value pairs --- core/encoding/json/marshal.odin | 16 ++++++++-- tests/core/encoding/json/test_core_json.odin | 48 +++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) (limited to 'core/encoding') diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index f0f0927a1..020facd14 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -209,13 +209,23 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: opt_write_end(w, opt, ']') or_return case runtime.Type_Info_Enumerated_Array: - opt_write_start(w, opt, '[') or_return + index_type := reflect.type_info_base(info.index) + enum_type := index_type.variant.(reflect.Type_Info_Enum) + + opt_write_start(w, opt, '{') or_return for i in 0.. u128 { } return u -} \ No newline at end of file +} diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index 27cce7faa..5fcdc7a27 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -482,4 +482,50 @@ map_with_integer_keys :: proc(t: ^testing.T) { testing.expectf(t, runtime.string_eq(item, my_map2[key]), "Expected value %s to be present in unmarshaled map", key) } } -} \ No newline at end of file +} + +@test +enumerated_array :: proc(t: ^testing.T) { + Fruit :: enum { Apple, Banana, Pear } + Fruit_Stock :: [Fruit]uint { + .Apple = 14, + .Banana = 3, + .Pear = 513, + } + + { // test unmarshaling from array + marshaled := "[14,3,513]" + + unmarshaled: [Fruit]uint + err := json.unmarshal_string(marshaled, &unmarshaled) + testing.expect_value(t, err, nil) + testing.expect_value(t, unmarshaled, Fruit_Stock) + } + + Sparse_Fruit :: enum { Apple, Banana, Cherry = 23, Pear } + Sparse_Fruit_Stock :: #partial #sparse [Sparse_Fruit]uint { + .Apple = 14, + .Banana = 3, + .Pear = 513, + } + + { // test unmarshaling from object + marshaled := `{"Apple":14,"Banana":3,"Cherry":0,"Pear":513}` + + unmarshaled: #sparse [Sparse_Fruit]uint + err := json.unmarshal_string(marshaled, &unmarshaled) + testing.expect_value(t, err, nil) + testing.expect_value(t, unmarshaled, Sparse_Fruit_Stock) + } + + { // test marshal -> unmarshal + marshaled, err_marshal := json.marshal(Sparse_Fruit_Stock) + defer delete(marshaled) + testing.expect_value(t, err_marshal, nil) + + unmarshaled: #sparse [Sparse_Fruit]uint + err_unmarshal := json.unmarshal(marshaled, &unmarshaled) + testing.expect_value(t, err_unmarshal, nil) + testing.expect_value(t, unmarshaled, Sparse_Fruit_Stock) + } +} -- cgit v1.2.3