diff options
| -rw-r--r-- | .github/workflows/ci.yml | 36 | ||||
| -rw-r--r-- | core/testing/runner.odin | 98 |
2 files changed, 75 insertions, 59 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 180941e26..a3d008300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,10 @@ jobs: gmake -C vendor/miniaudio/src ./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_amd64 ./odin check examples/all -vet -strict-style -disallow-do -target:netbsd_arm64 - ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true (cd tests/issues; ./run.sh) build_freebsd: name: FreeBSD Build, Check, and Test @@ -59,10 +59,10 @@ jobs: gmake -C vendor/cgltf/src gmake -C vendor/miniaudio/src ./odin check examples/all -vet -strict-style -disallow-do -target:freebsd_amd64 - ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal - ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/core/speed.odin -file -all-packages -o:speed -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true + ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true (cd tests/issues; ./run.sh) ci: strategy: @@ -116,15 +116,15 @@ jobs: - name: Odin check examples/all run: ./odin check examples/all -strict-style - name: Normal Core library tests - run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Optimized Core library tests - run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Vendor library tests - run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + run: ./odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Internals tests - run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Core library benchmarks - run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + run: ./odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: GitHub Issue tests run: | cd tests/issues @@ -188,28 +188,28 @@ jobs: shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat - odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Optimized core library tests shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat - odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Core library benchmarks shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat - odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + odin test tests/benchmark -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Vendor library tests shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat copy vendor\lua\5.4\windows\*.dll . - odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + odin test tests/vendor -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Odin internals tests shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat - odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_LOG_LEVEL_MEMORY=fatal + odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true - name: Odin documentation tests shell: cmd run: | diff --git a/core/testing/runner.odin b/core/testing/runner.odin index 5482d93e3..467b5a666 100644 --- a/core/testing/runner.odin +++ b/core/testing/runner.odin @@ -25,8 +25,8 @@ TEST_THREADS : int : #config(ODIN_TEST_THREADS, 0) TRACKING_MEMORY : bool : #config(ODIN_TEST_TRACK_MEMORY, true) // Always report how much memory is used, even when there are no leaks or bad frees. ALWAYS_REPORT_MEMORY : bool : #config(ODIN_TEST_ALWAYS_REPORT_MEMORY, false) -// Log level for memory leaks and bad frees: debug, info, warning, error, fatal -LOG_LEVEL_MEMORY : string : #config(ODIN_TEST_LOG_LEVEL_MEMORY, "warning") +// Treat memory leaks and bad frees as errors. +FAIL_ON_BAD_MEMORY : bool : #config(ODIN_TEST_FAIL_ON_BAD_MEMORY, false) // Specify how much memory each thread allocator starts with. PER_THREAD_MEMORY : int : #config(ODIN_TEST_THREAD_MEMORY, mem.ROLLBACK_STACK_DEFAULT_BLOCK_SIZE) // Select a specific set of tests to run by name. @@ -65,21 +65,6 @@ get_log_level :: #force_inline proc() -> runtime.Logger_Level { } } -get_memory_log_level :: #force_inline proc() -> runtime.Logger_Level { - when ODIN_DEBUG { - // Always use .Debug in `-debug` mode. - return .Debug - } else { - when LOG_LEVEL_MEMORY == "debug" { return .Debug } else - when LOG_LEVEL_MEMORY == "info" { return .Info } else - when LOG_LEVEL_MEMORY == "warning" { return .Warning } else - when LOG_LEVEL_MEMORY == "error" { return .Error } else - when LOG_LEVEL_MEMORY == "fatal" { return .Fatal } else { - #panic("Unknown `ODIN_TEST_LOG_LEVEL_MEMORY`: \"" + LOG_LEVEL_MEMORY + "\", possible levels are: \"debug\", \"info\", \"warning\", \"error\", or \"fatal\".") - } - } -} - JSON :: struct { total: int, success: int, @@ -103,10 +88,19 @@ end_t :: proc(t: ^T) { t.cleanups = {} } -Task_Data :: struct { - it: Internal_Test, - t: T, - allocator_index: int, +when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY { + Task_Data :: struct { + it: Internal_Test, + t: T, + allocator_index: int, + tracking_allocator: ^mem.Tracking_Allocator, + } +} else { + Task_Data :: struct { + it: Internal_Test, + t: T, + allocator_index: int, + } } Task_Timeout :: struct { @@ -150,6 +144,31 @@ run_test_task :: proc(task: thread.Task) { end_t(&data.t) + when TRACKING_MEMORY && FAIL_ON_BAD_MEMORY { + // NOTE(Feoramund): The simplest way to handle treating memory failures + // as errors is to allow the test task runner to access the tracking + // allocator itself. + // + // This way, it's still able to send up a log message, which will be + // used in the end summary, and it can set the test state to `Failed` + // under the usual conditions. + // + // No outside intervention needed. + memory_leaks := len(data.tracking_allocator.allocation_map) + bad_frees := len(data.tracking_allocator.bad_free_array) + + memory_is_in_bad_state := memory_leaks + bad_frees > 0 + + data.t.error_count += memory_leaks + bad_frees + + if memory_is_in_bad_state { + pkg_log.errorf("Memory failure in `%s.%s` with %i leak%s and %i bad free%s.", + data.it.pkg, data.it.name, + memory_leaks, "" if memory_leaks == 1 else "s", + bad_frees, "" if bad_frees == 1 else "s") + } + } + new_state : Test_State = .Failed if failed(&data.t) else .Successful chan.send(data.t.channel, Event_State_Change { @@ -239,10 +258,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { total_success_count := 0 total_done_count := 0 total_test_count := len(internal_tests) - when TRACKING_MEMORY { - memory_leak_count := 0 - bad_free_count := 0 - } when !FANCY_OUTPUT { // This is strictly for updating the window title when the progress @@ -439,6 +454,9 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { #no_bounds_check when TRACKING_MEMORY { task_allocator := mem.tracking_allocator(&task_memory_trackers[task_index]) + when FAIL_ON_BAD_MEMORY { + data.tracking_allocator = &task_memory_trackers[task_index] + } } else { task_allocator := mem.rollback_stack_allocator(&task_allocators[task_index]) } @@ -485,8 +503,13 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { pkg_log.info("Memory tracking is enabled. Tests will log their memory usage if there's an issue.") } pkg_log.info("< Final Mem/ Total Mem> < Peak Mem> (#Free/Alloc) :: [package.test_name]") - } else when ALWAYS_REPORT_MEMORY { - pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TRACK_MEMORY is false.") + } else { + when ALWAYS_REPORT_MEMORY { + pkg_log.warn("ODIN_TEST_ALWAYS_REPORT_MEMORY is true, but ODIN_TRACK_MEMORY is false.") + } + when FAIL_ON_BAD_MEMORY { + pkg_log.warn("ODIN_TEST_FAIL_ON_BAD_MEMORY is true, but ODIN_TRACK_MEMORY is false.") + } } start_time := time.now() @@ -519,9 +542,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { memory_is_in_bad_state := len(tracker.allocation_map) + len(tracker.bad_free_array) > 0 - memory_leak_count += len(tracker.allocation_map) - bad_free_count += len(tracker.bad_free_array) - when ALWAYS_REPORT_MEMORY { should_report := true } else { @@ -531,9 +551,11 @@ runner :: proc(internal_tests: []Internal_Test) -> bool { if should_report { write_memory_report(batch_writer, tracker, data.it.pkg, data.it.name) - memory_log_level := get_memory_log_level() if memory_is_in_bad_state else .Info - - pkg_log.log(memory_log_level, bytes.buffer_to_string(&batch_buffer)) + when FAIL_ON_BAD_MEMORY { + pkg_log.log(.Error if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer)) + } else { + pkg_log.log(.Warning if memory_is_in_bad_state else .Info, bytes.buffer_to_string(&batch_buffer)) + } bytes.buffer_reset(&batch_buffer) } @@ -917,11 +939,5 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_ fmt.assertf(err == nil, "Error writing JSON report: %v", err) } - fatal_memory_failures := false - when TRACKING_MEMORY { - if get_memory_log_level() >= .Error { - fatal_memory_failures = (memory_leak_count + bad_free_count) > 0 - } - } - return total_success_count == total_test_count && !fatal_memory_failures -}
\ No newline at end of file + return total_success_count == total_test_count +} |