Restructure `core:terminal` for better Windows support

6 files changed, 150 insertions, 117 deletions

diff --git a/core/terminal/internal.odin b/core/terminal/internal.odin
new file mode 100644
index 000000000..485f6868d
--- /dev/null
+++ b/core/terminal/internal.odin
@@ -0,0 +1,87 @@
+#+private
+package terminal
+
+import "core:os"
+import "core:strings"
+
+// Reference documentation:
+//
+// - [[ https://no-color.org/ ]]
+// - [[ https://github.com/termstandard/colors ]]
+// - [[ https://invisible-island.net/ncurses/terminfo.src.html ]]
+
+get_no_color :: proc() -> bool {
+	if no_color, ok := os.lookup_env("NO_COLOR"); ok {
+		defer delete(no_color)
+		return no_color != ""
+	}
+	return false
+}
+
+get_environment_color :: proc() -> Color_Depth {
+	// `COLORTERM` is non-standard but widespread and unambiguous.
+	if colorterm, ok := os.lookup_env("COLORTERM"); ok {
+		defer delete(colorterm)
+		// These are the only values that are typically advertised that have
+		// anything to do with color depth.
+		if colorterm == "truecolor" || colorterm == "24bit" {
+			return .True_Color
+		}
+	}
+
+	if term, ok := os.lookup_env("TERM"); ok {
+		defer delete(term)
+		if strings.contains(term, "-truecolor") {
+			return .True_Color
+		}
+		if strings.contains(term, "-256color") {
+			return .Eight_Bit
+		}
+		if strings.contains(term, "-16color") {
+			return .Four_Bit
+		}
+
+		// The `terminfo` database, which is stored in binary on *nix
+		// platforms, has an undocumented format that is not guaranteed to be
+		// portable, so beyond this point, we can only make safe assumptions.
+		//
+		// This section should only be necessary for terminals that do not
+		// define any of the previous environment values.
+		//
+		// Only a small sampling of some common values are checked here.
+		switch term {
+		case "ansi":       fallthrough
+		case "konsole":    fallthrough
+		case "putty":      fallthrough
+		case "rxvt":       fallthrough
+		case "rxvt-color": fallthrough
+		case "screen":     fallthrough
+		case "st":         fallthrough
+		case "tmux":       fallthrough
+		case "vte":        fallthrough
+		case "xterm":      fallthrough
+		case "xterm-color":
+			return .Three_Bit
+		}
+	}
+
+	return .None
+}
+
+@(init)
+init_terminal :: proc() {
+	_init_terminal()
+
+	// We respect `NO_COLOR` specifically as a color-disabler but not as a
+	// blanket ban on any terminal manipulation codes, hence why this comes
+	// after `_init_terminal` which will allow Windows to enable Virtual
+	// Terminal Processing for non-color control sequences.
+	if !get_no_color() {
+		color_enabled = color_depth > .None
+	}
+}
+
+@(fini)
+fini_terminal :: proc() {
+	_fini_terminal()
+}
diff --git a/core/terminal/terminal.odin b/core/terminal/terminal.odin
index fae6f880a..1e5566295 100644
--- a/core/terminal/terminal.odin
+++ b/core/terminal/terminal.odin
@@ -1,7 +1,6 @@
 package terminal
 
 import "core:os"
-import "core:strings"
 
 /*
 This describes the range of colors that a terminal is capable of supporting.
@@ -26,79 +25,12 @@ is_terminal :: proc(handle: os.Handle) -> bool {
 }
 
 /*
-Get the color depth support for the terminal.
-*/
-@(require_results)
-get_color_depth :: proc() -> Color_Depth {
-	// Reference documentation:
-	//
-	// - [[ https://no-color.org/ ]]
-	// - [[ https://github.com/termstandard/colors ]]
-	// - [[ https://invisible-island.net/ncurses/terminfo.src.html ]]
-
-	// Respect `NO_COLOR` above all.
-	if no_color, ok := os.lookup_env("NO_COLOR"); ok {
-		defer delete(no_color)
-		if no_color != "" {
-			return .None
-		}
-	}
-
-	// `COLORTERM` is non-standard but widespread and unambiguous.
-	if colorterm, ok := os.lookup_env("COLORTERM"); ok {
-		defer delete(colorterm)
-		// These are the only values that are typically advertised that have
-		// anything to do with color depth.
-		if colorterm == "truecolor" || colorterm == "24bit" {
-			return .True_Color
-		}
-	}
-
-	if term, ok := os.lookup_env("TERM"); ok {
-		defer delete(term)
-		if strings.contains(term, "-truecolor") {
-			return .True_Color
-		}
-		if strings.contains(term, "-256color") {
-			return .Eight_Bit
-		}
-		if strings.contains(term, "-16color") {
-			return .Four_Bit
-		}
-
-		// The `terminfo` database, which is stored in binary on *nix
-		// platforms, has an undocumented format that is not guaranteed to be
-		// portable, so beyond this point, we can only make safe assumptions.
-		//
-		// This section should only be necessary for terminals that do not
-		// define any of the previous environment values.
-		//
-		// Only a small sampling of some common values are checked here.
-		switch term {
-		case "ansi":       fallthrough
-		case "konsole":    fallthrough
-		case "putty":      fallthrough
-		case "rxvt":       fallthrough
-		case "rxvt-color": fallthrough
-		case "screen":     fallthrough
-		case "st":         fallthrough
-		case "tmux":       fallthrough
-		case "vte":        fallthrough
-		case "xterm":      fallthrough
-		case "xterm-color":
-			return .Three_Bit
-		}
-	}
-
-	return .None
-}
-
-/*
 This is true if the terminal is accepting any form of colored text output.
 */
 color_enabled: bool
 
-@(init, private)
-init_terminal_status :: proc() {
-	color_enabled = get_color_depth() > .None
-}
+/*
+This value reports the color depth support as reported by the terminal at the
+start of the program.
+*/
+color_depth: Color_Depth
diff --git a/core/terminal/terminal_posix.odin b/core/terminal/terminal_posix.odin
index adfb6a0da..f578e12c6 100644
--- a/core/terminal/terminal_posix.odin
+++ b/core/terminal/terminal_posix.odin
@@ -1,3 +1,4 @@
+#+private
 #+build linux, darwin, netbsd, openbsd, freebsd, haiku
 package terminal
 
@@ -7,3 +8,9 @@ import "core:sys/posix"
 _is_terminal :: proc(handle: os.Handle) -> bool {
 	return bool(posix.isatty(posix.FD(handle)))
 }
+
+_init_terminal :: proc() {
+	color_depth = get_environment_color()
+}
+
+_fini_terminal :: proc() { }
diff --git a/core/terminal/terminal_windows.odin b/core/terminal/terminal_windows.odin
index 55a3903fe..cc28add98 100644
--- a/core/terminal/terminal_windows.odin
+++ b/core/terminal/terminal_windows.odin
@@ -1,3 +1,4 @@
+#+private
 package terminal
 
 import "core:os"
@@ -7,3 +8,53 @@ _is_terminal :: proc(handle: os.Handle) -> bool {
 	is_tty := windows.GetFileType(windows.HANDLE(handle)) == windows.FILE_TYPE_CHAR
 	return is_tty
 }
+
+old_modes: [2]struct{
+	handle: windows.DWORD,
+	mode: windows.DWORD,
+} = {
+	{windows.STD_OUTPUT_HANDLE, 0},
+	{windows.STD_ERROR_HANDLE, 0},
+}
+
+@(init)
+_init_terminal :: proc() {
+	vtp_enabled: bool
+
+	for &v in old_modes {
+		handle := windows.GetStdHandle(v.handle)
+		if handle == windows.INVALID_HANDLE || handle == nil {
+			return
+		}
+		if windows.GetConsoleMode(handle, &v.mode) {
+			windows.SetConsoleMode(handle, v.mode | windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+
+			new_mode: windows.DWORD
+			windows.GetConsoleMode(handle, &new_mode)
+
+			if new_mode & windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0 {
+				vtp_enabled = true
+			}
+		}
+	}
+
+	if vtp_enabled {
+		// This color depth is available on Windows 10 since build 10586.
+		color_depth = .Four_Bit
+	} else {
+		// The user may be on a non-default terminal emulator.
+		color_depth = get_environment_color()
+	}
+}
+
+@(fini)
+_fini_terminal :: proc() {
+	for v in old_modes {
+		handle := windows.GetStdHandle(v.handle)
+		if handle == windows.INVALID_HANDLE || handle == nil {
+			return
+		}
+		
+		windows.SetConsoleMode(handle, v.mode)
+	}
+}
diff --git a/core/testing/runner.odin b/core/testing/runner.odin
index a184eb28c..56d561d3d 100644
--- a/core/testing/runner.odin
+++ b/core/testing/runner.odin
@@ -214,10 +214,6 @@ runner :: proc(internal_tests: []Internal_Test) -> bool {
 		}
 	}
 
-	when ODIN_OS == .Windows {
-		console_ansi_init()
-	}
-
 	stdout := io.to_writer(os.stream_from_handle(os.stdout))
 	stderr := io.to_writer(os.stream_from_handle(os.stderr))
 
@@ -981,9 +977,5 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_
 		fmt.assertf(err == nil, "Error writing JSON report: %v", err)
 	}
 
-	when ODIN_OS == .Windows {
-		console_ansi_fini()
-	}
-
 	return total_success_count == total_test_count
 }
diff --git a/core/testing/runner_windows.odin b/core/testing/runner_windows.odin
deleted file mode 100644
index b35914c72..000000000
--- a/core/testing/runner_windows.odin
+++ /dev/null
@@ -1,36 +0,0 @@
-#+private
-package testing
-
-import win32 "core:sys/windows"
-
-old_stdout_mode: u32
-old_stderr_mode: u32
-
-console_ansi_init :: proc() {
-	stdout := win32.GetStdHandle(win32.STD_OUTPUT_HANDLE)
-	if stdout != win32.INVALID_HANDLE && stdout != nil {
-		if win32.GetConsoleMode(stdout, &old_stdout_mode) {
-			win32.SetConsoleMode(stdout, old_stdout_mode | win32.ENABLE_VIRTUAL_TERMINAL_PROCESSING)
-		}
-	}
-
-	stderr := win32.GetStdHandle(win32.STD_ERROR_HANDLE)
-	if stderr != win32.INVALID_HANDLE && stderr != nil {
-		if win32.GetConsoleMode(stderr, &old_stderr_mode) {
-			win32.SetConsoleMode(stderr, old_stderr_mode | win32.ENABLE_VIRTUAL_TERMINAL_PROCESSING)
-		}
-	}
-}
-
-// Restore the cursor on exit
-console_ansi_fini :: proc() {
-	stdout := win32.GetStdHandle(win32.STD_OUTPUT_HANDLE)
-	if stdout != win32.INVALID_HANDLE && stdout != nil {
-		win32.SetConsoleMode(stdout, old_stdout_mode)
-	}
-
-	stderr := win32.GetStdHandle(win32.STD_ERROR_HANDLE)
-	if stderr != win32.INVALID_HANDLE && stderr != nil {
-		win32.SetConsoleMode(stderr, old_stderr_mode)
-	}
-}
\ No newline at end of file