Merge pull request #5530 from odin-lang/bill/utf16-strings

UTF-16 string types: `string16` & `cstring16`

1 files changed, 123 insertions, 7 deletions

diff --git a/src/check_expr.cpp b/src/check_expr.cpp
index 51fb5511b..faa338f36 100644
--- a/src/check_expr.cpp
+++ b/src/check_expr.cpp
@@ -2106,6 +2106,9 @@ gb_internal bool check_representable_as_constant(CheckerContext *c, ExactValue i
 	} else if (is_type_boolean(type)) {
 		return in_value.kind == ExactValue_Bool;
 	} else if (is_type_string(type)) {
+		if (in_value.kind == ExactValue_String16) {
+			return is_type_string16(type) || is_type_cstring16(type);
+		}
 		return in_value.kind == ExactValue_String;
 	} else if (is_type_integer(type) || is_type_rune(type)) {
 		if (in_value.kind == ExactValue_Bool) {
@@ -2320,6 +2323,9 @@ gb_internal bool check_representable_as_constant(CheckerContext *c, ExactValue i
 		if (in_value.kind == ExactValue_String) {
 			return false;
 		}
+		if (in_value.kind == ExactValue_String16) {
+			return false;
+		}
 		if (out_value) *out_value = in_value;
 	} else if (is_type_bit_set(type)) {
 		if (in_value.kind == ExactValue_Integer) {
@@ -2862,6 +2868,14 @@ gb_internal void add_comparison_procedures_for_fields(CheckerContext *c, Type *t
 			add_package_dependency(c, "runtime", "string_eq");
 			add_package_dependency(c, "runtime", "string_ne");
 			break;
+		case Basic_cstring16:
+			add_package_dependency(c, "runtime", "cstring16_eq");
+			add_package_dependency(c, "runtime", "cstring16_ne");
+			break;
+		case Basic_string16:
+			add_package_dependency(c, "runtime", "string16_eq");
+			add_package_dependency(c, "runtime", "string16_ne");
+			break;
 		}
 		break;
 	case Type_Struct:
@@ -3035,6 +3049,24 @@ gb_internal void check_comparison(CheckerContext *c, Ast *node, Operand *x, Oper
 				case Token_LtEq:  add_package_dependency(c, "runtime", "cstring_le"); break;
 				case Token_GtEq:  add_package_dependency(c, "runtime", "cstring_gt"); break;
 				}
+			} else if (is_type_cstring16(x->type) && is_type_cstring16(y->type)) {
+				switch (op) {
+				case Token_CmpEq: add_package_dependency(c, "runtime", "cstring16_eq"); break;
+				case Token_NotEq: add_package_dependency(c, "runtime", "cstring16_ne"); break;
+				case Token_Lt:    add_package_dependency(c, "runtime", "cstring16_lt"); break;
+				case Token_Gt:    add_package_dependency(c, "runtime", "cstring16_gt"); break;
+				case Token_LtEq:  add_package_dependency(c, "runtime", "cstring16_le"); break;
+				case Token_GtEq:  add_package_dependency(c, "runtime", "cstring16_gt"); break;
+				}
+			} else if (is_type_string16(x->type) || is_type_string16(y->type)) {
+				switch (op) {
+				case Token_CmpEq: add_package_dependency(c, "runtime", "string16_eq"); break;
+				case Token_NotEq: add_package_dependency(c, "runtime", "string16_ne"); break;
+				case Token_Lt:    add_package_dependency(c, "runtime", "string16_lt"); break;
+				case Token_Gt:    add_package_dependency(c, "runtime", "string16_gt"); break;
+				case Token_LtEq:  add_package_dependency(c, "runtime", "string16_le"); break;
+				case Token_GtEq:  add_package_dependency(c, "runtime", "string16_gt"); break;
+				}
 			} else if (is_type_string(x->type) || is_type_string(y->type)) {
 				switch (op) {
 				case Token_CmpEq: add_package_dependency(c, "runtime", "string_eq"); break;
@@ -3340,6 +3372,11 @@ gb_internal bool check_is_castable_to(CheckerContext *c, Operand *operand, Type
 		return true;
 	}
 
+	// []u16 <-> string16 (not cstring16)
+	if (is_type_u16_slice(src) && (is_type_string16(dst) && !is_type_cstring16(dst))) {
+		return true;
+	}
+
 	// cstring -> string
 	if (are_types_identical(src, t_cstring) && are_types_identical(dst, t_string)) {
 		if (operand->mode != Addressing_Constant) {
@@ -3347,6 +3384,14 @@ gb_internal bool check_is_castable_to(CheckerContext *c, Operand *operand, Type
 		}
 		return true;
 	}
+	// cstring16 -> string16
+	if (are_types_identical(src, t_cstring16) && are_types_identical(dst, t_string16)) {
+		if (operand->mode != Addressing_Constant) {
+			add_package_dependency(c, "runtime", "cstring16_to_string16");
+		}
+		return true;
+	}
+
 	// cstring -> ^u8
 	if (are_types_identical(src, t_cstring) && is_type_u8_ptr(dst)) {
 		return !is_constant;
@@ -3372,6 +3417,34 @@ gb_internal bool check_is_castable_to(CheckerContext *c, Operand *operand, Type
 	if (is_type_rawptr(src) && are_types_identical(dst, t_cstring)) {
 		return !is_constant;
 	}
+
+	// cstring -> ^u16
+	if (are_types_identical(src, t_cstring16) && is_type_u16_ptr(dst)) {
+		return !is_constant;
+	}
+	// cstring -> [^]u16
+	if (are_types_identical(src, t_cstring16) && is_type_u16_multi_ptr(dst)) {
+		return !is_constant;
+	}
+	// cstring16 -> rawptr
+	if (are_types_identical(src, t_cstring16) && is_type_rawptr(dst)) {
+		return !is_constant;
+	}
+
+
+	// ^u16 -> cstring16
+	if (is_type_u16_ptr(src) && are_types_identical(dst, t_cstring16)) {
+		return !is_constant;
+	}
+	// [^]u16 -> cstring
+	if (is_type_u16_multi_ptr(src) && are_types_identical(dst, t_cstring16)) {
+		return !is_constant;
+	}
+	// rawptr -> cstring16
+	if (is_type_rawptr(src) && are_types_identical(dst, t_cstring16)) {
+		return !is_constant;
+	}
+
 	// proc <-> proc
 	if (is_type_proc(src) && is_type_proc(dst)) {
 		if (is_type_polymorphic(dst)) {
@@ -4558,6 +4631,8 @@ gb_internal void convert_to_typed(CheckerContext *c, Operand *operand, Type *tar
 					// target_type = t_untyped_nil;
 				} else if (is_type_cstring(target_type)) {
 					// target_type = t_untyped_nil;
+				} else if (is_type_cstring16(target_type)) {
+					// target_type = t_untyped_nil;
 				} else if (!type_has_nil(target_type)) {
 					operand->mode = Addressing_Invalid;
 					convert_untyped_error(c, operand, target_type);
@@ -4585,6 +4660,13 @@ gb_internal void convert_to_typed(CheckerContext *c, Operand *operand, Type *tar
 						break;
 					}
 				}
+			} else if (operand->value.kind == ExactValue_String16) {
+				String16 s = operand->value.value_string16;
+				if (is_type_u16_array(t)) {
+					if (s.len == t->Array.count) {
+						break;
+					}
+				}
 			}
 			operand->mode = Addressing_Invalid;
 			convert_untyped_error(c, operand, target_type);
@@ -4914,6 +4996,12 @@ gb_internal ExactValue get_constant_field_single(CheckerContext *c, ExactValue v
 		if (success_) *success_ = true;
 		if (finish_) *finish_ = true;
 		return exact_value_u64(val);
+	} else if (value.kind == ExactValue_String16) {
+		GB_ASSERT(0 <= index && index < value.value_string.len);
+		u16 val = value.value_string16[index];
+		if (success_) *success_ = true;
+		if (finish_) *finish_ = true;
+		return exact_value_u64(val);
 	}
 	if (value.kind != ExactValue_Compound) {
 		if (success_) *success_ = true;
@@ -8226,6 +8314,7 @@ gb_internal bool check_set_index_data(Operand *o, Type *t, bool indirection, i64
 	case Type_Basic:
 		if (t->Basic.kind == Basic_string) {
 			if (o->mode == Addressing_Constant) {
+				GB_ASSERT(o->value.kind == ExactValue_String);
 				*max_count = o->value.value_string.len;
 			}
 			if (o->mode != Addressing_Constant) {
@@ -8233,6 +8322,16 @@ gb_internal bool check_set_index_data(Operand *o, Type *t, bool indirection, i64
 			}
 			o->type = t_u8;
 			return true;
+		} else if (t->Basic.kind == Basic_string16) {
+			if (o->mode == Addressing_Constant) {
+				GB_ASSERT(o->value.kind == ExactValue_String16);
+				*max_count = o->value.value_string16.len;
+			}
+			if (o->mode != Addressing_Constant) {
+				o->mode = Addressing_Value;
+			}
+			o->type = t_u16;
+			return true;
 		} else if (t->Basic.kind == Basic_UntypedString) {
 			if (o->mode == Addressing_Constant) {
 				*max_count = o->value.value_string.len;
@@ -10879,9 +10978,17 @@ gb_internal ExprKind check_slice_expr(CheckerContext *c, Operand *o, Ast *node,
 		if (t->Basic.kind == Basic_string || t->Basic.kind == Basic_UntypedString) {
 			valid = true;
 			if (o->mode == Addressing_Constant) {
+				GB_ASSERT(o->value.kind == ExactValue_String);
 				max_count = o->value.value_string.len;
 			}
 			o->type = type_deref(o->type);
+		} else if (t->Basic.kind == Basic_string16) {
+			valid = true;
+			if (o->mode == Addressing_Constant) {
+				GB_ASSERT(o->value.kind == ExactValue_String16);
+				max_count = o->value.value_string16.len;
+			}
+			o->type = type_deref(o->type);
 		}
 		break;
 
@@ -11036,15 +11143,21 @@ gb_internal ExprKind check_slice_expr(CheckerContext *c, Operand *o, Ast *node,
 			o->expr = node;
 			return kind;
 		}
-
-		String s = {};
-		if (o->value.kind == ExactValue_String) {
-			s = o->value.value_string;
-		}
-
 		o->mode = Addressing_Constant;
 		o->type = t;
-		o->value = exact_value_string(substring(s, cast(isize)indices[0], cast(isize)indices[1]));
+
+		if (o->value.kind == ExactValue_String16) {
+			String16 s = o->value.value_string16;
+
+			o->value = exact_value_string16(substring(s, cast(isize)indices[0], cast(isize)indices[1]));
+		} else {
+			String s = {};
+			if (o->value.kind == ExactValue_String) {
+				s = o->value.value_string;
+			}
+
+			o->value = exact_value_string(substring(s, cast(isize)indices[0], cast(isize)indices[1]));
+		}
 	}
 	return kind;
 }
@@ -11133,6 +11246,7 @@ gb_internal ExprKind check_expr_base_internal(CheckerContext *c, Operand *o, Ast
 		Type *t = t_invalid;
 		switch (node->tav.value.kind) {
 		case ExactValue_String:     t = t_untyped_string;     break;
+		case ExactValue_String16:   t = t_string16;           break; // TODO(bill): determine this correctly
 		case ExactValue_Float:      t = t_untyped_float;      break;
 		case ExactValue_Complex:    t = t_untyped_complex;    break;
 		case ExactValue_Quaternion: t = t_untyped_quaternion; break;
@@ -11569,6 +11683,8 @@ gb_internal bool is_exact_value_zero(ExactValue const &v) {
 		return !v.value_bool;
 	case ExactValue_String:
 		return v.value_string.len == 0;
+	case ExactValue_String16:
+		return v.value_string16.len == 0;
 	case ExactValue_Integer:
 		return big_int_is_zero(&v.value_integer);
 	case ExactValue_Float: