Delay determination of procedure abi types until as late as possible to prevent type undetermination in self-referential data types #454

5 files changed, 29 insertions, 12 deletions

diff --git a/src/check_expr.cpp b/src/check_expr.cpp
index b5d24e008..51497af7e 100644
--- a/src/check_expr.cpp
+++ b/src/check_expr.cpp
@@ -89,7 +89,7 @@ Type *           check_init_variable    (CheckerContext *c, Entity *e, Operand *
 Type *type_to_abi_compat_param_type(gbAllocator a, Type *original_type, ProcCallingConvention cc);
 Type *type_to_abi_compat_result_type(gbAllocator a, Type *original_type, ProcCallingConvention cc);
 bool abi_compat_return_by_pointer(gbAllocator a, ProcCallingConvention cc, Type *abi_return_type);
-void set_procedure_abi_types(CheckerContext *c, Type *type);
+void set_procedure_abi_types(gbAllocator a, Type *type);
 void check_assignment_error_suggestion(CheckerContext *c, Operand *o, Type *type);
 
 Entity *entity_from_expr(Ast *expr) {
@@ -963,7 +963,7 @@ bool is_polymorphic_type_assignable(CheckerContext *c, Type *poly, Type *source,
 			}
 
 			if (modify_type) {
-				set_procedure_abi_types(c, source);
+				set_procedure_abi_types(c->allocator, source);
 			}
 
 			return true;
diff --git a/src/check_type.cpp b/src/check_type.cpp
index de33cc1a5..45c59f2bc 100644
--- a/src/check_type.cpp
+++ b/src/check_type.cpp
@@ -1273,7 +1273,7 @@ Type *determine_type_from_polymorphic(CheckerContext *ctx, Type *poly_type, Oper
 
 	if (is_polymorphic_type_assignable(ctx, poly_type, operand.type, false, modify_type)) {
 		if (show_error) {
-			set_procedure_abi_types(ctx, poly_type);
+			set_procedure_abi_types(ctx->allocator, poly_type);
 		}
 		return poly_type;
 	}
@@ -2362,18 +2362,22 @@ bool abi_compat_return_by_pointer(gbAllocator a, ProcCallingConvention cc, Type
 	return false;
 }
 
-void set_procedure_abi_types(CheckerContext *c, Type *type) {
+void set_procedure_abi_types(gbAllocator allocator, Type *type) {
 	type = base_type(type);
 	if (type->kind != Type_Proc) {
 		return;
 	}
 
-	type->Proc.abi_compat_params = array_make<Type *>(c->allocator, cast(isize)type->Proc.param_count);
+	if (type->Proc.abi_types_set) {
+		return;
+	}
+
+	type->Proc.abi_compat_params = array_make<Type *>(allocator, cast(isize)type->Proc.param_count);
 	for (i32 i = 0; i < type->Proc.param_count; i++) {
 		Entity *e = type->Proc.params->Tuple.variables[i];
 		if (e->kind == Entity_Variable) {
 			Type *original_type = e->type;
-			Type *new_type = type_to_abi_compat_param_type(c->allocator, original_type, type->Proc.calling_convention);
+			Type *new_type = type_to_abi_compat_param_type(allocator, original_type, type->Proc.calling_convention);
 			type->Proc.abi_compat_params[i] = new_type;
 			switch (type->Proc.calling_convention) {
 			case ProcCC_Odin:
@@ -2387,8 +2391,10 @@ void set_procedure_abi_types(CheckerContext *c, Type *type) {
 	}
 
 	// NOTE(bill): The types are the same
-	type->Proc.abi_compat_result_type = type_to_abi_compat_result_type(c->allocator, type->Proc.results, type->Proc.calling_convention);
-	type->Proc.return_by_pointer = abi_compat_return_by_pointer(c->allocator, type->Proc.calling_convention, type->Proc.abi_compat_result_type);
+	type->Proc.abi_compat_result_type = type_to_abi_compat_result_type(allocator, type->Proc.results, type->Proc.calling_convention);
+	type->Proc.return_by_pointer = abi_compat_return_by_pointer(allocator, type->Proc.calling_convention, type->Proc.abi_compat_result_type);
+
+	type->Proc.abi_types_set = true;
 }
 
 // NOTE(bill): 'operands' is for generating non generic procedure type
@@ -2486,8 +2492,6 @@ bool check_procedure_type(CheckerContext *ctx, Type *type, Ast *proc_type_node,
 	}
 	type->Proc.is_polymorphic = is_polymorphic;
 
-	set_procedure_abi_types(c, type);
-
 	return success;
 }
 
diff --git a/src/ir.cpp b/src/ir.cpp
index ae60be7e3..d4aecebf0 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -3038,6 +3038,7 @@ irValue *ir_emit_call(irProcedure *p, irValue *value, Array<irValue *> const &ar
 		context_ptr = ir_find_or_generate_context_ptr(p);
 	}
 
+	set_procedure_abi_types(heap_allocator(), pt);
 
 	bool is_c_vararg = pt->Proc.c_vararg;
 	isize param_count = pt->Proc.param_count;
@@ -10019,6 +10020,8 @@ void ir_insert_code_before_proc(irProcedure* proc, irProcedure *parent) {
 void ir_build_proc(irValue *value, irProcedure *parent) {
 	irProcedure *proc = &value->Proc;
 
+	set_procedure_abi_types(heap_allocator(), proc->type);
+
 	proc->parent = parent;
 
 	if (proc->body != nullptr) {
diff --git a/src/ir_print.cpp b/src/ir_print.cpp
index 1d0c8af35..d47dfc898 100644
--- a/src/ir_print.cpp
+++ b/src/ir_print.cpp
@@ -323,12 +323,15 @@ void ir_print_proc_results(irFileBuffer *f, irModule *m, Type *t) {
 
 
 void ir_print_proc_type_without_pointer(irFileBuffer *f, irModule *m, Type *t) {
+	set_procedure_abi_types(heap_allocator(), t);
+
 	i64 word_bits = 8*build_context.word_size;
 	t = base_type(t);
 	GB_ASSERT(is_type_proc(t));
 
 	isize param_count = t->Proc.param_count;
 	isize result_count = t->Proc.result_count;
+
 	ir_print_proc_results(f, m, t);
 	ir_write_string(f, str_lit(" ("));
 	if (t->Proc.return_by_pointer) {
@@ -2125,6 +2128,8 @@ void ir_print_instr(irFileBuffer *f, irModule *m, irValue *value) {
 
 
 void ir_print_proc(irFileBuffer *f, irModule *m, irProcedure *proc) {
+	set_procedure_abi_types(heap_allocator(), proc->type);
+
 	if (proc->body == nullptr) {
 		ir_write_str_lit(f, "declare ");
 		// if (proc->tags & ProcTag_dll_import) {
diff --git a/src/types.cpp b/src/types.cpp
index 8ad352f62..bef69ee30 100644
--- a/src/types.cpp
+++ b/src/types.cpp
@@ -205,6 +205,7 @@ struct TypeUnion {
 		Type *   abi_compat_result_type;                  \
 		i32      variadic_index;                          \
 		bool     variadic;                                \
+		bool     abi_types_set;                           \
 		bool     require_results;                         \
 		bool     c_vararg;                                \
 		bool     is_polymorphic;                          \
@@ -2360,7 +2361,9 @@ i64 type_size_of(Type *t) {
 		return 0;
 	}
 	// NOTE(bill): Always calculate the size when it is a Type_Basic
-	if (t->kind != Type_Basic && t->cached_size >= 0) {
+	if (t->kind == Type_Named && t->cached_size >= 0) {
+
+	} else if (t->kind != Type_Basic && t->cached_size >= 0) {
 		return t->cached_size;
 	}
 	TypePath path = {0};
@@ -2375,7 +2378,9 @@ i64 type_align_of(Type *t) {
 		return 1;
 	}
 	// NOTE(bill): Always calculate the size when it is a Type_Basic
-	if (t->kind != Type_Basic && t->cached_align > 0) {
+	if (t->kind == Type_Named && t->cached_align >= 0) {
+
+	} if (t->kind != Type_Basic && t->cached_align > 0) {
 		return t->cached_align;
 	}