__atomic_* "intrinsics" using LLVM instructions

7 files changed, 765 insertions, 20 deletions

diff --git a/src/check_expr.cpp b/src/check_expr.cpp
index 2e1663f98..60e5d2649 100644
--- a/src/check_expr.cpp
+++ b/src/check_expr.cpp
@@ -2872,6 +2872,18 @@ Entity *check_selector(CheckerContext *c, Operand *operand, Ast *node, Type *typ
 	return entity;
 }
 
+bool is_type_normal_pointer(Type *ptr, Type **elem) {
+	ptr = base_type(ptr);
+	if (is_type_pointer(ptr)) {
+		if (is_type_rawptr(ptr)) {
+			return false;
+		}
+		if (elem) *elem = ptr->Pointer.elem;
+		return true;
+	}
+	return false;
+}
+
 
 bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 id) {
 	ast_node(ce, CallExpr, call);
@@ -2914,6 +2926,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32
 		break;
 	}
 
+	String builtin_name = builtin_procs[id].name;
+
 
 	if (ce->args.count > 0) {
 		if (ce->args[0]->kind == Ast_FieldValue) {
@@ -2924,7 +2938,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32
 
 	switch (id) {
 	default:
-		GB_PANIC("Implement built-in procedure: %.*s", LIT(builtin_procs[id].name));
+		GB_PANIC("Implement built-in procedure: %.*s", LIT(builtin_name));
 		break;
 
 	case BuiltinProc_DIRECTIVE: {
@@ -3023,9 +3037,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32
 		}
 
 		if (mode == Addressing_Invalid) {
-			String name = builtin_procs[id].name;
 			gbString t = type_to_string(operand->type);
-			error(call, "'%.*s' is not supported for '%s'", LIT(name), t);
+			error(call, "'%.*s' is not supported for '%s'", LIT(builtin_name), t);
 			return false;
 		}
 
@@ -3805,6 +3818,133 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32
 		break;
 	}
 
+	case BuiltinProc_atomic_fence:
+	case BuiltinProc_atomic_fence_acq:
+	case BuiltinProc_atomic_fence_rel:
+	case BuiltinProc_atomic_fence_acqrel:
+		operand->mode = Addressing_NoValue;
+		break;
+
+	case BuiltinProc_atomic_store:
+	case BuiltinProc_atomic_store_rel:
+	case BuiltinProc_atomic_store_relaxed:
+	case BuiltinProc_atomic_store_unordered:
+		{
+			Type *elem = nullptr;
+			if (!is_type_normal_pointer(operand->type, &elem)) {
+				error(operand->expr, "Expected a pointer for '%.*s'", LIT(builtin_name));
+				return false;
+			}
+			Operand x = {};
+			check_expr_with_type_hint(c, &x, ce->args[1], elem);
+			check_assignment(c, &x, elem, builtin_name);
+
+			operand->type = nullptr;
+			operand->mode = Addressing_NoValue;
+			break;
+		}
+	case BuiltinProc_atomic_load:
+	case BuiltinProc_atomic_load_acq:
+	case BuiltinProc_atomic_load_relaxed:
+	case BuiltinProc_atomic_load_unordered:
+		{
+			Type *elem = nullptr;
+			if (!is_type_normal_pointer(operand->type, &elem)) {
+				error(operand->expr, "Expected a pointer for '%.*s'", LIT(builtin_name));
+				return false;
+			}
+			operand->type = elem;
+			operand->mode = Addressing_Value;
+			break;
+		}
+
+	case BuiltinProc_atomic_add:
+	case BuiltinProc_atomic_add_acq:
+	case BuiltinProc_atomic_add_rel:
+	case BuiltinProc_atomic_add_acqrel:
+	case BuiltinProc_atomic_add_relaxed:
+	case BuiltinProc_atomic_sub:
+	case BuiltinProc_atomic_sub_acq:
+	case BuiltinProc_atomic_sub_rel:
+	case BuiltinProc_atomic_sub_acqrel:
+	case BuiltinProc_atomic_sub_relaxed:
+	case BuiltinProc_atomic_and:
+	case BuiltinProc_atomic_and_acq:
+	case BuiltinProc_atomic_and_rel:
+	case BuiltinProc_atomic_and_acqrel:
+	case BuiltinProc_atomic_and_relaxed:
+	case BuiltinProc_atomic_nand:
+	case BuiltinProc_atomic_nand_acq:
+	case BuiltinProc_atomic_nand_rel:
+	case BuiltinProc_atomic_nand_acqrel:
+	case BuiltinProc_atomic_nand_relaxed:
+	case BuiltinProc_atomic_or:
+	case BuiltinProc_atomic_or_acq:
+	case BuiltinProc_atomic_or_rel:
+	case BuiltinProc_atomic_or_acqrel:
+	case BuiltinProc_atomic_or_relaxed:
+	case BuiltinProc_atomic_xor:
+	case BuiltinProc_atomic_xor_acq:
+	case BuiltinProc_atomic_xor_rel:
+	case BuiltinProc_atomic_xor_acqrel:
+	case BuiltinProc_atomic_xor_relaxed:
+	case BuiltinProc_atomic_xchg:
+	case BuiltinProc_atomic_xchg_acq:
+	case BuiltinProc_atomic_xchg_rel:
+	case BuiltinProc_atomic_xchg_acqrel:
+	case BuiltinProc_atomic_xchg_relaxed:
+		{
+			Type *elem = nullptr;
+			if (!is_type_normal_pointer(operand->type, &elem)) {
+				error(operand->expr, "Expected a pointer for '%.*s'", LIT(builtin_name));
+				return false;
+			}
+			Operand x = {};
+			check_expr_with_type_hint(c, &x, ce->args[1], elem);
+			check_assignment(c, &x, elem, builtin_name);
+
+			operand->type = elem;
+			operand->mode = Addressing_Value;
+			break;
+		}
+
+	case BuiltinProc_atomic_cxchg:
+	case BuiltinProc_atomic_cxchg_acq:
+	case BuiltinProc_atomic_cxchg_rel:
+	case BuiltinProc_atomic_cxchg_acqrel:
+	case BuiltinProc_atomic_cxchg_relaxed:
+	case BuiltinProc_atomic_cxchg_failrelaxed:
+	case BuiltinProc_atomic_cxchg_failacq:
+	case BuiltinProc_atomic_cxchg_acq_failrelaxed:
+	case BuiltinProc_atomic_cxchg_acqrel_failrelaxed:
+
+	case BuiltinProc_atomic_cxchgweak:
+	case BuiltinProc_atomic_cxchgweak_acq:
+	case BuiltinProc_atomic_cxchgweak_rel:
+	case BuiltinProc_atomic_cxchgweak_acqrel:
+	case BuiltinProc_atomic_cxchgweak_relaxed:
+	case BuiltinProc_atomic_cxchgweak_failrelaxed:
+	case BuiltinProc_atomic_cxchgweak_failacq:
+	case BuiltinProc_atomic_cxchgweak_acq_failrelaxed:
+	case BuiltinProc_atomic_cxchgweak_acqrel_failrelaxed:
+		{
+			Type *elem = nullptr;
+			if (!is_type_normal_pointer(operand->type, &elem)) {
+				error(operand->expr, "Expected a pointer for '%.*s'", LIT(builtin_name));
+				return false;
+			}
+			Operand x = {};
+			Operand y = {};
+			check_expr_with_type_hint(c, &x, ce->args[1], elem);
+			check_expr_with_type_hint(c, &y, ce->args[2], elem);
+			check_assignment(c, &x, elem, builtin_name);
+			check_assignment(c, &y, elem, builtin_name);
+
+			operand->mode = Addressing_Value;
+			operand->type = make_optional_ok_type(elem);
+			break;
+		}
+		break;
 	}
 
 	return true;
diff --git a/src/check_type.cpp b/src/check_type.cpp
index 9704db081..f1c9875a1 100644
--- a/src/check_type.cpp
+++ b/src/check_type.cpp
@@ -2068,6 +2068,7 @@ i64 check_array_count(CheckerContext *ctx, Operand *o, Ast *e) {
 }
 
 Type *make_optional_ok_type(Type *value) {
+	// LEAK TODO(bill): probably don't reallocate everything here and reuse the same one for the same type if possible
 	gbAllocator a = heap_allocator();
 	bool typed = true;
 	Type *t = alloc_type_tuple();
diff --git a/src/checker.cpp b/src/checker.cpp
index 61910f959..14daeb7ed 100644
--- a/src/checker.cpp
+++ b/src/checker.cpp
@@ -494,13 +494,14 @@ void add_declaration_dependency(CheckerContext *c, Entity *e) {
 
 Entity *add_global_entity(Entity *entity) {
 	String name = entity->token.string;
+	defer (entity->state = EntityState_Resolved);
+
 	if (gb_memchr(name.text, ' ', name.len)) {
-		return entity; // NOTE(bill): 'untyped thing'
+		return entity; // NOTE(bill): Usually an 'untyped thing'
 	}
 	if (scope_insert(builtin_scope, entity)) {
 		compiler_error("double declaration");
 	}
-	entity->state = EntityState_Resolved;
 	return entity;
 }
 
diff --git a/src/checker.hpp b/src/checker.hpp
index 5d05d5809..855a48da3 100644
--- a/src/checker.hpp
+++ b/src/checker.hpp
@@ -81,6 +81,79 @@ enum BuiltinProcId {
 
 	BuiltinProc_DIRECTIVE, // NOTE(bill): This is used for specialized hash-prefixed procedures
 
+	// "Intrinsics"
+	BuiltinProc_atomic_fence,
+	BuiltinProc_atomic_fence_acq,
+	BuiltinProc_atomic_fence_rel,
+	BuiltinProc_atomic_fence_acqrel,
+
+	BuiltinProc_atomic_store,
+	BuiltinProc_atomic_store_rel,
+	BuiltinProc_atomic_store_relaxed,
+	BuiltinProc_atomic_store_unordered,
+
+	BuiltinProc_atomic_load,
+	BuiltinProc_atomic_load_acq,
+	BuiltinProc_atomic_load_relaxed,
+	BuiltinProc_atomic_load_unordered,
+
+	BuiltinProc_atomic_add,
+	BuiltinProc_atomic_add_acq,
+	BuiltinProc_atomic_add_rel,
+	BuiltinProc_atomic_add_acqrel,
+	BuiltinProc_atomic_add_relaxed,
+	BuiltinProc_atomic_sub,
+	BuiltinProc_atomic_sub_acq,
+	BuiltinProc_atomic_sub_rel,
+	BuiltinProc_atomic_sub_acqrel,
+	BuiltinProc_atomic_sub_relaxed,
+	BuiltinProc_atomic_and,
+	BuiltinProc_atomic_and_acq,
+	BuiltinProc_atomic_and_rel,
+	BuiltinProc_atomic_and_acqrel,
+	BuiltinProc_atomic_and_relaxed,
+	BuiltinProc_atomic_nand,
+	BuiltinProc_atomic_nand_acq,
+	BuiltinProc_atomic_nand_rel,
+	BuiltinProc_atomic_nand_acqrel,
+	BuiltinProc_atomic_nand_relaxed,
+	BuiltinProc_atomic_or,
+	BuiltinProc_atomic_or_acq,
+	BuiltinProc_atomic_or_rel,
+	BuiltinProc_atomic_or_acqrel,
+	BuiltinProc_atomic_or_relaxed,
+	BuiltinProc_atomic_xor,
+	BuiltinProc_atomic_xor_acq,
+	BuiltinProc_atomic_xor_rel,
+	BuiltinProc_atomic_xor_acqrel,
+	BuiltinProc_atomic_xor_relaxed,
+
+	BuiltinProc_atomic_xchg,
+	BuiltinProc_atomic_xchg_acq,
+	BuiltinProc_atomic_xchg_rel,
+	BuiltinProc_atomic_xchg_acqrel,
+	BuiltinProc_atomic_xchg_relaxed,
+
+	BuiltinProc_atomic_cxchg,
+	BuiltinProc_atomic_cxchg_acq,
+	BuiltinProc_atomic_cxchg_rel,
+	BuiltinProc_atomic_cxchg_acqrel,
+	BuiltinProc_atomic_cxchg_relaxed,
+	BuiltinProc_atomic_cxchg_failrelaxed,
+	BuiltinProc_atomic_cxchg_failacq,
+	BuiltinProc_atomic_cxchg_acq_failrelaxed,
+	BuiltinProc_atomic_cxchg_acqrel_failrelaxed,
+
+	BuiltinProc_atomic_cxchgweak,
+	BuiltinProc_atomic_cxchgweak_acq,
+	BuiltinProc_atomic_cxchgweak_rel,
+	BuiltinProc_atomic_cxchgweak_acqrel,
+	BuiltinProc_atomic_cxchgweak_relaxed,
+	BuiltinProc_atomic_cxchgweak_failrelaxed,
+	BuiltinProc_atomic_cxchgweak_failacq,
+	BuiltinProc_atomic_cxchgweak_acq_failrelaxed,
+	BuiltinProc_atomic_cxchgweak_acqrel_failrelaxed,
+
 	BuiltinProc_COUNT,
 };
 gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = {
@@ -111,6 +184,80 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = {
 	{STR_LIT("clamp"),            3, false, Expr_Expr},
 
 	{STR_LIT(""),                 0, true,  Expr_Expr}, // DIRECTIVE
+
+
+	// "Intrinsics"
+	{STR_LIT("__atomic_fence"),        0, false, Expr_Stmt},
+	{STR_LIT("__atomic_fence_acq"),    0, false, Expr_Stmt},
+	{STR_LIT("__atomic_fence_rel"),    0, false, Expr_Stmt},
+	{STR_LIT("__atomic_fence_acqrel"), 0, false, Expr_Stmt},
+
+	{STR_LIT("__atomic_store"),           2, false, Expr_Stmt},
+	{STR_LIT("__atomic_store_rel"),       2, false, Expr_Stmt},
+	{STR_LIT("__atomic_store_relaxed"),   2, false, Expr_Stmt},
+	{STR_LIT("__atomic_store_unordered"), 2, false, Expr_Stmt},
+
+	{STR_LIT("__atomic_load"),            1, false, Expr_Expr},
+	{STR_LIT("__atomic_load_acq"),        1, false, Expr_Expr},
+	{STR_LIT("__atomic_load_relaxed"),    1, false, Expr_Expr},
+	{STR_LIT("__atomic_load_unordered"),  1, false, Expr_Expr},
+
+	{STR_LIT("__atomic_add"),             2, false, Expr_Expr},
+	{STR_LIT("__atomic_add_acq"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_add_rel"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_add_acqrel"),      2, false, Expr_Expr},
+	{STR_LIT("__atomic_add_relaxed"),     2, false, Expr_Expr},
+	{STR_LIT("__atomic_sub"),             2, false, Expr_Expr},
+	{STR_LIT("__atomic_sub_acq"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_sub_rel"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_sub_acqrel"),      2, false, Expr_Expr},
+	{STR_LIT("__atomic_sub_relaxed"),     2, false, Expr_Expr},
+	{STR_LIT("__atomic_and"),             2, false, Expr_Expr},
+	{STR_LIT("__atomic_and_acq"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_and_rel"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_and_acqrel"),      2, false, Expr_Expr},
+	{STR_LIT("__atomic_and_relaxed"),     2, false, Expr_Expr},
+	{STR_LIT("__atomic_nand"),            2, false, Expr_Expr},
+	{STR_LIT("__atomic_nand_acq"),        2, false, Expr_Expr},
+	{STR_LIT("__atomic_nand_rel"),        2, false, Expr_Expr},
+	{STR_LIT("__atomic_nand_acqrel"),     2, false, Expr_Expr},
+	{STR_LIT("__atomic_nand_relaxed"),    2, false, Expr_Expr},
+	{STR_LIT("__atomic_or"),              2, false, Expr_Expr},
+	{STR_LIT("__atomic_or_acq"),          2, false, Expr_Expr},
+	{STR_LIT("__atomic_or_rel"),          2, false, Expr_Expr},
+	{STR_LIT("__atomic_or_acqrel"),       2, false, Expr_Expr},
+	{STR_LIT("__atomic_or_relaxed"),      2, false, Expr_Expr},
+	{STR_LIT("__atomic_xor"),             2, false, Expr_Expr},
+	{STR_LIT("__atomic_xor_acq"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_xor_rel"),         2, false, Expr_Expr},
+	{STR_LIT("__atomic_xor_acqrel"),      2, false, Expr_Expr},
+	{STR_LIT("__atomic_xor_relaxed"),     2, false, Expr_Expr},
+
+	{STR_LIT("__atomic_xchg"),            2, false, Expr_Expr},
+	{STR_LIT("__atomic_xchg_acq"),        2, false, Expr_Expr},
+	{STR_LIT("__atomic_xchg_rel"),        2, false, Expr_Expr},
+	{STR_LIT("__atomic_xchg_acqrel"),     2, false, Expr_Expr},
+	{STR_LIT("__atomic_xchg_relaxed"),    2, false, Expr_Expr},
+
+	{STR_LIT("__atomic_cxchg"),                    3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_acq"),                3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_rel"),                3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_acqrel"),             3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_relaxed"),            3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_failrelaxed"),        3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_failacq"),            3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_acq_failrelaxed"),    3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchg_acqrel_failrelaxed"), 3, false, Expr_Expr},
+
+	{STR_LIT("__atomic_cxchgweak"),                    3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_acq"),                3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_rel"),                3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_acqrel"),             3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_relaxed"),            3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_failrelaxed"),        3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_failacq"),            3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_acq_failrelaxed"),    3, false, Expr_Expr},
+	{STR_LIT("__atomic_cxchgweak_acqrel_failrelaxed"), 3, false, Expr_Expr},
 };
 
 
@@ -250,7 +397,7 @@ struct ImportGraphNode {
 
 
 struct ForeignContext {
-	Ast *             curr_library;
+	Ast *                 curr_library;
 	ProcCallingConvention default_cc;
 	String                link_prefix;
 	bool                  in_export;
diff --git a/src/entity.cpp b/src/entity.cpp
index 2dd4a4be0..15a70f0a5 100644
--- a/src/entity.cpp
+++ b/src/entity.cpp
@@ -122,13 +122,13 @@ struct Entity {
 			String ir_mangled_name;
 		} TypeName;
 		struct {
-			u64          tags;
-			Entity *     foreign_library;
-			Ast *    foreign_library_ident;
-			String       link_name;
-			String       link_prefix;
-			bool         is_foreign;
-			bool         is_export;
+			u64     tags;
+			Entity *foreign_library;
+			Ast *   foreign_library_ident;
+			String  link_name;
+			String  link_prefix;
+			bool    is_foreign;
+			bool    is_export;
 		} Procedure;
 		struct {
 			Array<Entity *> entities;
diff --git a/src/ir.cpp b/src/ir.cpp
index 29cc9247c..9189d63c3 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -184,10 +184,27 @@ gbAllocator ir_allocator(void) {
 		i64          alignment;                                       \
 	})                                                                \
 	IR_INSTR_KIND(ZeroInit, struct { irValue *address; })             \
-	IR_INSTR_KIND(Store,    struct {                                  \
-		irValue *address, *value; bool atomic;                        \
-	})                                                                \
+	IR_INSTR_KIND(Store,    struct { irValue *address, *value; })     \
 	IR_INSTR_KIND(Load,     struct { Type *type; irValue *address; }) \
+	IR_INSTR_KIND(AtomicFence, struct { BuiltinProcId id; })          \
+	IR_INSTR_KIND(AtomicStore, struct {                               \
+		irValue *address, *value;                                     \
+		BuiltinProcId id;                                             \
+	})                                                                \
+	IR_INSTR_KIND(AtomicLoad, struct {                                \
+		Type *type; irValue *address;                                 \
+		BuiltinProcId id;                                             \
+	})                                                                \
+	IR_INSTR_KIND(AtomicRmw, struct {                                 \
+		Type *type; irValue *address;                                 \
+		irValue *value;                                               \
+		BuiltinProcId id;                                             \
+	})                                                                \
+	IR_INSTR_KIND(AtomicCxchg, struct {                               \
+		Type *type; irValue *address;                                 \
+		irValue *old_value; irValue *new_value;                       \
+		BuiltinProcId id;                                             \
+	})                                                                \
 	IR_INSTR_KIND(PtrOffset, struct {                                 \
 		irValue *address;                                             \
 		irValue *offset;                                              \
@@ -485,7 +502,7 @@ irAddr ir_addr_map(irValue *addr, irValue *map_key, Type *map_type, Type *map_re
 }
 
 
-irAddr ir_addr_context(irValue *addr, Selection sel = {}) {
+irAddr ir_addr_context(irValue *addr, Selection sel = empty_selection) {
 	irAddr v = {irAddr_Context, addr};
 	v.ctx.sel = sel;
 	return v;
@@ -644,6 +661,12 @@ Type *ir_instr_type(irInstr *instr) {
 		return instr->Local.type;
 	case irInstr_Load:
 		return instr->Load.type;
+	case irInstr_AtomicLoad:
+		return instr->AtomicLoad.type;
+	case irInstr_AtomicRmw:
+		return instr->AtomicRmw.type;
+	case irInstr_AtomicCxchg:
+		return instr->AtomicCxchg.type;
 	case irInstr_StructElementPtr:
 		return instr->StructElementPtr.result_type;
 	case irInstr_ArrayElementPtr:
@@ -925,12 +948,11 @@ irValue *ir_instr_zero_init(irProcedure *p, irValue *address) {
 	return v;
 }
 
-irValue *ir_instr_store(irProcedure *p, irValue *address, irValue *value, bool atomic) {
+irValue *ir_instr_store(irProcedure *p, irValue *address, irValue *value) {
 	irValue *v = ir_alloc_instr(p, irInstr_Store);
 	irInstr *i = &v->Instr;
 	i->Store.address = address;
 	i->Store.value = value;
-	i->Store.atomic = atomic;
 	return v;
 }
 
@@ -942,6 +964,68 @@ irValue *ir_instr_load(irProcedure *p, irValue *address) {
 	return v;
 }
 
+irValue *ir_instr_atomic_fence(irProcedure *p, BuiltinProcId id) {
+	irValue *v = ir_alloc_instr(p, irInstr_AtomicFence);
+	irInstr *i = &v->Instr;
+	i->AtomicFence.id = id;
+	return v;
+}
+
+irValue *ir_instr_atomic_store(irProcedure *p, irValue *address, irValue *value, BuiltinProcId id) {
+	irValue *v = ir_alloc_instr(p, irInstr_AtomicStore);
+	irInstr *i = &v->Instr;
+	i->AtomicStore.address = address;
+	i->AtomicStore.value = value;
+	i->AtomicStore.id = id;
+	return v;
+}
+
+irValue *ir_instr_atomic_load(irProcedure *p, irValue *address, BuiltinProcId id) {
+	irValue *v = ir_alloc_instr(p, irInstr_AtomicLoad);
+	irInstr *i = &v->Instr;
+	i->AtomicLoad.address = address;
+	i->AtomicLoad.type = type_deref(ir_type(address));
+	i->AtomicLoad.id = id;
+	return v;
+}
+
+irValue *ir_instr_atomic_rmw(irProcedure *p, irValue *address, irValue *value, BuiltinProcId id) {
+	irValue *v = ir_alloc_instr(p, irInstr_AtomicRmw);
+	irInstr *i = &v->Instr;
+	i->AtomicRmw.type = type_deref(ir_type(address));
+	i->AtomicRmw.address = address;
+	i->AtomicRmw.value = value;
+	i->AtomicRmw.id = id;
+	return v;
+}
+
+
+irValue *ir_instr_atomic_cxchg(irProcedure *p, Type *type, irValue *address, irValue *old_value, irValue *new_value, BuiltinProcId id) {
+	irValue *v = ir_alloc_instr(p, irInstr_AtomicCxchg);
+	irInstr *i = &v->Instr;
+
+
+	if (type->kind == Type_Tuple) {
+		GB_ASSERT(type->Tuple.variables.count == 2);
+		Type *elem = type->Tuple.variables[0]->type;
+		// LEAK TODO(bill): LLVM returns {T, i1} whilst Odin does {T, bool}, fix this mapping hack
+		gbAllocator a = heap_allocator();
+		Type *llvm_type = alloc_type_tuple();
+		array_init(&llvm_type->Tuple.variables, a, 0, 2);
+		array_add (&llvm_type->Tuple.variables, alloc_entity_field(nullptr, blank_token, elem, false, 0));
+		array_add (&llvm_type->Tuple.variables, alloc_entity_field(nullptr, blank_token, t_llvm_bool, false, 1));
+
+		type = llvm_type;
+	}
+	i->AtomicCxchg.type = type;
+
+	i->AtomicCxchg.address = address;
+	i->AtomicCxchg.old_value = old_value;
+	i->AtomicCxchg.new_value = new_value;
+	i->AtomicCxchg.id = id;
+	return v;
+}
+
 irValue *ir_instr_array_element_ptr(irProcedure *p, irValue *address, irValue *elem_index) {
 	irValue *v = ir_alloc_instr(p, irInstr_ArrayElementPtr);
 	irInstr *i = &v->Instr;
@@ -1579,7 +1663,7 @@ irValue *ir_emit_store(irProcedure *p, irValue *address, irValue *value) {
 	if (!is_type_untyped(b)) {
 		GB_ASSERT_MSG(are_types_identical(core_type(a), core_type(b)), "%s %s", type_to_string(a), type_to_string(b));
 	}
-	return ir_emit(p, ir_instr_store(p, address, value, false));
+	return ir_emit(p, ir_instr_store(p, address, value));
 }
 irValue *ir_emit_load(irProcedure *p, irValue *address) {
 	GB_ASSERT(address != nullptr);
@@ -4710,6 +4794,106 @@ irValue *ir_build_builtin_proc(irProcedure *proc, Ast *expr, TypeAndValue tv, Bu
 		                     ir_build_expr(proc, ce->args[0]),
 		                     ir_build_expr(proc, ce->args[1]),
 		                     ir_build_expr(proc, ce->args[2]));
+
+
+
+	// "Intrinsics"
+	case BuiltinProc_atomic_fence:
+	case BuiltinProc_atomic_fence_acq:
+	case BuiltinProc_atomic_fence_rel:
+	case BuiltinProc_atomic_fence_acqrel:
+		return ir_emit(proc, ir_instr_atomic_fence(proc, id));
+
+	case BuiltinProc_atomic_store:
+	case BuiltinProc_atomic_store_rel:
+	case BuiltinProc_atomic_store_relaxed:
+	case BuiltinProc_atomic_store_unordered: {
+		irValue *dst = ir_build_expr(proc, ce->args[0]);
+		irValue *val = ir_build_expr(proc, ce->args[1]);
+		val = ir_emit_conv(proc, val, type_deref(ir_type(dst)));
+		return ir_emit(proc, ir_instr_atomic_store(proc, dst, val, id));
+	}
+
+	case BuiltinProc_atomic_load:
+	case BuiltinProc_atomic_load_acq:
+	case BuiltinProc_atomic_load_relaxed:
+	case BuiltinProc_atomic_load_unordered: {
+		irValue *dst = ir_build_expr(proc, ce->args[0]);
+		return ir_emit(proc, ir_instr_atomic_load(proc, dst, id));
+	}
+
+	case BuiltinProc_atomic_add:
+	case BuiltinProc_atomic_add_acq:
+	case BuiltinProc_atomic_add_rel:
+	case BuiltinProc_atomic_add_acqrel:
+	case BuiltinProc_atomic_add_relaxed:
+	case BuiltinProc_atomic_sub:
+	case BuiltinProc_atomic_sub_acq:
+	case BuiltinProc_atomic_sub_rel:
+	case BuiltinProc_atomic_sub_acqrel:
+	case BuiltinProc_atomic_sub_relaxed:
+	case BuiltinProc_atomic_and:
+	case BuiltinProc_atomic_and_acq:
+	case BuiltinProc_atomic_and_rel:
+	case BuiltinProc_atomic_and_acqrel:
+	case BuiltinProc_atomic_and_relaxed:
+	case BuiltinProc_atomic_nand:
+	case BuiltinProc_atomic_nand_acq:
+	case BuiltinProc_atomic_nand_rel:
+	case BuiltinProc_atomic_nand_acqrel:
+	case BuiltinProc_atomic_nand_relaxed:
+	case BuiltinProc_atomic_or:
+	case BuiltinProc_atomic_or_acq:
+	case BuiltinProc_atomic_or_rel:
+	case BuiltinProc_atomic_or_acqrel:
+	case BuiltinProc_atomic_or_relaxed:
+	case BuiltinProc_atomic_xor:
+	case BuiltinProc_atomic_xor_acq:
+	case BuiltinProc_atomic_xor_rel:
+	case BuiltinProc_atomic_xor_acqrel:
+	case BuiltinProc_atomic_xor_relaxed:
+	case BuiltinProc_atomic_xchg:
+	case BuiltinProc_atomic_xchg_acq:
+	case BuiltinProc_atomic_xchg_rel:
+	case BuiltinProc_atomic_xchg_acqrel:
+	case BuiltinProc_atomic_xchg_relaxed: {
+		irValue *dst = ir_build_expr(proc, ce->args[0]);
+		irValue *val = ir_build_expr(proc, ce->args[1]);
+		val = ir_emit_conv(proc, val, type_deref(ir_type(dst)));
+		return ir_emit(proc, ir_instr_atomic_rmw(proc, dst, val, id));
+	}
+
+	case BuiltinProc_atomic_cxchg:
+	case BuiltinProc_atomic_cxchg_acq:
+	case BuiltinProc_atomic_cxchg_rel:
+	case BuiltinProc_atomic_cxchg_acqrel:
+	case BuiltinProc_atomic_cxchg_relaxed:
+	case BuiltinProc_atomic_cxchg_failrelaxed:
+	case BuiltinProc_atomic_cxchg_failacq:
+	case BuiltinProc_atomic_cxchg_acq_failrelaxed:
+	case BuiltinProc_atomic_cxchg_acqrel_failrelaxed:
+	case BuiltinProc_atomic_cxchgweak:
+	case BuiltinProc_atomic_cxchgweak_acq:
+	case BuiltinProc_atomic_cxchgweak_rel:
+	case BuiltinProc_atomic_cxchgweak_acqrel:
+	case BuiltinProc_atomic_cxchgweak_relaxed:
+	case BuiltinProc_atomic_cxchgweak_failrelaxed:
+	case BuiltinProc_atomic_cxchgweak_failacq:
+	case BuiltinProc_atomic_cxchgweak_acq_failrelaxed:
+	case BuiltinProc_atomic_cxchgweak_acqrel_failrelaxed: {
+		Type *type = expr->tav.type;
+
+		irValue *address = ir_build_expr(proc, ce->args[0]);
+		Type *elem = type_deref(ir_type(address));
+		irValue *old_value = ir_build_expr(proc, ce->args[1]);
+		irValue *new_value = ir_build_expr(proc, ce->args[2]);
+		old_value = ir_emit_conv(proc, old_value, elem);
+		new_value = ir_emit_conv(proc, new_value, elem);
+
+		return ir_emit(proc, ir_instr_atomic_cxchg(proc, type, address, old_value, new_value, id));
+	}
+
+
 	}
 
 	GB_PANIC("Unhandled built-in procedure");
@@ -8430,6 +8614,9 @@ void ir_gen_tree(irGen *s) {
 			continue;
 		case Entity_ProcGroup:
 			continue;
+
+		case Entity_Procedure:
+			break;
 		}
 
 		bool polymorphic_struct = false;
diff --git a/src/ir_print.cpp b/src/ir_print.cpp
index 5796845fb..aec43b276 100644
--- a/src/ir_print.cpp
+++ b/src/ir_print.cpp
@@ -1040,6 +1040,275 @@ void ir_print_instr(irFileBuffer *f, irModule *m, irValue *value) {
 		ir_print_debug_location(f, m, value);
 		break;
 	}
+
+	case irInstr_AtomicFence:
+		ir_write_str_lit(f, "fence ");
+		switch (instr->AtomicFence.id) {
+		case BuiltinProc_atomic_fence:        ir_write_str_lit(f, "seq_cst"); break;
+		case BuiltinProc_atomic_fence_acq:    ir_write_str_lit(f, "acquire"); break;
+		case BuiltinProc_atomic_fence_rel:    ir_write_str_lit(f, "release"); break;
+		case BuiltinProc_atomic_fence_acqrel: ir_write_str_lit(f, "acq_rel"); break;
+		default: GB_PANIC("Unknown atomic fence"); break;
+		}
+		break;
+
+	case irInstr_AtomicStore: {
+		Type *type = type_deref(ir_type(instr->AtomicStore.address));
+		ir_write_str_lit(f, "store atomic ");
+		ir_print_type(f, m, type);
+		ir_write_byte(f, ' ');
+		ir_print_value(f, m, instr->AtomicStore.value, type);
+		ir_write_str_lit(f, ", ");
+		ir_print_type(f, m, type);
+		ir_write_str_lit(f, "* ");
+		ir_print_value(f, m, instr->AtomicStore.address, type);
+
+		switch (instr->AtomicStore.id) {
+		case BuiltinProc_atomic_store:           ir_write_str_lit(f, " seq_cst ");  break;
+		case BuiltinProc_atomic_store_rel:       ir_write_str_lit(f, " release");   break;
+		case BuiltinProc_atomic_store_relaxed:   ir_write_str_lit(f, " monotonic"); break;
+		case BuiltinProc_atomic_store_unordered: ir_write_str_lit(f, " unordered"); break;
+		default: GB_PANIC("Unknown atomic store"); break;
+		}
+
+		ir_fprintf(f, ", align %lld", type_align_of(type));
+
+		ir_print_debug_location(f, m, value);
+		break;
+	}
+
+	case irInstr_AtomicLoad: {
+		Type *type = instr->AtomicLoad.type;
+		ir_fprintf(f, "%%%d = load atomic ", value->index);
+		ir_print_type(f, m, type);
+		ir_write_str_lit(f, ", ");
+		ir_print_type(f, m, type);
+		ir_write_str_lit(f, "* ");
+		ir_print_value(f, m, instr->AtomicLoad.address, type);
+
+		switch (instr->AtomicLoad.id) {
+		case BuiltinProc_atomic_load:           ir_fprintf(f, " seq_cst");   break;
+		case BuiltinProc_atomic_load_acq:       ir_fprintf(f, " acquire");   break;
+		case BuiltinProc_atomic_load_relaxed:   ir_fprintf(f, " monotonic"); break;
+		case BuiltinProc_atomic_load_unordered: ir_fprintf(f, " unordered"); break;
+		default: GB_PANIC("Unknown atomic load"); break;
+		}
+
+		ir_fprintf(f, ", align %lld", type_align_of(type));
+		ir_print_debug_location(f, m, value);
+		break;
+	}
+
+	case irInstr_AtomicRmw: {
+		String operation = {};
+		String ordering = {};
+		switch (instr->AtomicRmw.id) {
+		case BuiltinProc_atomic_add:
+		case BuiltinProc_atomic_add_acq:
+		case BuiltinProc_atomic_add_rel:
+		case BuiltinProc_atomic_add_acqrel:
+		case BuiltinProc_atomic_add_relaxed:
+			operation = str_lit("add");
+			break;
+		case BuiltinProc_atomic_sub:
+		case BuiltinProc_atomic_sub_acq:
+		case BuiltinProc_atomic_sub_rel:
+		case BuiltinProc_atomic_sub_acqrel:
+		case BuiltinProc_atomic_sub_relaxed:
+			operation = str_lit("sub");
+			break;
+		case BuiltinProc_atomic_and:
+		case BuiltinProc_atomic_and_acq:
+		case BuiltinProc_atomic_and_rel:
+		case BuiltinProc_atomic_and_acqrel:
+		case BuiltinProc_atomic_and_relaxed:
+			operation = str_lit("and");
+			break;
+		case BuiltinProc_atomic_nand:
+		case BuiltinProc_atomic_nand_acq:
+		case BuiltinProc_atomic_nand_rel:
+		case BuiltinProc_atomic_nand_acqrel:
+		case BuiltinProc_atomic_nand_relaxed:
+			operation = str_lit("nand");
+			break;
+		case BuiltinProc_atomic_or:
+		case BuiltinProc_atomic_or_acq:
+		case BuiltinProc_atomic_or_rel:
+		case BuiltinProc_atomic_or_acqrel:
+		case BuiltinProc_atomic_or_relaxed:
+			operation = str_lit("or");
+			break;
+		case BuiltinProc_atomic_xor:
+		case BuiltinProc_atomic_xor_acq:
+		case BuiltinProc_atomic_xor_rel:
+		case BuiltinProc_atomic_xor_acqrel:
+		case BuiltinProc_atomic_xor_relaxed:
+			operation = str_lit("xor");
+			break;
+		case BuiltinProc_atomic_xchg:
+		case BuiltinProc_atomic_xchg_acq:
+		case BuiltinProc_atomic_xchg_rel:
+		case BuiltinProc_atomic_xchg_acqrel:
+		case BuiltinProc_atomic_xchg_relaxed:
+			operation = str_lit("xchg");
+			break;
+		}
+
+		switch (instr->AtomicRmw.id) {
+		case BuiltinProc_atomic_add:
+		case BuiltinProc_atomic_sub:
+		case BuiltinProc_atomic_and:
+		case BuiltinProc_atomic_nand:
+		case BuiltinProc_atomic_or:
+		case BuiltinProc_atomic_xor:
+		case BuiltinProc_atomic_xchg:
+			ordering = str_lit("seq_cst");
+			break;
+		case BuiltinProc_atomic_add_acq:
+		case BuiltinProc_atomic_sub_acq:
+		case BuiltinProc_atomic_and_acq:
+		case BuiltinProc_atomic_nand_acq:
+		case BuiltinProc_atomic_or_acq:
+		case BuiltinProc_atomic_xor_acq:
+		case BuiltinProc_atomic_xchg_acq:
+			ordering = str_lit("acquire");
+			break;
+		case BuiltinProc_atomic_add_rel:
+		case BuiltinProc_atomic_sub_rel:
+		case BuiltinProc_atomic_and_rel:
+		case BuiltinProc_atomic_nand_rel:
+		case BuiltinProc_atomic_or_rel:
+		case BuiltinProc_atomic_xor_rel:
+		case BuiltinProc_atomic_xchg_rel:
+			ordering = str_lit("release");
+			break;
+		case BuiltinProc_atomic_add_acqrel:
+		case BuiltinProc_atomic_sub_acqrel:
+		case BuiltinProc_atomic_and_acqrel:
+		case BuiltinProc_atomic_nand_acqrel:
+		case BuiltinProc_atomic_or_acqrel:
+		case BuiltinProc_atomic_xor_acqrel:
+		case BuiltinProc_atomic_xchg_acqrel:
+			ordering = str_lit("acq_rel");
+			break;
+		case BuiltinProc_atomic_add_relaxed:
+		case BuiltinProc_atomic_sub_relaxed:
+		case BuiltinProc_atomic_and_relaxed:
+		case BuiltinProc_atomic_nand_relaxed:
+		case BuiltinProc_atomic_or_relaxed:
+		case BuiltinProc_atomic_xor_relaxed:
+		case BuiltinProc_atomic_xchg_relaxed:
+			ordering = str_lit("monotonic");
+			break;
+		}
+
+		Type *type = type_deref(ir_type(instr->AtomicRmw.address));
+		ir_write_str_lit(f, "atomicrmw ");
+		ir_write_string(f, operation);
+		ir_write_byte(f, ' ');
+		ir_print_type(f, m, type);
+		ir_write_str_lit(f, "* ");
+		ir_print_value(f, m, instr->AtomicRmw.address, type);
+		ir_write_str_lit(f, ", ");
+		ir_print_type(f, m, type);
+		ir_write_byte(f, ' ');
+		ir_print_value(f, m, instr->AtomicRmw.value, type);
+
+		ir_write_byte(f, ' ');
+		ir_write_string(f, ordering);
+
+		ir_print_debug_location(f, m, value);
+		break;
+	}
+
+	case irInstr_AtomicCxchg: {
+		Type *type = type_deref(ir_type(instr->AtomicCxchg.address));
+		bool weak = false;
+		String success = {};
+		String failure = {};
+
+		switch (instr->AtomicCxchg.id) {
+		case BuiltinProc_atomic_cxchgweak:
+		case BuiltinProc_atomic_cxchgweak_acq:
+		case BuiltinProc_atomic_cxchgweak_rel:
+		case BuiltinProc_atomic_cxchgweak_acqrel:
+		case BuiltinProc_atomic_cxchgweak_relaxed:
+		case BuiltinProc_atomic_cxchgweak_failrelaxed:
+		case BuiltinProc_atomic_cxchgweak_failacq:
+		case BuiltinProc_atomic_cxchgweak_acq_failrelaxed:
+		case BuiltinProc_atomic_cxchgweak_acqrel_failrelaxed:
+			weak = true;
+			break;
+		}
+
+		switch (instr->AtomicCxchg.id) {
+		case BuiltinProc_atomic_cxchg:
+		case BuiltinProc_atomic_cxchgweak:
+			success = str_lit("seq_cst");
+			failure = str_lit("seq_cst");
+			break;
+		case BuiltinProc_atomic_cxchg_acq:
+		case BuiltinProc_atomic_cxchgweak_acq:
+			success = str_lit("acquire");
+			failure = str_lit("seq_cst");
+			break;
+		case BuiltinProc_atomic_cxchg_rel:
+		case BuiltinProc_atomic_cxchgweak_rel:
+			success = str_lit("release");
+			failure = str_lit("seq_cst");
+			break;
+		case BuiltinProc_atomic_cxchg_acqrel:
+		case BuiltinProc_atomic_cxchgweak_acqrel:
+			success = str_lit("acq_rel");
+			failure = str_lit("seq_cst");
+			break;
+		case BuiltinProc_atomic_cxchg_relaxed:
+		case BuiltinProc_atomic_cxchgweak_relaxed:
+			success = str_lit("monotonic");
+			failure = str_lit("monotonic");
+			break;
+		case BuiltinProc_atomic_cxchg_failrelaxed:
+		case BuiltinProc_atomic_cxchgweak_failrelaxed:
+			success = str_lit("seq_cst");
+			failure = str_lit("monotonic");
+			break;
+		case BuiltinProc_atomic_cxchg_failacq:
+		case BuiltinProc_atomic_cxchgweak_failacq:
+			success = str_lit("seq_cst");
+			failure = str_lit("acquire");
+			break;
+		case BuiltinProc_atomic_cxchg_acq_failrelaxed:
+		case BuiltinProc_atomic_cxchgweak_acq_failrelaxed:
+			success = str_lit("acquire");
+			failure = str_lit("monotonic");
+			break;
+		case BuiltinProc_atomic_cxchg_acqrel_failrelaxed:
+		case BuiltinProc_atomic_cxchgweak_acqrel_failrelaxed:
+			success = str_lit("acq_rel");
+			failure = str_lit("monotonic");
+			break;
+		}
+
+		ir_fprintf(f, "%%%d = cmpxchg ", value->index);
+		if (weak) {
+			ir_write_str_lit(f, "weak ");
+		}
+		ir_print_type(f, m, type);
+		ir_write_str_lit(f, "* ");
+		ir_print_value(f, m, instr->AtomicCxchg.address, type);
+		ir_write_str_lit(f, ", ");
+		ir_print_type(f, m, type); ir_write_str_lit(f, " ");
+		ir_print_value(f, m, instr->AtomicCxchg.old_value, type);
+		ir_write_str_lit(f, ", ");
+		ir_print_type(f, m, type); ir_write_str_lit(f, " ");
+		ir_print_value(f, m, instr->AtomicCxchg.new_value, type);
+		ir_write_str_lit(f, " ");
+		ir_write_string(f, success);
+		ir_write_str_lit(f, " ");
+		ir_write_string(f, failure);
+		break;
+	}
+
 	case irInstr_ArrayElementPtr: {
 		Type *et = ir_type(instr->ArrayElementPtr.address);
 		ir_fprintf(f, "%%%d = getelementptr inbounds ", value->index);