From caa344c88d7b2f736da58a8b9c88b510caafb68c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 12 Apr 2024 14:05:36 +0100 Subject: Simplify scalar -> array conversions in LLVM to use a loop after a certain size --- src/llvm_backend_general.cpp | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 0d8d9258a..73e4a00e6 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -954,16 +954,6 @@ gb_internal void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { GB_ASSERT(value.value != nullptr); value = lb_emit_conv(p, value, lb_addr_type(addr)); - // if (lb_is_const_or_global(value)) { - // // NOTE(bill): Just bypass the actual storage and set the initializer - // if (LLVMGetValueKind(addr.addr.value) == LLVMGlobalVariableValueKind) { - // LLVMValueRef dst = addr.addr.value; - // LLVMValueRef src = value.value; - // LLVMSetInitializer(dst, src); - // return; - // } - // } - lb_emit_store(p, addr.addr, value); } -- cgit v1.2.3 From a61ae7c861fa301684ee1582507061317b11426b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 16 Apr 2024 13:31:49 +0100 Subject: Fix #3427 --- src/check_type.cpp | 20 ++++++++++++++------ src/checker.hpp | 5 ++++- src/llvm_backend_general.cpp | 2 +- src/llvm_backend_type.cpp | 2 +- src/llvm_backend_utility.cpp | 4 ++-- src/types.cpp | 1 - 6 files changed, 22 insertions(+), 12 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/check_type.cpp b/src/check_type.cpp index f4e5d7c96..3bb1a4fd1 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2495,18 +2495,16 @@ gb_internal Type *get_map_cell_type(Type *type) { return s; } -gb_internal void init_map_internal_types(Type *type) { +gb_internal void init_map_internal_debug_types(Type *type) { GB_ASSERT(type->kind == Type_Map); GB_ASSERT(t_allocator != nullptr); - if (type->Map.lookup_result_type != nullptr) return; + if (type->Map.debug_metadata_type != nullptr) return; Type *key = type->Map.key; Type *value = type->Map.value; GB_ASSERT(key != nullptr); GB_ASSERT(value != nullptr); - - Type *key_cell = get_map_cell_type(key); Type *value_cell = get_map_cell_type(value); @@ -2541,6 +2539,18 @@ gb_internal void init_map_internal_types(Type *type) { gb_unused(type_size_of(debug_type)); type->Map.debug_metadata_type = debug_type; +} + + +gb_internal void init_map_internal_types(Type *type) { + GB_ASSERT(type->kind == Type_Map); + GB_ASSERT(t_allocator != nullptr); + if (type->Map.lookup_result_type != nullptr) return; + + Type *key = type->Map.key; + Type *value = type->Map.value; + GB_ASSERT(key != nullptr); + GB_ASSERT(value != nullptr); type->Map.lookup_result_type = make_optional_ok_type(value); } @@ -2613,8 +2623,6 @@ gb_internal void check_map_type(CheckerContext *ctx, Type *type, Ast *node) { init_core_map_type(ctx->checker); init_map_internal_types(type); - - // error(node, "'map' types are not yet implemented"); } gb_internal void check_matrix_type(CheckerContext *ctx, Type **type, Ast *node) { diff --git a/src/checker.hpp b/src/checker.hpp index 1701da58d..2ade9312e 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -563,4 +563,7 @@ gb_internal void init_mem_allocator(Checker *c); gb_internal void add_untyped_expressions(CheckerInfo *cinfo, UntypedExprInfoMap *untyped); -gb_internal GenTypesData *ensure_polymorphic_record_entity_has_gen_types(CheckerContext *ctx, Type *original_type); \ No newline at end of file +gb_internal GenTypesData *ensure_polymorphic_record_entity_has_gen_types(CheckerContext *ctx, Type *original_type); + + +gb_internal void init_map_internal_types(Type *type); \ No newline at end of file diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 73e4a00e6..7a5ed5635 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -2070,7 +2070,7 @@ gb_internal LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { break; case Type_Map: - init_map_internal_types(type); + init_map_internal_debug_types(type); GB_ASSERT(t_raw_map != nullptr); return lb_type_internal(m, t_raw_map); diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 93e2874a5..0bac2f732 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -903,7 +903,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ case Type_Map: { tag_type = t_type_info_map; - init_map_internal_types(t); + init_map_internal_debug_types(t); LLVMValueRef vals[3] = { get_type_info_ptr(m, t->Map.key), diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 865c3f1ec..0d1db2cbf 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1125,7 +1125,7 @@ gb_internal lbValue lb_emit_struct_ep(lbProcedure *p, lbValue s, i32 index) { case 3: result_type = t_allocator; break; } } else if (is_type_map(t)) { - init_map_internal_types(t); + init_map_internal_debug_types(t); Type *itp = alloc_type_pointer(t_raw_map); s = lb_emit_transmute(p, s, itp); @@ -1264,7 +1264,7 @@ gb_internal lbValue lb_emit_struct_ev(lbProcedure *p, lbValue s, i32 index) { case Type_Map: { - init_map_internal_types(t); + init_map_internal_debug_types(t); switch (index) { case 0: result_type = get_struct_field_type(t_raw_map, 0); break; case 1: result_type = get_struct_field_type(t_raw_map, 1); break; diff --git a/src/types.cpp b/src/types.cpp index 97512d29b..18cb12ea1 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -769,7 +769,6 @@ gb_internal gbString type_to_string (Type *type, bool shorthand=true); gb_internal gbString type_to_string (Type *type, gbAllocator allocator, bool shorthand=true); gb_internal i64 type_size_of_internal(Type *t, TypePath *path); gb_internal i64 type_align_of_internal(Type *t, TypePath *path); -gb_internal void init_map_internal_types(Type *type); gb_internal Type * bit_set_to_int(Type *t); gb_internal bool are_types_identical(Type *x, Type *y); -- cgit v1.2.3 From 5200e3fe7a0eff6ecc76838e20ad33762ba08d5d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 18 Apr 2024 12:45:20 +0100 Subject: Set `__$ti-` stuff to be private linkage --- src/llvm_backend_general.cpp | 2 +- src/llvm_backend_type.cpp | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 7a5ed5635..da69f94d7 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -211,7 +211,7 @@ gb_internal void lb_loop_end(lbProcedure *p, lbLoopData const &data) { gb_internal void lb_make_global_private_const(LLVMValueRef global_data) { - LLVMSetLinkage(global_data, LLVMPrivateLinkage); + LLVMSetLinkage(global_data, LLVMLinkerPrivateLinkage); LLVMSetUnnamedAddress(global_data, LLVMGlobalUnnamedAddr); LLVMSetGlobalConstant(global_data, true); } diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 78e6af852..e202a59ba 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -249,9 +249,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ char name[64] = {}; gb_snprintf(name, 63, "__$ti-%lld", cast(long long)index); LLVMValueRef g = LLVMAddGlobal(m->mod, type, name); - LLVMSetLinkage(g, LLVMInternalLinkage); - LLVMSetUnnamedAddress(g, LLVMGlobalUnnamedAddr); - LLVMSetGlobalConstant(g, true); + lb_make_global_private_const(g); return g; }; @@ -1103,8 +1101,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ LLVMValueRef giant_const = LLVMConstArray(lb_type(m, t_type_info_ptr), giant_const_values, cast(unsigned)global_type_info_data_entity_count); LLVMValueRef giant_array = lb_global_type_info_data_ptr(m).value; LLVMSetInitializer(giant_array, giant_const); - LLVMSetGlobalConstant(giant_array, true); - LLVMSetLinkage(giant_array, LLVMLinkerPrivateLinkage); + lb_make_global_private_const(giant_array); } -- cgit v1.2.3 From ec5a84a5379236a2413b8f3115509629879f5b53 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Apr 2024 13:10:58 +0100 Subject: Improve code generation for loading `bit_field` fields --- src/check_type.cpp | 15 ++++++--- src/llvm_backend_general.cpp | 80 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 75 insertions(+), 20 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/check_type.cpp b/src/check_type.cpp index a6dbb8dfc..77ac91c38 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -955,14 +955,19 @@ gb_internal void check_bit_field_type(CheckerContext *ctx, Type *bit_field_type, GB_ASSERT(is_type_bit_field(bit_field_type)); Type *backing_type = check_type(ctx, bf->backing_type); - if (backing_type == nullptr || !is_valid_bit_field_backing_type(backing_type)) { - error(node, "Backing type for a bit_field must be an integer or an array of an integer"); - return; - } - bit_field_type->BitField.backing_type = backing_type; + bit_field_type->BitField.backing_type = backing_type ? backing_type : t_u8; bit_field_type->BitField.scope = ctx->scope; + if (backing_type == nullptr) { + error(bf->backing_type, "Backing type for a bit_field must be an integer or an array of an integer"); + return; + } + if (!is_valid_bit_field_backing_type(backing_type)) { + error(bf->backing_type, "Backing type for a bit_field must be an integer or an array of an integer"); + return; + } + auto fields = array_make(permanent_allocator(), 0, bf->fields.count); auto bit_sizes = array_make (permanent_allocator(), 0, bf->fields.count); auto tags = array_make (permanent_allocator(), 0, bf->fields.count); diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index da69f94d7..b8fbd231e 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -774,13 +774,23 @@ gb_internal void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { if (addr.kind == lbAddr_BitField) { lbValue dst = addr.addr; + lbValue src = lb_address_from_load_or_generate_local(p, value); - auto args = array_make(temporary_allocator(), 4); - args[0] = dst; - args[1] = lb_address_from_load_or_generate_local(p, value); - args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); - args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); - lb_emit_runtime_call(p, "__write_bits", args); + if ((addr.bitfield.bit_offset & 7) == 0 && + (addr.bitfield.bit_size & 7) == 0) { + lbValue byte_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset/8); + lbValue byte_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size/8); + lbValue dst_offset = lb_emit_conv(p, dst, t_u8_ptr); + dst_offset = lb_emit_ptr_offset(p, dst_offset, byte_offset); + lb_mem_copy_non_overlapping(p, dst_offset, src, byte_size); + } else { + auto args = array_make(temporary_allocator(), 4); + args[0] = dst; + args[1] = src; + args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lb_emit_runtime_call(p, "__write_bits", args); + } return; } else if (addr.kind == lbAddr_RelativePointer) { Type *rel_ptr = base_type(lb_addr_type(addr)); @@ -1088,23 +1098,63 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(addr.addr.value != nullptr); if (addr.kind == lbAddr_BitField) { - lbAddr dst = lb_add_local_generated(p, addr.bitfield.type, true); + Type *ct = core_type(addr.bitfield.type); + bool do_mask = false; + if (is_type_unsigned(ct) || is_type_boolean(ct)) { + // Mask + if (addr.bitfield.bit_size != 8*type_size_of(ct)) { + do_mask = true; + } + } + + i64 total_bitfield_bit_size = 8*type_size_of(lb_addr_type(addr)); + i64 dst_byte_size = type_size_of(addr.bitfield.type); + lbAddr dst = lb_add_local_generated(p, addr.bitfield.type, false); lbValue src = addr.addr; - auto args = array_make(temporary_allocator(), 4); - args[0] = dst.addr; - args[1] = src; - args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); - args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); - lb_emit_runtime_call(p, "__read_bits", args); + lbValue bit_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + lbValue bit_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lbValue byte_offset = lb_const_int(p->module, t_uintptr, (addr.bitfield.bit_offset+7)/8); + lbValue byte_size = lb_const_int(p->module, t_uintptr, (addr.bitfield.bit_size+7)/8); + + GB_ASSERT(type_size_of(addr.bitfield.type) >= ((addr.bitfield.bit_size+7)/8)); + + if ((addr.bitfield.bit_offset & 7) == 0) { + lbValue copy_size = byte_size; + lbValue src_offset = lb_emit_conv(p, src, t_u8_ptr); + src_offset = lb_emit_ptr_offset(p, src_offset, byte_offset); + if (addr.bitfield.bit_offset + dst_byte_size <= total_bitfield_bit_size) { + do_mask = true; + copy_size = lb_const_int(p->module, t_uintptr, dst_byte_size); + } + lb_mem_copy_non_overlapping(p, dst.addr, src_offset, copy_size, false); + } else { + auto args = array_make(temporary_allocator(), 4); + args[0] = dst.addr; + args[1] = src; + args[2] = bit_offset; + args[3] = bit_size; + lb_emit_runtime_call(p, "__read_bits", args); + } lbValue r = lb_addr_load(p, dst); + Type *t = addr.bitfield.type; + + if (do_mask) { + GB_ASSERT(addr.bitfield.bit_size < 8*type_size_of(ct)); + + LLVMTypeRef lt = lb_type(p->module, t); + LLVMValueRef mask = LLVMConstInt(lt, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(lt, addr.bitfield.bit_size, false)); + mask = LLVMConstSub(mask, LLVMConstInt(lt, 1, false)); + lbValue m = {mask, t}; + r = lb_emit_arith(p, Token_And, r, m, t); + } - if (!is_type_unsigned(core_type(addr.bitfield.type))) { + if (!is_type_unsigned(ct) && !is_type_boolean(ct)) { // Sign extension // m := 1<<(bit_size-1) // r = (r XOR m) - m - Type *t = addr.bitfield.type; lbValue m = lb_const_int(p->module, t, 1ull<<(addr.bitfield.bit_size-1)); r = lb_emit_arith(p, Token_Xor, r, m, t); r = lb_emit_arith(p, Token_Sub, r, m, t); -- cgit v1.2.3 From c330e5b5c1b512e1b0ca7181941057e5f2e085e4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Apr 2024 14:46:34 +0100 Subject: Improve codegen for `bit_field` compound literals with an integer backing --- src/llvm_backend.hpp | 1 - src/llvm_backend_expr.cpp | 111 +++++++++++++++++++++++++++++++++++++------ src/llvm_backend_general.cpp | 3 +- src/llvm_backend_utility.cpp | 9 ++-- 4 files changed, 101 insertions(+), 23 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index c4bf2691d..7dc5f6b63 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -122,7 +122,6 @@ struct lbAddr { } swizzle_large; struct { Type *type; - i64 index; i64 bit_offset; i64 bit_size; } bitfield; diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index edd5daeca..4209ba1ea 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -4296,7 +4296,19 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { switch (bt->kind) { default: GB_PANIC("Unknown CompoundLit type: %s", type_to_string(type)); break; - case Type_BitField: + case Type_BitField: { + TEMPORARY_ALLOCATOR_GUARD(); + + // Type *backing_type = core_type(bt->BitField.backing_type); + + struct FieldData { + Type *field_type; + u64 bit_offset; + u64 bit_size; + }; + auto values = array_make(temporary_allocator(), 0, cl->elems.count); + auto fields = array_make(temporary_allocator(), 0, cl->elems.count); + for (Ast *elem : cl->elems) { ast_node(fv, FieldValue, elem); String name = fv->field->Ident.token.string; @@ -4307,26 +4319,97 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { GB_ASSERT(sel.entity != nullptr); i64 index = sel.index[0]; - i64 bit_offset = 0; - i64 bit_size = -1; - for_array(i, bt->BitField.fields) { - Entity *f = bt->BitField.fields[i]; - if (f == sel.entity) { - bit_offset = bt->BitField.bit_offsets[i]; - bit_size = bt->BitField.bit_sizes[i]; - break; - } - } + Entity *f = bt->BitField.fields[index]; + GB_ASSERT(f == sel.entity); + i64 bit_offset = bt->BitField.bit_offsets[index]; + i64 bit_size = bt->BitField.bit_sizes[index]; GB_ASSERT(bit_size > 0); Type *field_type = sel.entity->type; lbValue field_expr = lb_build_expr(p, fv->value); field_expr = lb_emit_conv(p, field_expr, field_type); + array_add(&values, field_expr); + array_add(&fields, FieldData{field_type, cast(u64)bit_offset, cast(u64)bit_size}); + } + + // NOTE(bill): inline insertion sort should be good enough, right? + for (isize i = 1; i < values.count; i++) { + for (isize j = i; + j > 0 && fields[i].bit_offset < fields[j].bit_offset; + j--) { + auto vtmp = values[j]; + values[j] = values[j-1]; + values[j-1] = vtmp; + + auto ftmp = fields[j]; + fields[j] = fields[j-1]; + fields[j-1] = ftmp; + } + } + + if (fields.count == bt->BitField.fields.count) { + Type *backing_type = core_type(bt->BitField.backing_type); + GB_ASSERT(is_type_integer(backing_type) || + (is_type_array(backing_type) && is_type_integer(backing_type->Array.elem))); + + // NOTE(bill): all fields are present + // this means no masking is necessary since on write, the bits will be overridden + + lbValue dst_byte_ptr = lb_emit_conv(p, v.addr, t_u8_ptr); + u64 total_bit_size = cast(u64)(8*type_size_of(bt)); + + if (is_type_integer(backing_type)) { + LLVMTypeRef lbt = lb_type(p->module, backing_type); + + LLVMValueRef res = LLVMConstInt(lbt, 0, false); + + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef mask = LLVMConstInt(lbt, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(lbt, f.bit_size, false)); + mask = LLVMConstSub(mask, LLVMConstInt(lbt, 1, false)); - lbAddr field_addr = lb_addr_bit_field(v.addr, field_type, index, bit_offset, bit_size); - lb_addr_store(p, field_addr, field_expr); + LLVMValueRef elem = values[i].value; + elem = LLVMBuildZExt(p->builder, elem, lbt, ""); + elem = LLVMBuildAnd(p->builder, elem, mask, ""); + + elem = LLVMBuildShl(p->builder, elem, LLVMConstInt(lbt, f.bit_offset, false), ""); + + res = LLVMBuildOr(p->builder, res, elem, ""); + } + LLVMBuildStore(p->builder, res, v.addr.value); + } else { + for_array(i, fields) { + auto const &f = fields[i]; + + if ((f.bit_offset & 7) == 0) { + u64 unpacked_bit_size = cast(u64)(8*type_size_of(f.field_type)); + u64 byte_size = (f.bit_size+7)/8; + + if (f.bit_offset + unpacked_bit_size <= total_bit_size) { + byte_size = unpacked_bit_size/8; + } + lbValue dst = lb_emit_ptr_offset(p, dst_byte_ptr, lb_const_int(p->module, t_int, f.bit_offset/8)); + lbValue src = lb_address_from_load_or_generate_local(p, values[i]); + lb_mem_copy_non_overlapping(p, dst, src, lb_const_int(p->module, t_uintptr, byte_size)); + } else { + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } + } + } + } else { + // individual storing + for_array(i, values) { + auto const &f = fields[i]; + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } } + return v; + } case Type_Struct: { // TODO(bill): "constant" '#raw_union's are not initialized constantly at the moment. @@ -4771,7 +4854,7 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { u8 bit_size = bf_type->BitField.bit_sizes[index]; i64 bit_offset = bf_type->BitField.bit_offsets[index]; - return lb_addr_bit_field(ptr, f->type, index, bit_offset, bit_size); + return lb_addr_bit_field(ptr, f->type, bit_offset, bit_size); } { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index b8fbd231e..bf23417c6 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -450,14 +450,13 @@ gb_internal lbAddr lb_addr_swizzle_large(lbValue addr, Type *array_type, Slicemodule, type); LLVMTypeKind kind = LLVMGetTypeKind(llvm_type); - + i64 sz = type_size_of(type); switch (kind) { case LLVMStructTypeKind: case LLVMArrayTypeKind: - { - // NOTE(bill): Enforce zeroing through memset to make sure padding is zeroed too - i32 sz = cast(i32)type_size_of(type); - lb_mem_zero_ptr_internal(p, ptr, lb_const_int(p->module, t_int, sz).value, alignment, false); - } + // NOTE(bill): Enforce zeroing through memset to make sure padding is zeroed too + lb_mem_zero_ptr_internal(p, ptr, lb_const_int(p->module, t_int, sz).value, alignment, false); break; default: LLVMBuildStore(p->builder, LLVMConstNull(lb_type(p->module, type)), ptr); -- cgit v1.2.3 From 214537b4209211f9ceb9932b8d9980ea6503e8ea Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Apr 2024 17:01:09 +0100 Subject: Improve codegen for `bit_field [N]T` compound literals --- base/runtime/internal.odin | 10 ++--- src/llvm_backend_expr.cpp | 91 ++++++++++++++++++++++++++++++++++++++++---- src/llvm_backend_general.cpp | 8 ++-- 3 files changed, 91 insertions(+), 18 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/base/runtime/internal.odin b/base/runtime/internal.odin index 6ca61c721..6f0445787 100644 --- a/base/runtime/internal.odin +++ b/base/runtime/internal.odin @@ -1042,19 +1042,17 @@ fixdfti :: proc(a: u64) -> i128 { __write_bits :: proc "contextless" (dst, src: [^]byte, offset: uintptr, size: uintptr) { for i in 0..>3]) & (1<<(i&7)) != 0) b := the_bit<<(j&7) - dst[j/8] &~= b - dst[j/8] |= b + dst[j>>3] = (dst[j>>3] &~ b) | b } } __read_bits :: proc "contextless" (dst, src: [^]byte, offset: uintptr, size: uintptr) { for j in 0..>3]) & (1<<(i&7)) != 0) b := the_bit<<(j&7) - dst[j/8] &~= b - dst[j/8] |= b + dst[j>>3] = (dst[j>>3] &~ b) | b } } \ No newline at end of file diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 4209ba1ea..ee1a384ae 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -4347,7 +4347,19 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { } } - if (fields.count == bt->BitField.fields.count) { + bool any_fields_different_endian = false; + for (auto const &f : fields) { + if (is_type_different_to_arch_endianness(f.field_type)) { + // NOTE(bill): Just be slow for this, to be correct + any_fields_different_endian = true; + break; + } + } + + if (!any_fields_different_endian && + fields.count == bt->BitField.fields.count) { + // SINGLE INTEGER BACKING ONLY + Type *backing_type = core_type(bt->BitField.backing_type); GB_ASSERT(is_type_integer(backing_type) || (is_type_array(backing_type) && is_type_integer(backing_type->Array.elem))); @@ -4359,27 +4371,90 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { u64 total_bit_size = cast(u64)(8*type_size_of(bt)); if (is_type_integer(backing_type)) { - LLVMTypeRef lbt = lb_type(p->module, backing_type); + LLVMTypeRef lit = lb_type(p->module, backing_type); - LLVMValueRef res = LLVMConstInt(lbt, 0, false); + LLVMValueRef res = LLVMConstInt(lit, 0, false); for (isize i = 0; i < fields.count; i++) { auto const &f = fields[i]; - LLVMValueRef mask = LLVMConstInt(lbt, 1, false); - mask = LLVMConstShl(mask, LLVMConstInt(lbt, f.bit_size, false)); - mask = LLVMConstSub(mask, LLVMConstInt(lbt, 1, false)); + LLVMValueRef mask = LLVMConstInt(lit, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(lit, f.bit_size, false)); + mask = LLVMConstSub(mask, LLVMConstInt(lit, 1, false)); LLVMValueRef elem = values[i].value; - elem = LLVMBuildZExt(p->builder, elem, lbt, ""); + elem = LLVMBuildZExt(p->builder, elem, lit, ""); elem = LLVMBuildAnd(p->builder, elem, mask, ""); - elem = LLVMBuildShl(p->builder, elem, LLVMConstInt(lbt, f.bit_offset, false), ""); + elem = LLVMBuildShl(p->builder, elem, LLVMConstInt(lit, f.bit_offset, false), ""); res = LLVMBuildOr(p->builder, res, elem, ""); } + LLVMBuildStore(p->builder, res, v.addr.value); + } else if (is_type_array(backing_type)) { + // ARRAY OF INTEGER BACKING + + i64 array_count = backing_type->Array.count; + LLVMTypeRef lit = lb_type(p->module, core_type(backing_type->Array.elem)); + gb_unused(array_count); + gb_unused(lit); + + LLVMValueRef *elems = gb_alloc_array(temporary_allocator(), LLVMValueRef, array_count); + for (i64 i = 0; i < array_count; i++) { + elems[i] = LLVMConstInt(lit, 0, false); + } + + u64 elem_bit_size = cast(u64)(8*type_size_of(backing_type->Array.elem)); + u64 curr_bit_offset = 0; + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef val = values[i].value; + LLVMTypeRef vt = lb_type(p->module, values[i].type); + for (u64 bits_to_set = f.bit_size; + bits_to_set > 0; + /**/) { + i64 elem_idx = curr_bit_offset/elem_bit_size; + u64 elem_bit_offset = curr_bit_offset%elem_bit_size; + + u64 mask_width = gb_min(bits_to_set, elem_bit_size-elem_bit_offset); + GB_ASSERT(mask_width > 0); + bits_to_set -= mask_width; + + LLVMValueRef mask = LLVMConstInt(vt, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(vt, mask_width, false)); + mask = LLVMConstSub(mask, LLVMConstInt(vt, 1, false)); + + LLVMValueRef to_set = LLVMBuildAnd(p->builder, val, mask, ""); + + if (elem_bit_offset != 0) { + to_set = LLVMBuildShl(p->builder, to_set, LLVMConstInt(vt, elem_bit_offset, false), ""); + } + to_set = LLVMBuildTrunc(p->builder, to_set, lit, ""); + + if (LLVMIsNull(elems[elem_idx])) { + elems[elem_idx] = to_set; // don't even bother doing `0 | to_set` + } else { + elems[elem_idx] = LLVMBuildOr(p->builder, elems[elem_idx], to_set, ""); + } + + if (mask_width != 0) { + val = LLVMBuildLShr(p->builder, val, LLVMConstInt(vt, mask_width, false), ""); + } + curr_bit_offset += mask_width; + } + + GB_ASSERT(curr_bit_offset == f.bit_offset + f.bit_size); + } + + for (i64 i = 0; i < array_count; i++) { + LLVMValueRef elem_ptr = LLVMBuildStructGEP2(p->builder, lb_type(p->module, backing_type), v.addr.value, cast(unsigned)i, ""); + LLVMBuildStore(p->builder, elems[i], elem_ptr); + } } else { + // SLOW STORAGE + for_array(i, fields) { auto const &f = fields[i]; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index bf23417c6..494af9056 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -775,8 +775,8 @@ gb_internal void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { lbValue dst = addr.addr; lbValue src = lb_address_from_load_or_generate_local(p, value); - if ((addr.bitfield.bit_offset & 7) == 0 && - (addr.bitfield.bit_size & 7) == 0) { + if ((addr.bitfield.bit_offset % 8) == 0 && + (addr.bitfield.bit_size % 8) == 0) { lbValue byte_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset/8); lbValue byte_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size/8); lbValue dst_offset = lb_emit_conv(p, dst, t_u8_ptr); @@ -1108,7 +1108,7 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { i64 total_bitfield_bit_size = 8*type_size_of(lb_addr_type(addr)); i64 dst_byte_size = type_size_of(addr.bitfield.type); - lbAddr dst = lb_add_local_generated(p, addr.bitfield.type, false); + lbAddr dst = lb_add_local_generated(p, addr.bitfield.type, true); lbValue src = addr.addr; lbValue bit_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); @@ -1118,7 +1118,7 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(type_size_of(addr.bitfield.type) >= ((addr.bitfield.bit_size+7)/8)); - if ((addr.bitfield.bit_offset & 7) == 0) { + if ((addr.bitfield.bit_offset % 8) == 0) { lbValue copy_size = byte_size; lbValue src_offset = lb_emit_conv(p, src, t_u8_ptr); src_offset = lb_emit_ptr_offset(p, src_offset, byte_offset); -- cgit v1.2.3 From 3b53c99576de5cb1e294f316bf5ce7c95d1cd51a Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Apr 2024 20:55:18 +0100 Subject: Improve support for big-endian `bit_field`s --- src/llvm_backend_general.cpp | 52 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 11 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 494af9056..02afa628c 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -773,16 +773,33 @@ gb_internal void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { if (addr.kind == lbAddr_BitField) { lbValue dst = addr.addr; - lbValue src = lb_address_from_load_or_generate_local(p, value); + if (is_type_endian_big(addr.bitfield.type)) { + i64 shift_amount = 8*type_size_of(value.type) - addr.bitfield.bit_size; + lbValue shifted_value = value; + shifted_value.value = LLVMBuildLShr(p->builder, + shifted_value.value, + LLVMConstInt(LLVMTypeOf(shifted_value.value), shift_amount, false), ""); + + lbValue src = lb_address_from_load_or_generate_local(p, shifted_value); + + auto args = array_make(temporary_allocator(), 4); + args[0] = dst; + args[1] = src; + args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lb_emit_runtime_call(p, "__write_bits", args); + } else if ((addr.bitfield.bit_offset % 8) == 0 && + (addr.bitfield.bit_size % 8) == 0) { + lbValue src = lb_address_from_load_or_generate_local(p, value); - if ((addr.bitfield.bit_offset % 8) == 0 && - (addr.bitfield.bit_size % 8) == 0) { lbValue byte_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset/8); lbValue byte_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size/8); lbValue dst_offset = lb_emit_conv(p, dst, t_u8_ptr); dst_offset = lb_emit_ptr_offset(p, dst_offset, byte_offset); lb_mem_copy_non_overlapping(p, dst_offset, src, byte_size); } else { + lbValue src = lb_address_from_load_or_generate_local(p, value); + auto args = array_make(temporary_allocator(), 4); args[0] = dst; args[1] = src; @@ -1118,7 +1135,23 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(type_size_of(addr.bitfield.type) >= ((addr.bitfield.bit_size+7)/8)); - if ((addr.bitfield.bit_offset % 8) == 0) { + lbValue r = {}; + if (is_type_endian_big(addr.bitfield.type)) { + auto args = array_make(temporary_allocator(), 4); + args[0] = dst.addr; + args[1] = src; + args[2] = bit_offset; + args[3] = bit_size; + lb_emit_runtime_call(p, "__read_bits", args); + + LLVMValueRef shift_amount = LLVMConstInt( + lb_type(p->module, lb_addr_type(dst)), + 8*dst_byte_size - addr.bitfield.bit_size, + false + ); + r = lb_addr_load(p, dst); + r.value = LLVMBuildShl(p->builder, r.value, shift_amount, ""); + } else if ((addr.bitfield.bit_offset % 8) == 0) { lbValue copy_size = byte_size; lbValue src_offset = lb_emit_conv(p, src, t_u8_ptr); src_offset = lb_emit_ptr_offset(p, src_offset, byte_offset); @@ -1127,6 +1160,7 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { copy_size = lb_const_int(p->module, t_uintptr, dst_byte_size); } lb_mem_copy_non_overlapping(p, dst.addr, src_offset, copy_size, false); + r = lb_addr_load(p, dst); } else { auto args = array_make(temporary_allocator(), 4); args[0] = dst.addr; @@ -1134,20 +1168,16 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { args[2] = bit_offset; args[3] = bit_size; lb_emit_runtime_call(p, "__read_bits", args); + r = lb_addr_load(p, dst); } - lbValue r = lb_addr_load(p, dst); Type *t = addr.bitfield.type; if (do_mask) { GB_ASSERT(addr.bitfield.bit_size < 8*type_size_of(ct)); - LLVMTypeRef lt = lb_type(p->module, t); - LLVMValueRef mask = LLVMConstInt(lt, 1, false); - mask = LLVMConstShl(mask, LLVMConstInt(lt, addr.bitfield.bit_size, false)); - mask = LLVMConstSub(mask, LLVMConstInt(lt, 1, false)); - lbValue m = {mask, t}; - r = lb_emit_arith(p, Token_And, r, m, t); + lbValue mask = lb_const_int(p->module, t, (1ull<