diff options
Diffstat (limited to 'src/llvm_backend_expr.cpp')
| -rw-r--r-- | src/llvm_backend_expr.cpp | 1089 |
1 files changed, 854 insertions, 235 deletions
diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 8678a125c..df9dca801 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -130,12 +130,17 @@ gb_internal lbValue lb_emit_unary_arith(lbProcedure *p, TokenKind op, lbValue x, LLVMTypeRef vector_type = nullptr; if (op != Token_Not && lb_try_vector_cast(p->module, val, &vector_type)) { LLVMValueRef vp = LLVMBuildPointerCast(p->builder, val.value, LLVMPointerType(vector_type, 0), ""); - LLVMValueRef v = LLVMBuildLoad2(p->builder, vector_type, vp, ""); + LLVMValueRef v = OdinLLVMBuildLoad(p, vector_type, vp); LLVMValueRef opv = nullptr; switch (op) { case Token_Xor: opv = LLVMBuildNot(p->builder, v, ""); + if (is_type_bit_set(elem_type)) { + ExactValue ev_mask = exact_bit_set_all_set_mask(elem_type); + lbValue mask = lb_const_value(p->module, elem_type, ev_mask); + opv = LLVMBuildAnd(p->builder, opv, mask.value, ""); + } break; case Token_Sub: if (is_type_float(elem_type)) { @@ -176,8 +181,14 @@ gb_internal lbValue lb_emit_unary_arith(lbProcedure *p, TokenKind op, lbValue x, if (op == Token_Xor) { lbValue cmp = {}; - cmp.value = LLVMBuildNot(p->builder, x.value, ""); cmp.type = x.type; + if (is_type_bit_set(x.type)) { + ExactValue ev_mask = exact_bit_set_all_set_mask(x.type); + lbValue mask = lb_const_value(p->module, x.type, ev_mask); + cmp.value = LLVMBuildXor(p->builder, x.value, mask.value, ""); + } else { + cmp.value = LLVMBuildNot(p->builder, x.value, ""); + } return lb_emit_conv(p, cmp, type); } @@ -296,12 +307,6 @@ gb_internal bool lb_try_direct_vector_arith(lbProcedure *p, TokenKind op, lbValu GB_ASSERT(vector_type0 == vector_type1); LLVMTypeRef vector_type = vector_type0; - LLVMValueRef lhs_vp = LLVMBuildPointerCast(p->builder, lhs_ptr.value, LLVMPointerType(vector_type, 0), ""); - LLVMValueRef rhs_vp = LLVMBuildPointerCast(p->builder, rhs_ptr.value, LLVMPointerType(vector_type, 0), ""); - LLVMValueRef x = LLVMBuildLoad2(p->builder, vector_type, lhs_vp, ""); - LLVMValueRef y = LLVMBuildLoad2(p->builder, vector_type, rhs_vp, ""); - LLVMValueRef z = nullptr; - Type *integral_type = base_type(elem_type); if (is_type_simd_vector(integral_type)) { integral_type = core_array_type(integral_type); @@ -311,8 +316,18 @@ gb_internal bool lb_try_direct_vector_arith(lbProcedure *p, TokenKind op, lbValu case Token_Add: op = Token_Or; break; case Token_Sub: op = Token_AndNot; break; } + Type *u = bit_set_to_int(type); + if (is_type_array(u)) { + return false; + } } + LLVMValueRef lhs_vp = LLVMBuildPointerCast(p->builder, lhs_ptr.value, LLVMPointerType(vector_type, 0), ""); + LLVMValueRef rhs_vp = LLVMBuildPointerCast(p->builder, rhs_ptr.value, LLVMPointerType(vector_type, 0), ""); + LLVMValueRef x = OdinLLVMBuildLoad(p, vector_type, lhs_vp); + LLVMValueRef y = OdinLLVMBuildLoad(p, vector_type, rhs_vp); + LLVMValueRef z = nullptr; + if (is_type_float(integral_type)) { switch (op) { case Token_Add: @@ -504,6 +519,10 @@ gb_internal bool lb_is_matrix_simdable(Type *t) { if ((mt->Matrix.row_count & 1) ^ (mt->Matrix.column_count & 1)) { return false; } + if (mt->Matrix.is_row_major) { + // TODO(bill): make #row_major matrices work with SIMD + return false; + } if (elem->kind == Type_Basic) { switch (elem->Basic.kind) { @@ -532,15 +551,14 @@ gb_internal LLVMValueRef lb_matrix_to_vector(lbProcedure *p, lbValue matrix) { Type *mt = base_type(matrix.type); GB_ASSERT(mt->kind == Type_Matrix); LLVMTypeRef elem_type = lb_type(p->module, mt->Matrix.elem); - + unsigned total_count = cast(unsigned)matrix_type_total_internal_elems(mt); LLVMTypeRef total_matrix_type = LLVMVectorType(elem_type, total_count); - + #if 1 LLVMValueRef ptr = lb_address_from_load_or_generate_local(p, matrix).value; LLVMValueRef matrix_vector_ptr = LLVMBuildPointerCast(p->builder, ptr, LLVMPointerType(total_matrix_type, 0), ""); - LLVMValueRef matrix_vector = LLVMBuildLoad2(p->builder, total_matrix_type, matrix_vector_ptr, ""); - LLVMSetAlignment(matrix_vector, cast(unsigned)type_align_of(mt)); + LLVMValueRef matrix_vector = OdinLLVMBuildLoadAligned(p, total_matrix_type, matrix_vector_ptr, type_align_of(mt)); return matrix_vector; #else LLVMValueRef matrix_vector = LLVMBuildBitCast(p->builder, matrix.value, total_matrix_type, ""); @@ -684,39 +702,39 @@ gb_internal lbValue lb_emit_matrix_flatten(lbProcedure *p, lbValue m, Type *type Type *mt = base_type(m.type); GB_ASSERT(mt->kind == Type_Matrix); - // TODO(bill): Determine why this fails on Windows sometimes - if (false && lb_is_matrix_simdable(mt)) { - LLVMValueRef vector = lb_matrix_to_trimmed_vector(p, m); - return lb_matrix_cast_vector_to_type(p, vector, type); - } - lbAddr res = lb_add_local_generated(p, type, true); - i64 row_count = mt->Matrix.row_count; - i64 column_count = mt->Matrix.column_count; - TEMPORARY_ALLOCATOR_GUARD(); - - auto srcs = array_make<lbValue>(temporary_allocator(), 0, row_count*column_count); - auto dsts = array_make<lbValue>(temporary_allocator(), 0, row_count*column_count); - - for (i64 j = 0; j < column_count; j++) { - for (i64 i = 0; i < row_count; i++) { - lbValue src = lb_emit_matrix_ev(p, m, i, j); - array_add(&srcs, src); - } - } - - for (i64 j = 0; j < column_count; j++) { - for (i64 i = 0; i < row_count; i++) { - lbValue dst = lb_emit_array_epi(p, res.addr, i + j*row_count); - array_add(&dsts, dst); - } - } - - GB_ASSERT(srcs.count == dsts.count); - for_array(i, srcs) { - lb_emit_store(p, dsts[i], srcs[i]); - } + GB_ASSERT(type_size_of(type) == type_size_of(m.type)); + + lbValue m_ptr = lb_address_from_load_or_generate_local(p, m); + lbValue n = lb_const_int(p->module, t_int, type_size_of(type)); + lb_mem_copy_non_overlapping(p, res.addr, m_ptr, n); + + // i64 row_count = mt->Matrix.row_count; + // i64 column_count = mt->Matrix.column_count; + // TEMPORARY_ALLOCATOR_GUARD(); + + // auto srcs = array_make<lbValue>(temporary_allocator(), 0, row_count*column_count); + // auto dsts = array_make<lbValue>(temporary_allocator(), 0, row_count*column_count); + + // for (i64 j = 0; j < column_count; j++) { + // for (i64 i = 0; i < row_count; i++) { + // lbValue src = lb_emit_matrix_ev(p, m, i, j); + // array_add(&srcs, src); + // } + // } + + // for (i64 j = 0; j < column_count; j++) { + // for (i64 i = 0; i < row_count; i++) { + // lbValue dst = lb_emit_array_epi(p, res.addr, i + j*row_count); + // array_add(&dsts, dst); + // } + // } + + // GB_ASSERT(srcs.count == dsts.count); + // for_array(i, srcs) { + // lb_emit_store(p, dsts[i], srcs[i]); + // } return lb_addr_load(p, res); } @@ -763,6 +781,7 @@ gb_internal lbValue lb_emit_matrix_mul(lbProcedure *p, lbValue lhs, lbValue rhs, GB_ASSERT(is_type_matrix(yt)); GB_ASSERT(xt->Matrix.column_count == yt->Matrix.row_count); GB_ASSERT(are_types_identical(xt->Matrix.elem, yt->Matrix.elem)); + GB_ASSERT(xt->Matrix.is_row_major == yt->Matrix.is_row_major); Type *elem = xt->Matrix.elem; @@ -770,7 +789,7 @@ gb_internal lbValue lb_emit_matrix_mul(lbProcedure *p, lbValue lhs, lbValue rhs, unsigned inner = cast(unsigned)xt->Matrix.column_count; unsigned outer_columns = cast(unsigned)yt->Matrix.column_count; - if (lb_is_matrix_simdable(xt)) { + if (!xt->Matrix.is_row_major && lb_is_matrix_simdable(xt)) { unsigned x_stride = cast(unsigned)matrix_type_stride_in_elems(xt); unsigned y_stride = cast(unsigned)matrix_type_stride_in_elems(yt); @@ -812,7 +831,7 @@ gb_internal lbValue lb_emit_matrix_mul(lbProcedure *p, lbValue lhs, lbValue rhs, return lb_addr_load(p, res); } - { + if (!xt->Matrix.is_row_major) { lbAddr res = lb_add_local_generated(p, type, true); auto inners = slice_make<lbValue[2]>(permanent_allocator(), inner); @@ -836,6 +855,30 @@ gb_internal lbValue lb_emit_matrix_mul(lbProcedure *p, lbValue lhs, lbValue rhs, } return lb_addr_load(p, res); + } else { + lbAddr res = lb_add_local_generated(p, type, true); + + auto inners = slice_make<lbValue[2]>(permanent_allocator(), inner); + + for (unsigned i = 0; i < outer_rows; i++) { + for (unsigned j = 0; j < outer_columns; j++) { + lbValue dst = lb_emit_matrix_epi(p, res.addr, i, j); + for (unsigned k = 0; k < inner; k++) { + inners[k][0] = lb_emit_matrix_ev(p, lhs, i, k); + inners[k][1] = lb_emit_matrix_ev(p, rhs, k, j); + } + + lbValue sum = lb_const_nil(p->module, elem); + for (unsigned k = 0; k < inner; k++) { + lbValue a = inners[k][0]; + lbValue b = inners[k][1]; + sum = lb_emit_mul_add(p, a, b, sum, elem); + } + lb_emit_store(p, dst, sum); + } + } + + return lb_addr_load(p, res); } } @@ -855,7 +898,7 @@ gb_internal lbValue lb_emit_matrix_mul_vector(lbProcedure *p, lbValue lhs, lbVal Type *elem = mt->Matrix.elem; - if (lb_is_matrix_simdable(mt)) { + if (!mt->Matrix.is_row_major && lb_is_matrix_simdable(mt)) { unsigned stride = cast(unsigned)matrix_type_stride_in_elems(mt); unsigned row_count = cast(unsigned)mt->Matrix.row_count; @@ -924,7 +967,7 @@ gb_internal lbValue lb_emit_vector_mul_matrix(lbProcedure *p, lbValue lhs, lbVal Type *elem = mt->Matrix.elem; - if (lb_is_matrix_simdable(mt)) { + if (!mt->Matrix.is_row_major && lb_is_matrix_simdable(mt)) { unsigned stride = cast(unsigned)matrix_type_stride_in_elems(mt); unsigned row_count = cast(unsigned)mt->Matrix.row_count; @@ -1104,12 +1147,21 @@ gb_internal lbValue lb_emit_arith(lbProcedure *p, TokenKind op, lbValue lhs, lbV switch (op) { case Token_Add: - real = lb_emit_arith(p, Token_Add, a, c, ft); - imag = lb_emit_arith(p, Token_Add, b, d, ft); - break; case Token_Sub: - real = lb_emit_arith(p, Token_Sub, a, c, ft); - imag = lb_emit_arith(p, Token_Sub, b, d, ft); + if (type_size_of(ft) == 2) { + a = lb_emit_conv(p, a, t_f32); + b = lb_emit_conv(p, b, t_f32); + c = lb_emit_conv(p, c, t_f32); + d = lb_emit_conv(p, d, t_f32); + real = lb_emit_arith(p, op, a, c, t_f32); + imag = lb_emit_arith(p, op, b, d, t_f32); + + real = lb_emit_conv(p, real, ft); + imag = lb_emit_conv(p, imag, ft); + } else { + real = lb_emit_arith(p, op, a, c, ft); + imag = lb_emit_arith(p, op, b, d, ft); + } break; case Token_Mul: { lbValue x = lb_emit_arith(p, Token_Mul, a, c, ft); @@ -1133,6 +1185,11 @@ gb_internal lbValue lb_emit_arith(lbProcedure *p, TokenKind op, lbValue lhs, lbV Type *ft = base_complex_elem_type(type); if (op == Token_Add || op == Token_Sub) { + Type *immediate_type = ft; + if (type_size_of(ft) == 2) { + immediate_type = t_f32; + } + lbAddr res = lb_add_local_generated(p, type, false); // NOTE: initialized in full later lbValue x0 = lb_emit_struct_ev(p, lhs, 0); lbValue x1 = lb_emit_struct_ev(p, lhs, 1); @@ -1144,15 +1201,39 @@ gb_internal lbValue lb_emit_arith(lbProcedure *p, TokenKind op, lbValue lhs, lbV lbValue y2 = lb_emit_struct_ev(p, rhs, 2); lbValue y3 = lb_emit_struct_ev(p, rhs, 3); - lbValue z0 = lb_emit_arith(p, op, x0, y0, ft); - lbValue z1 = lb_emit_arith(p, op, x1, y1, ft); - lbValue z2 = lb_emit_arith(p, op, x2, y2, ft); - lbValue z3 = lb_emit_arith(p, op, x3, y3, ft); + if (immediate_type != ft) { + x0 = lb_emit_conv(p, x0, immediate_type); + x1 = lb_emit_conv(p, x1, immediate_type); + x2 = lb_emit_conv(p, x2, immediate_type); + x3 = lb_emit_conv(p, x3, immediate_type); + + y0 = lb_emit_conv(p, y0, immediate_type); + y1 = lb_emit_conv(p, y1, immediate_type); + y2 = lb_emit_conv(p, y2, immediate_type); + y3 = lb_emit_conv(p, y3, immediate_type); + } + + lbValue z0 = lb_emit_arith(p, op, x0, y0, immediate_type); + lbValue z1 = lb_emit_arith(p, op, x1, y1, immediate_type); + lbValue z2 = lb_emit_arith(p, op, x2, y2, immediate_type); + lbValue z3 = lb_emit_arith(p, op, x3, y3, immediate_type); + + lbValue d0 = lb_emit_struct_ep(p, res.addr, 0); + lbValue d1 = lb_emit_struct_ep(p, res.addr, 1); + lbValue d2 = lb_emit_struct_ep(p, res.addr, 2); + lbValue d3 = lb_emit_struct_ep(p, res.addr, 3); + + if (immediate_type != ft) { + z0 = lb_emit_conv(p, z0, ft); + z1 = lb_emit_conv(p, z1, ft); + z2 = lb_emit_conv(p, z2, ft); + z3 = lb_emit_conv(p, z3, ft); + } - lb_emit_store(p, lb_emit_struct_ep(p, res.addr, 0), z0); - lb_emit_store(p, lb_emit_struct_ep(p, res.addr, 1), z1); - lb_emit_store(p, lb_emit_struct_ep(p, res.addr, 2), z2); - lb_emit_store(p, lb_emit_struct_ep(p, res.addr, 3), z3); + lb_emit_store(p, d0, z0); + lb_emit_store(p, d1, z1); + lb_emit_store(p, d2, z2); + lb_emit_store(p, d3, z3); return lb_addr_load(p, res); } else if (op == Token_Mul) { @@ -1225,6 +1306,14 @@ handle_op:; case Token_Add: op = Token_Or; break; case Token_Sub: op = Token_AndNot; break; } + Type *u = bit_set_to_int(type); + if (is_type_array(u)) { + lhs.type = u; + rhs.type = u; + res = lb_emit_arith(p, op, lhs, rhs, u); + res.type = type; + return res; + } } Type *integral_type = type; @@ -1354,6 +1443,58 @@ gb_internal bool lb_is_empty_string_constant(Ast *expr) { return false; } +gb_internal lbValue lb_build_binary_in(lbProcedure *p, lbValue left, lbValue right, TokenKind op) { + Type *rt = base_type(right.type); + if (is_type_pointer(rt)) { + right = lb_emit_load(p, right); + rt = base_type(type_deref(rt)); + } + + switch (rt->kind) { + case Type_Map: + { + lbValue map_ptr = lb_address_from_load_or_generate_local(p, right); + lbValue key = left; + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map_ptr, key); + if (op == Token_in) { + return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); + } else { + return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_CmpEq, ptr), t_bool); + } + } + break; + case Type_BitSet: + { + Type *key_type = rt->BitSet.elem; + GB_ASSERT(are_types_identical(left.type, key_type)); + + Type *it = bit_set_to_int(rt); + + left = lb_emit_conv(p, left, it); + if (is_type_different_to_arch_endianness(it)) { + left = lb_emit_byte_swap(p, left, integer_endian_type_to_platform_type(it)); + } + + lbValue lower = lb_const_value(p->module, left.type, exact_value_i64(rt->BitSet.lower)); + lbValue key = lb_emit_arith(p, Token_Sub, left, lower, left.type); + lbValue bit = lb_emit_arith(p, Token_Shl, lb_const_int(p->module, left.type, 1), key, left.type); + bit = lb_emit_conv(p, bit, it); + + lbValue old_value = lb_emit_transmute(p, right, it); + lbValue new_value = lb_emit_arith(p, Token_And, old_value, bit, it); + + if (op == Token_in) { + return lb_emit_conv(p, lb_emit_comp(p, Token_NotEq, new_value, lb_const_int(p->module, new_value.type, 0)), t_bool); + } else { + return lb_emit_conv(p, lb_emit_comp(p, Token_CmpEq, new_value, lb_const_int(p->module, new_value.type, 0)), t_bool); + } + } + break; + } + GB_PANIC("Invalid 'in' type"); + return {}; +} + gb_internal lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { ast_node(be, BinaryExpr, expr); @@ -1461,57 +1602,8 @@ gb_internal lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { { lbValue left = lb_build_expr(p, be->left); lbValue right = lb_build_expr(p, be->right); - Type *rt = base_type(right.type); - if (is_type_pointer(rt)) { - right = lb_emit_load(p, right); - rt = base_type(type_deref(rt)); - } - - switch (rt->kind) { - case Type_Map: - { - lbValue map_ptr = lb_address_from_load_or_generate_local(p, right); - lbValue key = left; - lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map_ptr, key); - if (be->op.kind == Token_in) { - return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); - } else { - return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_CmpEq, ptr), t_bool); - } - } - break; - case Type_BitSet: - { - Type *key_type = rt->BitSet.elem; - GB_ASSERT(are_types_identical(left.type, key_type)); - - Type *it = bit_set_to_int(rt); - left = lb_emit_conv(p, left, it); - if (is_type_different_to_arch_endianness(it)) { - left = lb_emit_byte_swap(p, left, integer_endian_type_to_platform_type(it)); - } - - lbValue lower = lb_const_value(p->module, left.type, exact_value_i64(rt->BitSet.lower)); - lbValue key = lb_emit_arith(p, Token_Sub, left, lower, left.type); - lbValue bit = lb_emit_arith(p, Token_Shl, lb_const_int(p->module, left.type, 1), key, left.type); - bit = lb_emit_conv(p, bit, it); - - lbValue old_value = lb_emit_transmute(p, right, it); - lbValue new_value = lb_emit_arith(p, Token_And, old_value, bit, it); - - if (be->op.kind == Token_in) { - return lb_emit_conv(p, lb_emit_comp(p, Token_NotEq, new_value, lb_const_int(p->module, new_value.type, 0)), t_bool); - } else { - return lb_emit_conv(p, lb_emit_comp(p, Token_CmpEq, new_value, lb_const_int(p->module, new_value.type, 0)), t_bool); - } - } - break; - default: - GB_PANIC("Invalid 'in' type"); - } - break; + return lb_build_binary_in(p, left, right, be->op.kind); } - break; default: GB_PANIC("Invalid binary expression"); break; @@ -1555,7 +1647,7 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { lb_emit_store(p, a1, id); return lb_addr_load(p, res); } else if (dst->kind == Type_Basic) { - if (src->Basic.kind == Basic_string && dst->Basic.kind == Basic_cstring) { + if (src->kind == Type_Basic && src->Basic.kind == Basic_string && dst->Basic.kind == Basic_cstring) { String str = lb_get_const_string(m, value); lbValue res = {}; res.type = t; @@ -1848,13 +1940,40 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { lbValue res_i128 = lb_emit_runtime_call(p, call, args); return lb_emit_conv(p, res_i128, t); } + i64 sz = type_size_of(src); lbValue res = {}; res.type = t; if (is_type_unsigned(dst)) { - res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t), ""); + switch (sz) { + case 2: + case 4: + res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t_u32), ""); + res.value = LLVMBuildIntCast2(p->builder, res.value, lb_type(m, t), false, ""); + break; + case 8: + res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t_u64), ""); + res.value = LLVMBuildIntCast2(p->builder, res.value, lb_type(m, t), false, ""); + break; + default: + GB_PANIC("Unhandled float type"); + break; + } } else { - res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t), ""); + switch (sz) { + case 2: + case 4: + res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t_i32), ""); + res.value = LLVMBuildIntCast2(p->builder, res.value, lb_type(m, t), true, ""); + break; + case 8: + res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t_i64), ""); + res.value = LLVMBuildIntCast2(p->builder, res.value, lb_type(m, t), true, ""); + break; + default: + GB_PANIC("Unhandled float type"); + break; + } } return res; } @@ -1925,7 +2044,11 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { } else if (is_type_integer(src_elem) && is_type_boolean(dst_elem)) { LLVMValueRef i1vector = LLVMBuildICmp(p->builder, LLVMIntNE, value.value, LLVMConstNull(LLVMTypeOf(value.value)), ""); res.value = LLVMBuildIntCast2(p->builder, i1vector, lb_type(m, t), !is_type_unsigned(src_elem), ""); - } else { + } else if (is_type_pointer(src_elem) && is_type_integer(dst_elem)) { + res.value = LLVMBuildPtrToInt(p->builder, value.value, lb_type(m, t), ""); + } else if (is_type_integer(src_elem) && is_type_pointer(dst_elem)) { + res.value = LLVMBuildIntToPtr(p->builder, value.value, lb_type(m, t), ""); + }else { GB_PANIC("Unhandled simd vector conversion: %s -> %s", type_to_string(src), type_to_string(dst)); } return res; @@ -1946,6 +2069,44 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { } } + // bit_field <-> backing type + if (is_type_bit_field(src)) { + if (are_types_identical(src->BitField.backing_type, dst)) { + lbValue res = {}; + res.type = t; + res.value = value.value; + return res; + } + } + if (is_type_bit_field(dst)) { + if (are_types_identical(src, dst->BitField.backing_type)) { + lbValue res = {}; + res.type = t; + res.value = value.value; + return res; + } + } + + // bit_set <-> backing type + if (is_type_bit_set(src)) { + Type *backing = bit_set_to_int(src); + if (are_types_identical(backing, dst)) { + lbValue res = {}; + res.type = t; + res.value = value.value; + return res; + } + } + if (is_type_bit_set(dst)) { + Type *backing = bit_set_to_int(dst); + if (are_types_identical(src, backing)) { + lbValue res = {}; + res.type = t; + res.value = value.value; + return res; + } + } + // Pointer <-> uintptr if (is_type_pointer(src) && is_type_uintptr(dst)) { @@ -1974,22 +2135,59 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { } if (is_type_union(dst)) { - for (Type *vt : dst->Union.variants) { - if (are_types_identical(vt, src_type)) { + if (dst->Union.variants.count == 1) { + Type *vt = dst->Union.variants[0]; + if (internal_check_is_assignable_to(src_type, vt)) { + value = lb_emit_conv(p, value, vt); lbAddr parent = lb_add_local_generated(p, t, true); lb_emit_store_union_variant(p, parent.addr, value, vt); return lb_addr_load(p, parent); } } - if (dst->Union.variants.count == 1) { - Type *vt = dst->Union.variants[0]; - if (internal_check_is_assignable_to(src_type, vt)) { - value = lb_emit_conv(p, value, vt); + for (Type *vt : dst->Union.variants) { + if (are_types_identical(src_type, vt)) { lbAddr parent = lb_add_local_generated(p, t, true); lb_emit_store_union_variant(p, parent.addr, value, vt); return lb_addr_load(p, parent); } } + ValidIndexAndScore *valids = gb_alloc_array(temporary_allocator(), ValidIndexAndScore, dst->Union.variants.count); + isize valid_count = 0; + isize first_success_index = -1; + for_array(i, dst->Union.variants) { + Type *vt = dst->Union.variants[i]; + i64 score = 0; + if (internal_check_is_assignable_to(src_type, vt)) { + valids[valid_count].index = i; + valids[valid_count].score = score; + valid_count += 1; + if (first_success_index < 0) { + first_success_index = i; + } + } + } + if (valid_count > 1) { + gb_sort_array(valids, valid_count, valid_index_and_score_cmp); + i64 best_score = valids[0].score; + for (isize i = 1; i < valid_count; i++) { + auto v = valids[i]; + if (best_score > v.score) { + valid_count = i; + break; + } + best_score = v.score; + } + first_success_index = valids[0].index; + } + + if (valid_count == 1) { + Type *vt = dst->Union.variants[first_success_index]; + value = lb_emit_conv(p, value, vt); + lbAddr parent = lb_add_local_generated(p, t, true); + lb_emit_store_union_variant(p, parent.addr, value, vt); + return lb_addr_load(p, parent); + } + } // NOTE(bill): This has to be done before 'Pointer <-> Pointer' as it's @@ -2099,14 +2297,45 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { if (is_type_array_like(dst)) { Type *elem = base_array_type(dst); + isize index_count = cast(isize)get_array_type_count(dst); + + isize inlineable = type_size_of(dst) <= build_context.max_simd_align; lbValue e = lb_emit_conv(p, value, elem); + if (inlineable && lb_is_const(e)) { + lbAddr v = {}; + if (e.value) { + TEMPORARY_ALLOCATOR_GUARD(); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, index_count); + for (isize i = 0; i < index_count; i++) { + values[i] = e.value; + } + lbValue array_const_value = {}; + array_const_value.type = t; + array_const_value.value = LLVMConstArray(lb_type(m, elem), values, cast(unsigned)index_count); + v = lb_add_global_generated(m, t, array_const_value); + } else { + v = lb_add_global_generated(m, t); + } + + lb_make_global_private_const(v); + return lb_addr_load(p, v); + } + // NOTE(bill): Doesn't need to be zero because it will be initialized in the loops lbAddr v = lb_add_local_generated(p, t, false); - isize index_count = cast(isize)get_array_type_count(dst); - for (isize i = 0; i < index_count; i++) { - lbValue elem = lb_emit_array_epi(p, v.addr, i); + if (!inlineable) { + auto loop_data = lb_loop_start(p, index_count, t_int); + + lbValue elem = lb_emit_array_ep(p, v.addr, loop_data.idx); lb_emit_store(p, elem, e); + + lb_loop_end(p, loop_data); + } else { + for (isize i = 0; i < index_count; i++) { + lbValue elem = lb_emit_array_epi(p, v.addr, i); + lb_emit_store(p, elem, e); + } } return lb_addr_load(p, v); } @@ -2134,12 +2363,23 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { GB_ASSERT(src->kind == Type_Matrix); lbAddr v = lb_add_local_generated(p, t, true); - if (is_matrix_square(dst) && is_matrix_square(dst)) { + if (dst->Matrix.row_count == src->Matrix.row_count && + dst->Matrix.column_count == src->Matrix.column_count) { + for (i64 j = 0; j < dst->Matrix.column_count; j++) { + for (i64 i = 0; i < dst->Matrix.row_count; i++) { + lbValue d = lb_emit_matrix_epi(p, v.addr, i, j); + lbValue s = lb_emit_matrix_ev(p, value, i, j); + s = lb_emit_conv(p, s, dst->Matrix.elem); + lb_emit_store(p, d, s); + } + } + } else if (is_matrix_square(dst) && is_matrix_square(dst)) { for (i64 j = 0; j < dst->Matrix.column_count; j++) { for (i64 i = 0; i < dst->Matrix.row_count; i++) { if (i < src->Matrix.row_count && j < src->Matrix.column_count) { lbValue d = lb_emit_matrix_epi(p, v.addr, i, j); lbValue s = lb_emit_matrix_ev(p, value, i, j); + s = lb_emit_conv(p, s, dst->Matrix.elem); lb_emit_store(p, d, s); } else if (i == j) { lbValue d = lb_emit_matrix_epi(p, v.addr, i, j); @@ -2256,6 +2496,17 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { return {}; } +gb_internal lbValue lb_emit_c_vararg(lbProcedure *p, lbValue arg, Type *type) { + Type *core = core_type(type); + if (core->kind == Type_BitSet) { + core = core_type(bit_set_to_int(core)); + arg = lb_emit_transmute(p, arg, core); + } + + Type *promoted = c_vararg_promote_type(core); + return lb_emit_conv(p, arg, promoted); +} + gb_internal lbValue lb_compare_records(lbProcedure *p, TokenKind op_kind, lbValue left, lbValue right, Type *type) { GB_ASSERT((is_type_struct(type) || is_type_union(type)) && is_type_comparable(type)); lbValue left_ptr = lb_address_from_load_or_generate_local(p, left); @@ -2314,9 +2565,26 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left if (are_types_identical(a, b)) { // NOTE(bill): No need for a conversion - } else if (lb_is_const(left) || lb_is_const_nil(left)) { + } else if ((lb_is_const(left) && !is_type_array(left.type)) || lb_is_const_nil(left)) { + // NOTE(karl): !is_type_array(left.type) is there to avoid lb_emit_conv + // trying to convert a constant array into a non-array. In that case we + // want the `else` branch to happen, so it can try to convert the + // non-array into an array instead. + + if (lb_is_const_nil(left)) { + if (internal_check_is_assignable_to(right.type, left.type)) { + right = lb_emit_conv(p, right, left.type); + } + return lb_emit_comp_against_nil(p, op_kind, right); + } left = lb_emit_conv(p, left, right.type); - } else if (lb_is_const(right) || lb_is_const_nil(right)) { + } else if ((lb_is_const(right) && !is_type_array(right.type)) || lb_is_const_nil(right)) { + if (lb_is_const_nil(right)) { + if (internal_check_is_assignable_to(left.type, right.type)) { + left = lb_emit_conv(p, left, right.type); + } + return lb_emit_comp_against_nil(p, op_kind, left); + } right = lb_emit_conv(p, right, left.type); } else { Type *lt = left.type; @@ -2444,7 +2712,7 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left case Token_Lt: runtime_procedure = "cstring_lt"; break; case Token_Gt: runtime_procedure = "cstring_gt"; break; case Token_LtEq: runtime_procedure = "cstring_le"; break; - case Token_GtEq: runtime_procedure = "cstring_gt"; break; + case Token_GtEq: runtime_procedure = "cstring_ge"; break; } GB_ASSERT(runtime_procedure != nullptr); @@ -2467,7 +2735,7 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left case Token_Lt: runtime_procedure = "string_lt"; break; case Token_Gt: runtime_procedure = "string_gt"; break; case Token_LtEq: runtime_procedure = "string_le"; break; - case Token_GtEq: runtime_procedure = "string_gt"; break; + case Token_GtEq: runtime_procedure = "string_ge"; break; } GB_ASSERT(runtime_procedure != nullptr); @@ -2549,6 +2817,12 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left Type *it = bit_set_to_int(a); lbValue lhs = lb_emit_transmute(p, left, it); lbValue rhs = lb_emit_transmute(p, right, it); + if (is_type_different_to_arch_endianness(it)) { + it = integer_endian_type_to_platform_type(it); + lhs = lb_emit_byte_swap(p, lhs, it); + rhs = lb_emit_byte_swap(p, rhs, it); + } + lbValue res = lb_emit_arith(p, Token_And, lhs, rhs, it); if (op_kind == Token_Lt || op_kind == Token_LtEq) { @@ -2646,6 +2920,12 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left } } + if (is_type_different_to_arch_endianness(left.type)) { + Type *pt = integer_endian_type_to_platform_type(left.type); + lhs = lb_emit_byte_swap(p, {lhs, pt}, pt).value; + rhs = lb_emit_byte_swap(p, {rhs, pt}, pt).value; + } + res.value = LLVMBuildICmp(p->builder, pred, lhs, rhs, ""); } else if (is_type_float(a)) { LLVMRealPredicate pred = {}; @@ -2657,6 +2937,13 @@ gb_internal lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left case Token_LtEq: pred = LLVMRealOLE; break; case Token_NotEq: pred = LLVMRealONE; break; } + + if (is_type_different_to_arch_endianness(left.type)) { + Type *pt = integer_endian_type_to_platform_type(left.type); + left = lb_emit_byte_swap(p, left, pt); + right = lb_emit_byte_swap(p, right, pt); + } + res.value = LLVMBuildFCmp(p->builder, pred, left.value, right.value, ""); } else if (is_type_typeid(a)) { LLVMIntPredicate pred = {}; @@ -2775,13 +3062,32 @@ gb_internal lbValue lb_emit_comp_against_nil(lbProcedure *p, TokenKind op_kind, case Type_Pointer: case Type_MultiPointer: case Type_Proc: - case Type_BitSet: if (op_kind == Token_CmpEq) { res.value = LLVMBuildIsNull(p->builder, x.value, ""); } else if (op_kind == Token_NotEq) { res.value = LLVMBuildIsNotNull(p->builder, x.value, ""); } return res; + case Type_BitSet: + { + Type *u = bit_set_to_int(bt); + if (is_type_array(u)) { + auto args = array_make<lbValue>(permanent_allocator(), 2); + lbValue lhs = lb_address_from_load_or_generate_local(p, x); + args[0] = lb_emit_conv(p, lhs, t_rawptr); + args[1] = lb_const_int(p->module, t_int, type_size_of(t)); + lbValue val = lb_emit_runtime_call(p, "memory_compare_zero", args); + lbValue res = lb_emit_comp(p, op_kind, val, lb_const_int(p->module, t_int, 0)); + return res; + } else { + if (op_kind == Token_CmpEq) { + res.value = LLVMBuildIsNull(p->builder, x.value, ""); + } else if (op_kind == Token_NotEq) { + res.value = LLVMBuildIsNotNull(p->builder, x.value, ""); + } + } + return res; + } case Type_Slice: { @@ -2890,15 +3196,6 @@ gb_internal lbValue lb_emit_comp_against_nil(lbProcedure *p, TokenKind op_kind, return {}; } -gb_internal lbValue lb_make_soa_pointer(lbProcedure *p, Type *type, lbValue const &addr, lbValue const &index) { - lbAddr v = lb_add_local_generated(p, type, false); - lbValue ptr = lb_emit_struct_ep(p, v.addr, 0); - lbValue idx = lb_emit_struct_ep(p, v.addr, 1); - lb_emit_store(p, ptr, addr); - lb_emit_store(p, idx, lb_emit_conv(p, index, t_int)); - - return lb_addr_load(p, v); -} gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { ast_node(ue, UnaryExpr, expr); @@ -2939,6 +3236,12 @@ gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { } else if (is_type_soa_pointer(tv.type)) { ast_node(ie, IndexExpr, ue_expr); lbValue addr = lb_build_addr_ptr(p, ie->expr); + + if (is_type_pointer(type_deref(addr.type))) { + addr = lb_emit_load(p, addr); + } + GB_ASSERT(is_type_pointer(addr.type)); + lbValue index = lb_build_expr(p, ie->index); if (!build_context.no_bounds_check) { @@ -3040,7 +3343,7 @@ gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { Type *dst_type = type; - if ((p->state_flags & StateFlag_no_type_assert) == 0) { + if (!build_context.no_type_assert && (p->state_flags & StateFlag_no_type_assert) == 0) { lbValue src_tag = {}; lbValue dst_tag = {}; if (is_type_union_maybe_pointer(src_type)) { @@ -3080,7 +3383,7 @@ gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { v = lb_emit_load(p, v); } lbValue data_ptr = lb_emit_struct_ev(p, v, 0); - if ((p->state_flags & StateFlag_no_type_assert) == 0) { + if (!build_context.no_type_assert && (p->state_flags & StateFlag_no_type_assert) == 0) { GB_ASSERT(!build_context.no_rtti); lbValue any_id = lb_emit_struct_ev(p, v, 1); @@ -3176,13 +3479,8 @@ gb_internal lbValue lb_build_expr_internal(lbProcedure *p, Ast *expr) { Type *type = type_of_expr(expr); GB_ASSERT_MSG(tv.mode != Addressing_Invalid, "invalid expression '%s' (tv.mode = %d, tv.type = %s) @ %s\n Current Proc: %.*s : %s", expr_to_string(expr), tv.mode, type_to_string(tv.type), token_pos_to_string(expr_pos), LIT(p->name), type_to_string(p->type)); - if (tv.value.kind != ExactValue_Invalid) { - // NOTE(bill): The commented out code below is just for debug purposes only - // if (is_type_untyped(type)) { - // gb_printf_err("%s %s : %s @ %p\n", token_pos_to_string(expr_pos), expr_to_string(expr), type_to_string(expr->tav.type), expr); - // GB_PANIC("%s\n", type_to_string(tv.type)); - // } + if (tv.value.kind != ExactValue_Invalid) { // NOTE(bill): Short on constant values return lb_const_value(p->module, type, tv.value); } else if (tv.mode == Addressing_Type) { @@ -3192,13 +3490,25 @@ gb_internal lbValue lb_build_expr_internal(lbProcedure *p, Ast *expr) { switch (expr->kind) { case_ast_node(bl, BasicLit, expr); + if (type != nullptr && type->Named.name == "Error") { + Entity *e = type->Named.type_name; + if (e->pkg && e->pkg->name == "os") { + return lb_const_nil(p->module, type); + } + } TokenPos pos = bl->token.pos; - GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(token_strings[bl->token.kind])); + GB_PANIC("Non-constant basic literal %s - %.*s (%s)", token_pos_to_string(pos), LIT(token_strings[bl->token.kind]), type_to_string(type)); case_end; case_ast_node(bd, BasicDirective, expr); TokenPos pos = bd->token.pos; - GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(bd->name.string)); + String name = bd->name.string; + if (name == "branch_location") { + GB_ASSERT(p->uses_branch_location); + String proc_name = p->entity->token.string; + return lb_emit_source_code_location_as_global(p, proc_name, p->branch_location_pos); + } + GB_PANIC("Non-constant basic literal %s - %.*s", token_pos_to_string(pos), LIT(name)); case_end; case_ast_node(i, Implicit, expr); @@ -3364,7 +3674,7 @@ gb_internal lbValue lb_build_expr_internal(lbProcedure *p, Ast *expr) { lb_emit_if(p, lb_emit_try_has_value(p, rhs), then, else_); lb_start_block(p, else_); - lb_emit_defer_stmts(p, lbDeferExit_Branch, block); + lb_emit_defer_stmts(p, lbDeferExit_Branch, block, expr); lb_emit_jump(p, block); lb_start_block(p, then); @@ -3503,7 +3813,9 @@ gb_internal lbValue lb_get_using_variable(lbProcedure *p, Entity *e) { lbValue v = {}; + bool is_soa = false; if (pv == nullptr && parent->flags & EntityFlag_SoaPtrField) { + is_soa = true; // NOTE(bill): using SOA value (probably from for-in statement) lbAddr parent_addr = lb_get_soa_variable_addr(p, parent); v = lb_addr_get_ptr(p, parent_addr); @@ -3514,7 +3826,7 @@ gb_internal lbValue lb_get_using_variable(lbProcedure *p, Entity *e) { v = lb_build_addr_ptr(p, e->using_expr); } GB_ASSERT(v.value != nullptr); - GB_ASSERT_MSG(parent->type == type_deref(v.type), "%s %s", type_to_string(parent->type), type_to_string(v.type)); + GB_ASSERT_MSG(is_soa || parent->type == type_deref(v.type), "%s %s", type_to_string(parent->type), type_to_string(v.type)); lbValue ptr = lb_emit_deep_field_gep(p, v, sel); if (parent->scope) { if ((parent->scope->flags & (ScopeFlag_File|ScopeFlag_Pkg)) == 0) { @@ -3578,7 +3890,7 @@ gb_internal lbAddr lb_build_array_swizzle_addr(lbProcedure *p, AstCallExpr *ce, Type *type = base_type(lb_addr_type(addr)); GB_ASSERT(type->kind == Type_Array); i64 count = type->Array.count; - if (count <= 4) { + if (count <= 4 && index_count <= 4) { u8 indices[4] = {}; u8 index_count = 0; for (i32 i = 1; i < ce->args.count; i++) { @@ -3651,7 +3963,7 @@ gb_internal void lb_build_addr_compound_lit_populate(lbProcedure *p, Slice<Ast * Ast *elem = elems[i]; if (elem->kind == Ast_FieldValue) { ast_node(fv, FieldValue, elem); - if (lb_is_elem_const(fv->value, et)) { + if (bt->kind != Type_DynamicArray && lb_is_elem_const(fv->value, et)) { continue; } if (is_ast_range(fv->field)) { @@ -3775,27 +4087,39 @@ gb_internal lbAddr lb_build_addr_index_expr(lbProcedure *p, Ast *expr) { if (ie->expr->tav.mode == Addressing_SoaVariable) { // SOA Structures for slices/dynamic arrays - GB_ASSERT(is_type_pointer(type_of_expr(ie->expr))); + GB_ASSERT_MSG(is_type_multi_pointer(type_of_expr(ie->expr)), "%s", type_to_string(type_of_expr(ie->expr))); lbValue field = lb_build_expr(p, ie->expr); lbValue index = lb_build_expr(p, ie->index); - if (!build_context.no_bounds_check) { - // TODO HACK(bill): Clean up this hack to get the length for bounds checking - // GB_ASSERT(LLVMIsALoadInst(field.value)); - - // lbValue a = {}; - // a.value = LLVMGetOperand(field.value, 0); - // a.type = alloc_type_pointer(field.type); - - // irInstr *b = &a->Instr; - // GB_ASSERT(b->kind == irInstr_StructElementPtr); - // lbValue base_struct = b->StructElementPtr.address; + Ast *se_expr = unparen_expr(ie->expr); + if (se_expr->kind == Ast_SelectorExpr) { + ast_node(se, SelectorExpr, se_expr); + lbValue len = {}; + + Type *type = base_type(type_deref(type_of_expr(se->expr))); + GB_ASSERT_MSG(is_type_soa_struct(type), "%s", type_to_string(type)); + if (type->Struct.soa_kind == StructSoa_Fixed) { + len = lb_const_int(p->module, t_int, type->Struct.soa_count); + } else { + lbAddr *found = map_get(&p->selector_addr, se_expr); + if (found) { + lbAddr addr = *found; + lbValue parent = lb_addr_get_ptr(p, addr); + if (is_type_pointer(type_deref(parent.type))) { + parent = lb_emit_load(p, parent); + } + len = lb_soa_struct_len(p, parent); + } + } - // GB_ASSERT(is_type_soa_struct(type_deref(ir_type(base_struct)))); - // lbValue len = ir_soa_struct_len(p, base_struct); - // lb_emit_bounds_check(p, ast_token(ie->index), index, len); + if (len.value) { + lb_emit_bounds_check(p, ast_token(ie->index), index, len); + } + } else { + // TODO(bill): how do you even do bounds checking here? + } } lbValue val = lb_emit_ptr_offset(p, field, index); return lb_addr(val); @@ -3901,30 +4225,6 @@ gb_internal lbAddr lb_build_addr_index_expr(lbProcedure *p, Ast *expr) { return lb_addr(v); } - case Type_RelativeMultiPointer: { - lbAddr rel_ptr_addr = {}; - if (deref) { - lbValue rel_ptr_ptr = lb_build_expr(p, ie->expr); - rel_ptr_addr = lb_addr(rel_ptr_ptr); - } else { - rel_ptr_addr = lb_build_addr(p, ie->expr); - } - lbValue rel_ptr = lb_relative_pointer_to_pointer(p, rel_ptr_addr); - - lbValue index = lb_build_expr(p, ie->index); - index = lb_emit_conv(p, index, t_int); - lbValue v = {}; - - Type *pointer_type = base_type(t->RelativeMultiPointer.pointer_type); - GB_ASSERT(pointer_type->kind == Type_MultiPointer); - Type *elem = pointer_type->MultiPointer.elem; - - LLVMValueRef indices[1] = {index.value}; - v.value = LLVMBuildGEP2(p->builder, lb_type(p->module, elem), rel_ptr.value, indices, 1, ""); - v.type = alloc_type_pointer(elem); - return lb_addr(v); - } - case Type_DynamicArray: { lbValue dynamic_array = {}; dynamic_array = lb_build_expr(p, ie->expr); @@ -3947,12 +4247,21 @@ gb_internal lbAddr lb_build_addr_index_expr(lbProcedure *p, Ast *expr) { } lbValue index = lb_build_expr(p, ie->index); index = lb_emit_conv(p, index, t_int); - lbValue elem = lb_emit_matrix_ep(p, matrix, lb_const_int(p->module, t_int, 0), index); + + isize bounds_len = 0; + lbValue elem = {}; + if (t->Matrix.is_row_major) { + bounds_len = t->Matrix.row_count; + elem = lb_emit_matrix_ep(p, matrix, index, lb_const_int(p->module, t_int, 0)); + } else { + bounds_len = t->Matrix.column_count; + elem = lb_emit_matrix_ep(p, matrix, lb_const_int(p->module, t_int, 0), index); + } elem = lb_emit_conv(p, elem, alloc_type_pointer(type_of_expr(expr))); auto index_tv = type_and_value_of_expr(ie->index); if (index_tv.mode != Addressing_Constant) { - lbValue len = lb_const_int(p->module, t_int, t->Matrix.column_count); + lbValue len = lb_const_int(p->module, t_int, bounds_len); lb_emit_bounds_check(p, ast_token(ie->index), index, len); } return lb_addr(elem); @@ -4025,13 +4334,6 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { return slice; } - case Type_RelativePointer: - GB_PANIC("TODO(bill): Type_RelativePointer should be handled above already on the lb_addr_load"); - break; - case Type_RelativeMultiPointer: - GB_PANIC("TODO(bill): Type_RelativeMultiPointer should be handled above already on the lb_addr_load"); - break; - case Type_DynamicArray: { Type *elem_type = type->DynamicArray.elem; Type *slice_type = alloc_type_slice(elem_type); @@ -4056,7 +4358,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { if (se->high == nullptr) { lbValue offset = base; LLVMValueRef indices[1] = {low.value}; - offset.value = LLVMBuildGEP2(p->builder, lb_type(p->module, offset.type->MultiPointer.elem), offset.value, indices, 1, ""); + offset.value = LLVMBuildGEP2(p->builder, lb_type(p->module, base_type(offset.type)->MultiPointer.elem), offset.value, indices, 1, ""); lb_addr_store(p, res, offset); } else { low = lb_emit_conv(p, low, t_int); @@ -4065,7 +4367,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { lb_emit_multi_pointer_slice_bounds_check(p, se->open, low, high); LLVMValueRef indices[1] = {low.value}; - LLVMValueRef ptr = LLVMBuildGEP2(p->builder, lb_type(p->module, base.type->MultiPointer.elem), base.value, indices, 1, ""); + LLVMValueRef ptr = LLVMBuildGEP2(p->builder, lb_type(p->module, base_type(base.type)->MultiPointer.elem), base.value, indices, 1, ""); LLVMValueRef len = LLVMBuildSub(p->builder, high.value, low.value, ""); LLVMValueRef gep0 = lb_emit_struct_ep(p, res.addr, 0).value; @@ -4099,7 +4401,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { } case Type_Basic: { - GB_ASSERT_MSG(type == t_string, "got %s", type_to_string(type)); + GB_ASSERT_MSG(are_types_identical(type, t_string), "got %s", type_to_string(type)); lbValue len = lb_string_len(p, base); if (high.value == nullptr) high = len; @@ -4133,6 +4435,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { lbValue field_dst = lb_emit_struct_ep(p, dst.addr, i); lbValue field_src = lb_emit_struct_ep(p, lb_addr_get_ptr(p, addr), i); field_src = lb_emit_array_ep(p, field_src, low); + field_src = lb_emit_conv(p, field_src, type_deref(field_dst.type)); lb_emit_store(p, field_dst, field_src); } @@ -4148,6 +4451,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { lbValue field_dst = lb_emit_struct_ep(p, dst.addr, i); lbValue field_src = lb_emit_struct_ev(p, base, i); field_src = lb_emit_ptr_offset(p, field_src, low); + field_src = lb_emit_conv(p, field_src, type_deref(field_dst.type)); lb_emit_store(p, field_dst, field_src); } @@ -4162,6 +4466,7 @@ gb_internal lbAddr lb_build_addr_slice_expr(lbProcedure *p, Ast *expr) { lbValue field_dst = lb_emit_struct_ep(p, dst.addr, i); lbValue field_src = lb_emit_struct_ev(p, base, i); field_src = lb_emit_ptr_offset(p, field_src, low); + field_src = lb_emit_conv(p, field_src, type_deref(field_dst.type)); lb_emit_store(p, field_dst, field_src); } @@ -4211,6 +4516,208 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { switch (bt->kind) { default: GB_PANIC("Unknown CompoundLit type: %s", type_to_string(type)); break; + case Type_BitField: { + TEMPORARY_ALLOCATOR_GUARD(); + + // Type *backing_type = core_type(bt->BitField.backing_type); + + struct FieldData { + Type *field_type; + u64 bit_offset; + u64 bit_size; + }; + auto values = array_make<lbValue>(temporary_allocator(), 0, cl->elems.count); + auto fields = array_make<FieldData>(temporary_allocator(), 0, cl->elems.count); + + for (Ast *elem : cl->elems) { + ast_node(fv, FieldValue, elem); + String name = fv->field->Ident.token.string; + Selection sel = lookup_field(bt, name, false); + GB_ASSERT(sel.is_bit_field); + GB_ASSERT(!sel.indirect); + GB_ASSERT(sel.index.count == 1); + GB_ASSERT(sel.entity != nullptr); + + i64 index = sel.index[0]; + Entity *f = bt->BitField.fields[index]; + GB_ASSERT(f == sel.entity); + i64 bit_offset = bt->BitField.bit_offsets[index]; + i64 bit_size = bt->BitField.bit_sizes[index]; + GB_ASSERT(bit_size > 0); + + Type *field_type = sel.entity->type; + lbValue field_expr = lb_build_expr(p, fv->value); + field_expr = lb_emit_conv(p, field_expr, field_type); + array_add(&values, field_expr); + array_add(&fields, FieldData{field_type, cast(u64)bit_offset, cast(u64)bit_size}); + } + + // NOTE(bill): inline insertion sort should be good enough, right? + for (isize i = 1; i < values.count; i++) { + for (isize j = i; + j > 0 && fields[i].bit_offset < fields[j].bit_offset; + j--) { + auto vtmp = values[j]; + values[j] = values[j-1]; + values[j-1] = vtmp; + + auto ftmp = fields[j]; + fields[j] = fields[j-1]; + fields[j-1] = ftmp; + } + } + + bool any_fields_different_endian = false; + for (auto const &f : fields) { + if (is_type_different_to_arch_endianness(f.field_type)) { + // NOTE(bill): Just be slow for this, to be correct + any_fields_different_endian = true; + break; + } + } + + if (!any_fields_different_endian && + fields.count == bt->BitField.fields.count) { + // SINGLE INTEGER BACKING ONLY + + Type *backing_type = core_type(bt->BitField.backing_type); + GB_ASSERT(is_type_integer(backing_type) || + (is_type_array(backing_type) && is_type_integer(backing_type->Array.elem))); + + // NOTE(bill): all fields are present + // this means no masking is necessary since on write, the bits will be overridden + + lbValue dst_byte_ptr = lb_emit_conv(p, v.addr, t_u8_ptr); + u64 total_bit_size = cast(u64)(8*type_size_of(bt)); + + if (is_type_integer(backing_type)) { + LLVMTypeRef lit = lb_type(p->module, backing_type); + + LLVMValueRef res = LLVMConstInt(lit, 0, false); + + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef mask = LLVMConstInt(lit, 1, false); + #if LLVM_VERSION_MAJOR >= 19 + mask = LLVMBuildShl(p->builder, mask, LLVMConstInt(lit, f.bit_size, false), ""); + #else + mask = LLVMConstShl(mask, LLVMConstInt(lit, f.bit_size, false)); + #endif + mask = LLVMConstSub(mask, LLVMConstInt(lit, 1, false)); + + LLVMValueRef elem = values[i].value; + if (lb_sizeof(lit) < lb_sizeof(LLVMTypeOf(elem))) { + elem = LLVMBuildTrunc(p->builder, elem, lit, ""); + } else { + elem = LLVMBuildZExt(p->builder, elem, lit, ""); + } + elem = LLVMBuildAnd(p->builder, elem, mask, ""); + + elem = LLVMBuildShl(p->builder, elem, LLVMConstInt(lit, f.bit_offset, false), ""); + + res = LLVMBuildOr(p->builder, res, elem, ""); + } + + LLVMBuildStore(p->builder, res, v.addr.value); + } else if (is_type_array(backing_type)) { + // ARRAY OF INTEGER BACKING + + i64 array_count = backing_type->Array.count; + LLVMTypeRef lit = lb_type(p->module, core_type(backing_type->Array.elem)); + gb_unused(array_count); + gb_unused(lit); + + LLVMValueRef *elems = gb_alloc_array(temporary_allocator(), LLVMValueRef, array_count); + for (i64 i = 0; i < array_count; i++) { + elems[i] = LLVMConstInt(lit, 0, false); + } + + u64 elem_bit_size = cast(u64)(8*type_size_of(backing_type->Array.elem)); + u64 curr_bit_offset = 0; + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef val = values[i].value; + LLVMTypeRef vt = lb_type(p->module, values[i].type); + for (u64 bits_to_set = f.bit_size; + bits_to_set > 0; + /**/) { + i64 elem_idx = curr_bit_offset/elem_bit_size; + u64 elem_bit_offset = curr_bit_offset%elem_bit_size; + + u64 mask_width = gb_min(bits_to_set, elem_bit_size-elem_bit_offset); + GB_ASSERT(mask_width > 0); + bits_to_set -= mask_width; + + LLVMValueRef mask = LLVMConstInt(vt, 1, false); + #if LLVM_VERSION_MAJOR >= 19 + mask = LLVMBuildShl(p->builder, mask, LLVMConstInt(vt, mask_width, false), ""); + #else + mask = LLVMConstShl(mask, LLVMConstInt(vt, mask_width, false)); + #endif + mask = LLVMConstSub(mask, LLVMConstInt(vt, 1, false)); + + LLVMValueRef to_set = LLVMBuildAnd(p->builder, val, mask, ""); + + if (elem_bit_offset != 0) { + to_set = LLVMBuildShl(p->builder, to_set, LLVMConstInt(vt, elem_bit_offset, false), ""); + } + to_set = LLVMBuildTrunc(p->builder, to_set, lit, ""); + + if (LLVMIsNull(elems[elem_idx])) { + elems[elem_idx] = to_set; // don't even bother doing `0 | to_set` + } else { + elems[elem_idx] = LLVMBuildOr(p->builder, elems[elem_idx], to_set, ""); + } + + if (mask_width != 0) { + val = LLVMBuildLShr(p->builder, val, LLVMConstInt(vt, mask_width, false), ""); + } + curr_bit_offset += mask_width; + } + + GB_ASSERT(curr_bit_offset == f.bit_offset + f.bit_size); + } + + for (i64 i = 0; i < array_count; i++) { + LLVMValueRef elem_ptr = LLVMBuildStructGEP2(p->builder, lb_type(p->module, backing_type), v.addr.value, cast(unsigned)i, ""); + LLVMBuildStore(p->builder, elems[i], elem_ptr); + } + } else { + // SLOW STORAGE + + for_array(i, fields) { + auto const &f = fields[i]; + + if ((f.bit_offset & 7) == 0) { + u64 unpacked_bit_size = cast(u64)(8*type_size_of(f.field_type)); + u64 byte_size = (f.bit_size+7)/8; + + if (f.bit_offset + unpacked_bit_size <= total_bit_size) { + byte_size = unpacked_bit_size/8; + } + lbValue dst = lb_emit_ptr_offset(p, dst_byte_ptr, lb_const_int(p->module, t_int, f.bit_offset/8)); + lbValue src = lb_address_from_load_or_generate_local(p, values[i]); + lb_mem_copy_non_overlapping(p, dst, src, lb_const_int(p->module, t_uintptr, byte_size)); + } else { + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } + } + } + } else { + // individual storing + for_array(i, values) { + auto const &f = fields[i]; + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } + } + + return v; + } + case Type_Struct: { // TODO(bill): "constant" '#raw_union's are not initialized constantly at the moment. // NOTE(bill): This is due to the layout of the unions when printed to LLVM-IR @@ -4239,10 +4746,26 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { if (lb_is_nested_possibly_constant(type, sel, elem)) { continue; } - lbValue dst = lb_emit_deep_field_gep(p, comp_lit_ptr, sel); field_expr = lb_build_expr(p, elem); field_expr = lb_emit_conv(p, field_expr, sel.entity->type); - lb_emit_store(p, dst, field_expr); + if (sel.is_bit_field) { + Selection sub_sel = trim_selection(sel); + lbValue trimmed_dst = lb_emit_deep_field_gep(p, comp_lit_ptr, sub_sel); + Type *bf = base_type(type_deref(trimmed_dst.type)); + if (is_type_pointer(bf)) { + trimmed_dst = lb_emit_load(p, trimmed_dst); + bf = base_type(type_deref(trimmed_dst.type)); + } + GB_ASSERT(bf->kind == Type_BitField); + + isize idx = sel.index[sel.index.count-1]; + lbAddr dst = lb_addr_bit_field(trimmed_dst, bf->BitField.fields[idx]->type, bf->BitField.bit_offsets[idx], bf->BitField.bit_sizes[idx]); + lb_addr_store(p, dst, field_expr); + + } else { + lbValue dst = lb_emit_deep_field_gep(p, comp_lit_ptr, sel); + lb_emit_store(p, dst, field_expr); + } continue; } @@ -4290,7 +4813,7 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { if (cl->elems.count == 0) { break; } - GB_ASSERT(!build_context.no_dynamic_literals); + GB_ASSERT(expr->file()->feature_flags & OptInFeatureFlag_DynamicLiterals); lbValue err = lb_dynamic_map_reserve(p, v.addr, 2*cl->elems.count, pos); gb_unused(err); @@ -4379,7 +4902,7 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { if (cl->elems.count == 0) { break; } - GB_ASSERT(!build_context.no_dynamic_literals); + GB_ASSERT(expr->file()->feature_flags & OptInFeatureFlag_DynamicLiterals); Type *et = bt->DynamicArray.elem; lbValue size = lb_const_int(p->module, t_int, type_size_of(et)); @@ -4467,29 +4990,43 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { case Type_BitSet: { i64 sz = type_size_of(type); if (cl->elems.count > 0 && sz > 0) { - lb_addr_store(p, v, lb_const_value(p->module, type, exact_value_compound(expr))); - lbValue lower = lb_const_value(p->module, t_int, exact_value_i64(bt->BitSet.lower)); - for (Ast *elem : cl->elems) { - GB_ASSERT(elem->kind != Ast_FieldValue); - if (lb_is_elem_const(elem, et)) { - continue; + Type *backing = bit_set_to_int(type); + if (is_type_array(backing)) { + GB_PANIC("TODO: bit_set [N]T"); + Type *base_it = core_array_type(backing); + i64 bits_per_elem = 8*type_size_of(base_it); + gb_unused(bits_per_elem); + lbValue one = lb_const_value(p->module, t_i64, exact_value_i64(1)); + for (Ast *elem : cl->elems) { + GB_ASSERT(elem->kind != Ast_FieldValue); + lbValue expr = lb_build_expr(p, elem); + GB_ASSERT(expr.type->kind != Type_Tuple); + + lbValue e = lb_emit_conv(p, expr, t_i64); + e = lb_emit_arith(p, Token_Sub, e, lower, t_i64); + // lbValue idx = lb_emit_arith(p, Token_Div, e, bits_per_elem, t_i64); + // lbValue val = lb_emit_arith(p, Token_Div, e, bits_per_elem, t_i64); } - - lbValue expr = lb_build_expr(p, elem); - GB_ASSERT(expr.type->kind != Type_Tuple); - + } else { Type *it = bit_set_to_int(bt); lbValue one = lb_const_value(p->module, it, exact_value_i64(1)); - lbValue e = lb_emit_conv(p, expr, it); - e = lb_emit_arith(p, Token_Sub, e, lower, it); - e = lb_emit_arith(p, Token_Shl, one, e, it); - - lbValue old_value = lb_emit_transmute(p, lb_addr_load(p, v), it); - lbValue new_value = lb_emit_arith(p, Token_Or, old_value, e, it); - new_value = lb_emit_transmute(p, new_value, type); - lb_addr_store(p, v, new_value); + for (Ast *elem : cl->elems) { + GB_ASSERT(elem->kind != Ast_FieldValue); + + lbValue expr = lb_build_expr(p, elem); + GB_ASSERT(expr.type->kind != Type_Tuple); + + lbValue e = lb_emit_conv(p, expr, it); + e = lb_emit_arith(p, Token_Sub, e, lower, it); + e = lb_emit_arith(p, Token_Shl, one, e, it); + + lbValue old_value = lb_emit_transmute(p, lb_addr_load(p, v), it); + lbValue new_value = lb_emit_arith(p, Token_Or, old_value, e, it); + new_value = lb_emit_transmute(p, new_value, type); + lb_addr_store(p, v, new_value); + } } } break; @@ -4591,15 +5128,17 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { if (tav.mode == Addressing_Type) { // Addressing_Type Selection sel = lookup_field(tav.type, selector, true); if (sel.pseudo_field) { - GB_ASSERT(sel.entity->kind == Entity_Procedure); - return lb_addr(lb_find_value_from_entity(p->module, sel.entity)); + GB_ASSERT(sel.entity->kind == Entity_Procedure || sel.entity->kind == Entity_ProcGroup); + Entity *e = entity_of_node(sel_node); + GB_ASSERT(e->kind == Entity_Procedure); + return lb_addr(lb_find_value_from_entity(p->module, e)); } GB_PANIC("Unreachable %.*s", LIT(selector)); } if (se->swizzle_count > 0) { Type *array_type = base_type(type_deref(tav.type)); - GB_ASSERT(array_type->kind == Type_Array); + GB_ASSERT(array_type->kind == Type_Array || array_type->kind == Type_SimdVector); u8 swizzle_count = se->swizzle_count; u8 swizzle_indices_raw = se->swizzle_indices; u8 swizzle_indices[4] = {}; @@ -4615,8 +5154,9 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { a = lb_addr_get_ptr(p, addr); } - GB_ASSERT(is_type_array(expr->tav.type)); - return lb_addr_swizzle(a, expr->tav.type, swizzle_count, swizzle_indices); + Type *type = type_deref(expr->tav.type); + GB_ASSERT(is_type_array(type) || is_type_simd_vector(type)); + return lb_addr_swizzle(a, type, swizzle_count, swizzle_indices); } Selection sel = lookup_field(type, selector, false); @@ -4628,6 +5168,33 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { return lb_addr(lb_find_value_from_entity(p->module, e)); } + if (sel.is_bit_field) { + lbAddr addr = lb_build_addr(p, se->expr); + + Selection sub_sel = sel; + sub_sel.index.count -= 1; + + lbValue ptr = lb_addr_get_ptr(p, addr); + if (sub_sel.index.count > 0) { + ptr = lb_emit_deep_field_gep(p, ptr, sub_sel); + } + if (is_type_pointer(type_deref(ptr.type))) { + ptr = lb_emit_load(p, ptr); + } + + Type *bf_type = type_deref(ptr.type); + bf_type = base_type(bf_type); + GB_ASSERT(bf_type->kind == Type_BitField); + + i32 index = sel.index[sel.index.count-1]; + + Entity *f = bf_type->BitField.fields[index]; + u8 bit_size = bf_type->BitField.bit_sizes[index]; + i64 bit_offset = bf_type->BitField.bit_offsets[index]; + + return lb_addr_bit_field(ptr, f->type, bit_offset, bit_size); + } + { lbAddr addr = lb_build_addr(p, se->expr); if (addr.kind == lbAddr_Map) { @@ -4670,6 +5237,9 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { if (sub_sel.index.count > 0) { item = lb_emit_deep_field_gep(p, item, sub_sel); } + // make sure it's ^T and not [^]T + item.type = alloc_type_multi_pointer_to_pointer(item.type); + return lb_addr(item); } else if (addr.kind == lbAddr_Swizzle) { GB_ASSERT(sel.index.count > 0); @@ -4681,6 +5251,11 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { sel.index[0] = addr.swizzle.indices[sel.index[0]]; } + Type *atype = type_deref(lb_addr_type(addr)); + if (is_type_soa_struct(atype)) { + map_set(&p->selector_addr, expr, addr); + } + lbValue a = lb_addr_get_ptr(p, addr); a = lb_emit_deep_field_gep(p, a, sel); return lb_addr(a); @@ -4770,11 +5345,7 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { case_ast_node(de, DerefExpr, expr); Type *t = type_of_expr(de->expr); - if (is_type_relative_pointer(t)) { - lbAddr addr = lb_build_addr(p, de->expr); - addr.relative.deref = true; - return addr; - } else if (is_type_soa_pointer(t)) { + if (is_type_soa_pointer(t)) { lbValue value = lb_build_expr(p, de->expr); lbValue ptr = lb_emit_struct_ev(p, value, 0); lbValue idx = lb_emit_struct_ev(p, value, 1); @@ -4886,6 +5457,54 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { lbValue ptr = lb_address_from_load_or_generate_local(p, lb_build_expr(p, expr)); return lb_addr(ptr); case_end; + + + case_ast_node(be, OrBranchExpr, expr); + lbBlock *block = nullptr; + + if (be->label != nullptr) { + lbBranchBlocks bb = lb_lookup_branch_blocks(p, be->label); + switch (be->token.kind) { + case Token_or_break: block = bb.break_; break; + case Token_or_continue: block = bb.continue_; break; + } + } else { + for (lbTargetList *t = p->target_list; t != nullptr && block == nullptr; t = t->prev) { + if (t->is_block) { + continue; + } + + switch (be->token.kind) { + case Token_or_break: block = t->break_; break; + case Token_or_continue: block = t->continue_; break; + } + } + } + + GB_ASSERT(block != nullptr); + TypeAndValue tv = expr->tav; + + lbValue lhs = {}; + lbValue rhs = {}; + lb_emit_try_lhs_rhs(p, be->expr, tv, &lhs, &rhs); + Type *type = default_type(tv.type); + if (lhs.value) { + lhs = lb_emit_conv(p, lhs, type); + } else if (type != nullptr && type != t_invalid) { + lhs = lb_const_nil(p->module, type); + } + + lbBlock *then = lb_create_block(p, "or_branch.then"); + lbBlock *else_ = lb_create_block(p, "or_branch.else"); + + lb_emit_if(p, lb_emit_try_has_value(p, rhs), then, else_); + lb_start_block(p, else_); + lb_emit_defer_stmts(p, lbDeferExit_Branch, block, expr); + lb_emit_jump(p, block); + lb_start_block(p, then); + + return lb_addr(lb_address_from_load_or_generate_local(p, lhs)); + case_end; } TokenPos token_pos = ast_token(expr).pos; |