From 4c655865e5d9af83a98c137609b01972f4e51beb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 18 Oct 2021 16:52:19 +0100 Subject: Begin work on matrix type --- src/llvm_backend_general.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 094275429..ee8f220ef 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -1930,6 +1930,24 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { fields[1] = base_integer; return LLVMStructTypeInContext(ctx, fields, field_count, false); } + + case Type_Matrix: + { + i64 size = type_size_of(type); + i64 elem_size = type_size_of(type->Matrix.elem); + GB_ASSERT(elem_size > 0); + i64 elem_count = size/elem_size; + GB_ASSERT(elem_count > 0); + + m->internal_type_level -= 1; + + LLVMTypeRef elem = lb_type(m, type->Matrix.elem); + LLVMTypeRef t = LLVMArrayType(elem, cast(unsigned)elem_count); + + m->internal_type_level += 1; + return t; + } + } GB_PANIC("Invalid type %s", type_to_string(type)); -- cgit v1.2.3 From 243e2e2b8a7566087375178a66b25b5d9ac9a356 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 19 Oct 2021 11:24:26 +0100 Subject: Basic support for matrix*vector, vector*matrix operations --- core/fmt/fmt.odin | 16 ++-- src/check_expr.cpp | 30 +++--- src/llvm_backend.cpp | 57 ++++++++++-- src/llvm_backend_expr.cpp | 213 ++++++++++++++++++++++++------------------- src/llvm_backend_general.cpp | 8 +- src/llvm_backend_utility.cpp | 10 +- src/types.cpp | 3 + 7 files changed, 207 insertions(+), 130 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 804a29cab..46b1fc14c 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -1960,13 +1960,13 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { fi.indent += 1; defer fi.indent -= 1 - if fi.hash { + if fi.hash { io.write_byte(fi.writer, '\n') // TODO(bill): Should this render it like in written form? e.g. tranposed - for col in 0.. 0 { io.write_string(fi.writer, ", ") } + for col in 0.. 0 { io.write_string(fi.writer, ", ") } offset := row*info.elem_size + col*info.stride @@ -1976,10 +1976,10 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { io.write_string(fi.writer, ";\n") } } else { - for col in 0.. 0 { io.write_string(fi.writer, "; ") } - for row in 0.. 0 { io.write_string(fi.writer, ", ") } + for row in 0.. 0 { io.write_string(fi.writer, ", ") } + for col in 0.. 0 { io.write_string(fi.writer, "; ") } offset := row*info.elem_size + col*info.stride diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 9c12802d7..1ca5b895d 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -2686,10 +2686,11 @@ void check_binary_matrix(CheckerContext *c, Token const &op, Operand *x, Operand x->mode = Addressing_Invalid; return; } + + Type *xt = base_type(x->type); + Type *yt = base_type(y->type); if (is_type_matrix(x->type)) { - Type *xt = base_type(x->type); - Type *yt = base_type(y->type); GB_ASSERT(xt->kind == Type_Matrix); if (op.kind == Token_Mul) { if (yt->kind == Type_Matrix) { @@ -2714,7 +2715,11 @@ void check_binary_matrix(CheckerContext *c, Token const &op, Operand *x, Operand // Treat arrays as column vectors x->mode = Addressing_Value; - x->type = alloc_type_matrix(xt->Matrix.elem, xt->Matrix.row_count, 1); + if (type_hint == nullptr && xt->Matrix.row_count == yt->Array.count) { + x->type = y->type; + } else { + x->type = alloc_type_matrix(xt->Matrix.elem, xt->Matrix.row_count, 1); + } goto matrix_success; } } @@ -2725,8 +2730,6 @@ void check_binary_matrix(CheckerContext *c, Token const &op, Operand *x, Operand x->type = xt; goto matrix_success; } else { - Type *xt = base_type(x->type); - Type *yt = base_type(y->type); GB_ASSERT(is_type_matrix(yt)); GB_ASSERT(!is_type_matrix(xt)); @@ -2743,7 +2746,11 @@ void check_binary_matrix(CheckerContext *c, Token const &op, Operand *x, Operand // Treat arrays as row vectors x->mode = Addressing_Value; - x->type = alloc_type_matrix(xt->Matrix.elem, 1, xt->Matrix.column_count); + if (type_hint == nullptr && yt->Matrix.column_count == xt->Array.count) { + x->type = x->type; + } else { + x->type = alloc_type_matrix(yt->Matrix.elem, 1, yt->Matrix.column_count); + } goto matrix_success; } } @@ -2775,13 +2782,13 @@ matrix_success: matrix_error: - gbString xt = type_to_string(x->type); - gbString yt = type_to_string(y->type); + gbString xts = type_to_string(x->type); + gbString yts = type_to_string(y->type); gbString expr_str = expr_to_string(x->expr); - error(op, "Mismatched types in binary matrix expression '%s' for operator '%.*s' : '%s' vs '%s'", expr_str, LIT(op.string), xt, yt); + error(op, "Mismatched types in binary matrix expression '%s' for operator '%.*s' : '%s' vs '%s'", expr_str, LIT(op.string), xts, yts); gb_string_free(expr_str); - gb_string_free(yt); - gb_string_free(xt); + gb_string_free(yts); + gb_string_free(xts); x->type = t_invalid; x->mode = Addressing_Invalid; return; @@ -2994,6 +3001,7 @@ void check_binary_expr(CheckerContext *c, Operand *x, Ast *node, Type *type_hint } if (is_type_matrix(x->type) || is_type_matrix(y->type)) { check_binary_matrix(c, op, x, y, type_hint, use_lhs_as_type_hint); + x->expr = node; return; } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index a72ddc646..a853a6224 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -1135,13 +1135,46 @@ void lb_generate_code(lbGenerator *gen) { auto *min_dep_set = &info->minimum_dependency_set; - LLVMInitializeAllTargetInfos(); - LLVMInitializeAllTargets(); - LLVMInitializeAllTargetMCs(); - LLVMInitializeAllAsmPrinters(); - LLVMInitializeAllAsmParsers(); - LLVMInitializeAllDisassemblers(); - LLVMInitializeNativeTarget(); + switch (build_context.metrics.arch) { + case TargetArch_amd64: + case TargetArch_386: + LLVMInitializeX86TargetInfo(); + LLVMInitializeX86Target(); + LLVMInitializeX86TargetMC(); + LLVMInitializeX86AsmPrinter(); + LLVMInitializeX86AsmParser(); + LLVMInitializeX86Disassembler(); + break; + case TargetArch_arm64: + LLVMInitializeAArch64TargetInfo(); + LLVMInitializeAArch64Target(); + LLVMInitializeAArch64TargetMC(); + LLVMInitializeAArch64AsmPrinter(); + LLVMInitializeAArch64AsmParser(); + LLVMInitializeAArch64Disassembler(); + break; + case TargetArch_wasm32: + LLVMInitializeWebAssemblyTargetInfo(); + LLVMInitializeWebAssemblyTarget(); + LLVMInitializeWebAssemblyTargetMC(); + LLVMInitializeWebAssemblyAsmPrinter(); + LLVMInitializeWebAssemblyAsmParser(); + LLVMInitializeWebAssemblyDisassembler(); + break; + default: + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargets(); + LLVMInitializeAllTargetMCs(); + LLVMInitializeAllAsmPrinters(); + LLVMInitializeAllAsmParsers(); + LLVMInitializeAllDisassemblers(); + break; + } + + + if (build_context.microarch == "native") { + LLVMInitializeNativeTarget(); + } char const *target_triple = alloc_cstring(permanent_allocator(), build_context.metrics.target_triplet); for_array(i, gen->modules.entries) { @@ -1174,6 +1207,14 @@ void lb_generate_code(lbGenerator *gen) { if (gb_strcmp(llvm_cpu, host_cpu_name) == 0) { llvm_features = LLVMGetHostCPUFeatures(); } + } else if (build_context.metrics.arch == TargetArch_amd64) { + // NOTE(bill): x86-64-v2 is more than enough for everyone + // + // x86-64: CMOV, CMPXCHG8B, FPU, FXSR, MMX, FXSR, SCE, SSE, SSE2 + // x86-64-v2: (close to Nehalem) CMPXCHG16B, LAHF-SAHF, POPCNT, SSE3, SSE4.1, SSE4.2, SSSE3 + // x86-64-v3: (close to Haswell) AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE + // x86-64-v4: AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL + llvm_cpu = "x86-64-v2"; } // GB_ASSERT_MSG(LLVMTargetHasAsmBackend(target)); @@ -1640,6 +1681,7 @@ void lb_generate_code(lbGenerator *gen) { code_gen_file_type = LLVMAssemblyFile; } + for_array(j, gen->modules.entries) { lbModule *m = gen->modules.entries[j].value; if (LLVMVerifyModule(m->mod, LLVMReturnStatusAction, &llvm_error)) { @@ -1684,7 +1726,6 @@ void lb_generate_code(lbGenerator *gen) { } } - TIME_SECTION("LLVM Add Foreign Library Paths"); for_array(j, gen->modules.entries) { diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 2e2d45991..ed98c6845 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -509,12 +509,16 @@ LLVMValueRef llvm_matrix_column_major_load(lbProcedure *p, lbValue lhs) { GB_ASSERT(mt->kind == Type_Matrix); GB_ASSERT(lb_matrix_elem_simple(mt)); - unsigned total_elem_count = cast(unsigned)matrix_type_total_elems(mt); + + i64 stride = matrix_type_stride_in_elems(mt); + i64 rows = mt->Matrix.row_count; + i64 columns = mt->Matrix.column_count; + unsigned elem_count = cast(unsigned)(rows*columns); Type *elem = mt->Matrix.elem; LLVMTypeRef elem_type = lb_type(m, elem); - LLVMTypeRef vector_type = LLVMVectorType(elem_type, total_elem_count); + LLVMTypeRef vector_type = LLVMVectorType(elem_type, elem_count); LLVMTypeRef types[] = {vector_type}; char const *name = "llvm.matrix.column.major.load"; @@ -524,44 +528,18 @@ LLVMValueRef llvm_matrix_column_major_load(lbProcedure *p, lbValue lhs) { lbValue ptr = lb_address_from_load_or_generate_local(p, lhs); ptr = lb_emit_matrix_epi(p, ptr, 0, 0); - + LLVMValueRef values[5] = {}; values[0] = ptr.value; - values[1] = lb_const_int(m, t_u64, 8*matrix_type_stride(mt)).value; // bit width - values[2] = LLVMConstNull(lb_type(m, t_llvm_bool)); - values[3] = lb_const_int(m, t_u32, mt->Matrix.row_count).value; - values[4] = lb_const_int(m, t_u32, mt->Matrix.column_count).value; - - return LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); -} -LLVMValueRef llvm_matrix_column_major_load_from_ptr(lbProcedure *p, lbValue ptr) { - lbModule *m = p->module; - - Type *mt = base_type(type_deref(ptr.type)); - GB_ASSERT(mt->kind == Type_Matrix); - GB_ASSERT(lb_matrix_elem_simple(mt)); - - unsigned total_elem_count = cast(unsigned)matrix_type_total_elems(mt); - - Type *elem = mt->Matrix.elem; - LLVMTypeRef elem_type = lb_type(m, elem); - - LLVMTypeRef vector_type = LLVMVectorType(elem_type, total_elem_count); - LLVMTypeRef types[] = {vector_type}; - - char const *name = "llvm.matrix.column.major.load"; - unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); - GB_ASSERT_MSG(id != 0, "Unable to find %s", name); - LLVMValueRef ip = LLVMGetIntrinsicDeclaration(m->mod, id, types, gb_count_of(types)); - - LLVMValueRef values[5] = {}; - values[0] = lb_emit_matrix_epi(p, ptr, 0, 0).value; - values[1] = lb_const_int(m, t_u64, 8*matrix_type_stride(mt)).value; // bit width + values[1] = lb_const_int(m, t_u64, stride).value; values[2] = LLVMConstNull(lb_type(m, t_llvm_bool)); values[3] = lb_const_int(m, t_u32, mt->Matrix.row_count).value; values[4] = lb_const_int(m, t_u32, mt->Matrix.column_count).value; - return LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + LLVMValueRef call = LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + gb_printf_err("%s\n", LLVMPrintValueToString(call)); + // LLVMAddAttributeAtIndex(call, 0, lb_create_enum_attribute(p->module->ctx, "align", cast(u64)type_align_of(mt))); + return call; } void llvm_matrix_column_major_store(lbProcedure *p, lbAddr addr, LLVMValueRef vector_value) { @@ -571,12 +549,7 @@ void llvm_matrix_column_major_store(lbProcedure *p, lbAddr addr, LLVMValueRef ve GB_ASSERT(mt->kind == Type_Matrix); GB_ASSERT(lb_matrix_elem_simple(mt)); - unsigned total_elem_count = cast(unsigned)matrix_type_total_elems(mt); - - Type *elem = mt->Matrix.elem; - LLVMTypeRef elem_type = lb_type(m, elem); - - LLVMTypeRef vector_type = LLVMVectorType(elem_type, total_elem_count); + LLVMTypeRef vector_type = LLVMTypeOf(vector_value); LLVMTypeRef types[] = {vector_type}; char const *name = "llvm.matrix.column.major.store"; @@ -587,56 +560,26 @@ void llvm_matrix_column_major_store(lbProcedure *p, lbAddr addr, LLVMValueRef ve lbValue ptr = lb_addr_get_ptr(p, addr); ptr = lb_emit_matrix_epi(p, ptr, 0, 0); - GB_ASSERT(LLVMTypeOf(vector_value) == vector_type); unsigned vector_size = LLVMGetVectorSize(vector_type); GB_ASSERT((mt->Matrix.row_count*mt->Matrix.column_count) == cast(i64)vector_size); - LLVMValueRef values[6] = {}; - values[0] = vector_value; - values[1] = ptr.value; - values[2] = lb_const_int(m, t_u64, 8*matrix_type_stride(mt)).value; // bit width - values[3] = LLVMConstNull(lb_type(m, t_llvm_bool)); - values[4] = lb_const_int(m, t_u32, mt->Matrix.row_count).value; - values[5] = lb_const_int(m, t_u32, mt->Matrix.column_count).value; - - LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); -} - -void llvm_matrix_column_major_store_to_raw_ptr(lbProcedure *p, Type *mt, lbValue ptr, LLVMValueRef vector_value) { - lbModule *m = p->module; - - mt = base_type(mt); - GB_ASSERT(mt->kind == Type_Matrix); - GB_ASSERT(lb_matrix_elem_simple(mt)); - - unsigned total_elem_count = cast(unsigned)matrix_type_total_elems(mt); - - Type *elem = mt->Matrix.elem; - LLVMTypeRef elem_type = lb_type(m, elem); - - LLVMTypeRef vector_type = LLVMVectorType(elem_type, total_elem_count); - LLVMTypeRef types[] = {vector_type}; - - char const *name = "llvm.matrix.column.major.store"; - unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); - GB_ASSERT_MSG(id != 0, "Unable to find %s", name); - LLVMValueRef ip = LLVMGetIntrinsicDeclaration(m->mod, id, types, gb_count_of(types)); - - GB_ASSERT(LLVMTypeOf(vector_value) == vector_type); - unsigned vector_size = LLVMGetVectorSize(vector_type); - GB_ASSERT((mt->Matrix.row_count*mt->Matrix.column_count) == cast(i64)vector_size); + i64 stride = matrix_type_stride_in_elems(mt); LLVMValueRef values[6] = {}; values[0] = vector_value; values[1] = ptr.value; - values[2] = lb_const_int(m, t_u64, 8*matrix_type_stride(mt)).value; // bit width + values[2] = lb_const_int(m, t_u64, stride).value; values[3] = LLVMConstNull(lb_type(m, t_llvm_bool)); values[4] = lb_const_int(m, t_u32, mt->Matrix.row_count).value; values[5] = lb_const_int(m, t_u32, mt->Matrix.column_count).value; - LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + LLVMValueRef call = LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + gb_printf_err("%s\n", LLVMPrintValueToString(call)); + // LLVMAddAttributeAtIndex(call, 1, lb_create_enum_attribute(p->module->ctx, "align", cast(u64)type_align_of(mt))); + gb_unused(call); } + LLVMValueRef llvm_matrix_multiply(lbProcedure *p, LLVMValueRef a, LLVMValueRef b, i64 outer_rows, i64 inner, i64 outer_columns) { lbModule *m = p->module; @@ -648,6 +591,7 @@ LLVMValueRef llvm_matrix_multiply(lbProcedure *p, LLVMValueRef a, LLVMValueRef b LLVMTypeRef elem_type = LLVMGetElementType(a_type); LLVMTypeRef res_vector_type = LLVMVectorType(elem_type, cast(unsigned)(outer_rows*outer_columns)); + LLVMTypeRef types[] = {res_vector_type, a_type, b_type}; char const *name = "llvm.matrix.multiply"; @@ -662,7 +606,9 @@ LLVMValueRef llvm_matrix_multiply(lbProcedure *p, LLVMValueRef a, LLVMValueRef b values[3] = lb_const_int(m, t_u32, inner).value; values[4] = lb_const_int(m, t_u32, outer_columns).value; - return LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + LLVMValueRef call = LLVMBuildCall(p->builder, ip, values, gb_count_of(values), ""); + gb_printf_err("%s\n", LLVMPrintValueToString(call)); + return call; } @@ -684,19 +630,13 @@ lbValue lb_emit_matrix_mul(lbProcedure *p, lbValue lhs, lbValue rhs, Type *type) // TODO(bill): LLVM ERROR: Do not know how to split the result of this operator! lbAddr res = lb_add_local_generated(p, type, true); - lbValue res_ptr = lb_addr_get_ptr(p, res); - res_ptr = lb_emit_matrix_epi(p, res_ptr, 0, 0); - - lbValue lhs_ptr = lb_address_from_load_or_generate_local(p, lhs); - lbValue rhs_ptr = lb_address_from_load_or_generate_local(p, rhs); - LLVMValueRef a = llvm_matrix_column_major_load_from_ptr(p, lhs_ptr); - LLVMValueRef b = llvm_matrix_column_major_load_from_ptr(p, rhs_ptr); - LLVMValueRef c = llvm_matrix_multiply(p, a, b, xt->Matrix.row_count, xt->Matrix.column_count, yt->Matrix.column_count); - - llvm_matrix_column_major_store_to_raw_ptr(p, type, res_ptr, c); + LLVMValueRef a = llvm_matrix_column_major_load(p, lhs); gb_unused(a); + LLVMValueRef b = llvm_matrix_column_major_load(p, rhs); gb_unused(b); + LLVMValueRef c = llvm_matrix_multiply(p, a, b, xt->Matrix.row_count, xt->Matrix.column_count, yt->Matrix.column_count); gb_unused(c); + llvm_matrix_column_major_store(p, res, c); return lb_addr_load(p, res); - } + } slow_form: { @@ -704,18 +644,21 @@ slow_form: lbAddr res = lb_add_local_generated(p, type, true); - for (i64 i = 0; i < xt->Matrix.row_count; i++) { - for (i64 j = 0; j < yt->Matrix.column_count; j++) { - for (i64 k = 0; k < xt->Matrix.column_count; k++) { + i64 outer_rows = xt->Matrix.row_count; + i64 inner = xt->Matrix.column_count; + i64 outer_columns = yt->Matrix.column_count; + + for (i64 j = 0; j < outer_columns; j++) { + for (i64 i = 0; i < outer_rows; i++) { + for (i64 k = 0; k < inner; k++) { lbValue dst = lb_emit_matrix_epi(p, res.addr, i, j); + lbValue d0 = lb_emit_load(p, dst); lbValue a = lb_emit_matrix_ev(p, lhs, i, k); lbValue b = lb_emit_matrix_ev(p, rhs, k, j); lbValue c = lb_emit_arith(p, Token_Mul, a, b, elem); - lbValue d = lb_emit_load(p, dst); - lbValue e = lb_emit_arith(p, Token_Add, d, c, elem); - lb_emit_store(p, dst, e); - + lbValue d = lb_emit_arith(p, Token_Add, d0, c, elem); + lb_emit_store(p, dst, d); } } } @@ -724,6 +667,72 @@ slow_form: } } +lbValue lb_emit_matrix_mul_vector(lbProcedure *p, lbValue lhs, lbValue rhs, Type *type) { + Type *mt = base_type(lhs.type); + Type *vt = base_type(rhs.type); + + GB_ASSERT(is_type_matrix(mt)); + GB_ASSERT(is_type_array_like(vt)); + + i64 vector_count = get_array_type_count(vt); + + GB_ASSERT(mt->Matrix.column_count == vector_count); + GB_ASSERT(are_types_identical(mt->Matrix.elem, base_array_type(vt))); + + Type *elem = mt->Matrix.elem; + + lbAddr res = lb_add_local_generated(p, type, true); + + for (i64 i = 0; i < mt->Matrix.row_count; i++) { + for (i64 j = 0; j < mt->Matrix.column_count; j++) { + lbValue dst = lb_emit_matrix_epi(p, res.addr, i, 0); + lbValue d0 = lb_emit_load(p, dst); + + lbValue a = lb_emit_matrix_ev(p, lhs, i, j); + lbValue b = lb_emit_struct_ev(p, rhs, cast(i32)j); + lbValue c = lb_emit_arith(p, Token_Mul, a, b, elem); + lbValue d = lb_emit_arith(p, Token_Add, d0, c, elem); + lb_emit_store(p, dst, d); + } + } + + return lb_addr_load(p, res); +} + +lbValue lb_emit_vector_mul_matrix(lbProcedure *p, lbValue lhs, lbValue rhs, Type *type) { + Type *mt = base_type(rhs.type); + Type *vt = base_type(lhs.type); + + GB_ASSERT(is_type_matrix(mt)); + GB_ASSERT(is_type_array_like(vt)); + + i64 vector_count = get_array_type_count(vt); + + GB_ASSERT(mt->Matrix.row_count == vector_count); + GB_ASSERT(are_types_identical(mt->Matrix.elem, base_array_type(vt))); + + Type *elem = mt->Matrix.elem; + + lbAddr res = lb_add_local_generated(p, type, true); + + for (i64 j = 0; j < mt->Matrix.column_count; j++) { + for (i64 k = 0; k < mt->Matrix.row_count; k++) { + lbValue dst = lb_emit_matrix_epi(p, res.addr, 0, j); + lbValue d0 = lb_emit_load(p, dst); + + lbValue a = lb_emit_struct_ev(p, lhs, cast(i32)k); + lbValue b = lb_emit_matrix_ev(p, rhs, k, j); + lbValue c = lb_emit_arith(p, Token_Mul, a, b, elem); + lbValue d = lb_emit_arith(p, Token_Add, d0, c, elem); + lb_emit_store(p, dst, d); + } + } + + return lb_addr_load(p, res); +} + + + lbValue lb_emit_arith_matrix(lbProcedure *p, TokenKind op, lbValue lhs, lbValue rhs, Type *type) { GB_ASSERT(is_type_matrix(lhs.type) || is_type_matrix(rhs.type)); @@ -735,7 +744,12 @@ lbValue lb_emit_arith_matrix(lbProcedure *p, TokenKind op, lbValue lhs, lbValue if (xt->kind == Type_Matrix) { if (yt->kind == Type_Matrix) { return lb_emit_matrix_mul(p, lhs, rhs, type); + } else if (is_type_array_like(yt)) { + return lb_emit_matrix_mul_vector(p, lhs, rhs, type); } + } else if (is_type_array_like(xt)) { + GB_ASSERT(yt->kind == Type_Matrix); + return lb_emit_vector_mul_matrix(p, lhs, rhs, type); } } else { @@ -1036,6 +1050,13 @@ lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { ast_node(be, BinaryExpr, expr); TypeAndValue tv = type_and_value_of_expr(expr); + + if (is_type_matrix(be->left->tav.type) || is_type_matrix(be->right->tav.type)) { + lbValue left = lb_build_expr(p, be->left); + lbValue right = lb_build_expr(p, be->right); + return lb_emit_arith_matrix(p, be->op.kind, left, right, default_type(tv.type)); + } + switch (be->op.kind) { case Token_Add: diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index ee8f220ef..63a63349a 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -1937,7 +1937,7 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { i64 elem_size = type_size_of(type->Matrix.elem); GB_ASSERT(elem_size > 0); i64 elem_count = size/elem_size; - GB_ASSERT(elem_count > 0); + GB_ASSERT_MSG(elem_count > 0, "%s", type_to_string(type)); m->internal_type_level -= 1; @@ -2611,8 +2611,10 @@ lbAddr lb_add_local(lbProcedure *p, Type *type, Entity *e, bool zero_init, i32 p LLVMTypeRef llvm_type = lb_type(p->module, type); LLVMValueRef ptr = LLVMBuildAlloca(p->builder, llvm_type, name); - // unsigned alignment = 16; // TODO(bill): Make this configurable - unsigned alignment = cast(unsigned)lb_alignof(llvm_type); + unsigned alignment = cast(unsigned)gb_max(type_align_of(type), lb_alignof(llvm_type)); + if (is_type_matrix(type)) { + alignment *= 2; // NOTE(bill): Just in case + } LLVMSetAlignment(ptr, alignment); LLVMPositionBuilderAtEnd(p->builder, p->curr_block->block); diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 1b41be2a3..3971c0ca6 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1224,12 +1224,14 @@ lbValue lb_emit_ptr_offset(lbProcedure *p, lbValue ptr, lbValue index) { lbValue lb_emit_matrix_epi(lbProcedure *p, lbValue s, isize row, isize column) { Type *t = s.type; GB_ASSERT(is_type_pointer(t)); - Type *st = base_type(type_deref(t)); - GB_ASSERT_MSG(is_type_matrix(st), "%s", type_to_string(st)); + Type *mt = base_type(type_deref(t)); + GB_ASSERT_MSG(is_type_matrix(mt), "%s", type_to_string(mt)); - Type *ptr = base_array_type(st); + Type *ptr = base_array_type(mt); + + i64 stride_elems = matrix_type_stride_in_elems(mt); - isize index = row*column; + isize index = row + column*stride_elems; GB_ASSERT(0 <= index); LLVMValueRef indices[2] = { diff --git a/src/types.cpp b/src/types.cpp index fd9b20c91..8bce69cf3 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1249,6 +1249,7 @@ bool is_type_matrix(Type *t) { } i64 matrix_type_stride(Type *t) { + // TODO(bill): precompute matrix stride t = base_type(t); GB_ASSERT(t->kind == Type_Matrix); i64 align = type_align_of(t); @@ -1258,6 +1259,7 @@ i64 matrix_type_stride(Type *t) { } i64 matrix_type_stride_in_elems(Type *t) { + // TODO(bill): precompute matrix stride t = base_type(t); GB_ASSERT(t->kind == Type_Matrix); i64 stride = matrix_type_stride(t); @@ -1266,6 +1268,7 @@ i64 matrix_type_stride_in_elems(Type *t) { i64 matrix_type_total_elems(Type *t) { + // TODO(bill): precompute matrix total elems t = base_type(t); GB_ASSERT(t->kind == Type_Matrix); i64 size = type_size_of(t); -- cgit v1.2.3 From 662cbaf425a54127dea206c3a35d776853bac169 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 19 Oct 2021 12:13:19 +0100 Subject: Support indexing matrices --- core/runtime/error_checks.odin | 23 +++++++++++++++ src/check_expr.cpp | 66 ++++++++++++++++++++++++++++++++++++++++-- src/checker.cpp | 1 + src/llvm_backend.hpp | 1 + src/llvm_backend_expr.cpp | 54 +++++++++++++++++++++++++++++++++- src/llvm_backend_general.cpp | 30 +++++++++++++++++++ src/llvm_backend_utility.cpp | 31 ++++++++++++++++++++ src/types.cpp | 4 +++ 8 files changed, 206 insertions(+), 4 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/core/runtime/error_checks.odin b/core/runtime/error_checks.odin index bdd010b50..7f1aeb2d7 100644 --- a/core/runtime/error_checks.odin +++ b/core/runtime/error_checks.odin @@ -96,6 +96,29 @@ dynamic_array_expr_error :: proc "contextless" (file: string, line, column: i32, } +matrix_bounds_check_error :: proc "contextless" (file: string, line, column: i32, row_index, column_index, row_count, column_count: int) { + if 0 <= row_index && row_index < row_count && + 0 <= column_index && column_index < column_count { + return + } + handle_error :: proc "contextless" (file: string, line, column: i32, row_index, column_index, row_count, column_count: int) { + print_caller_location(Source_Code_Location{file, line, column, ""}) + print_string(" Matrix indices [") + print_i64(i64(row_index)) + print_string(", ") + print_i64(i64(column_index)) + print_string(" is out of bounds range [0..<") + print_i64(i64(row_count)) + print_string(", 0..<") + print_i64(i64(column_count)) + print_string("]") + print_byte('\n') + bounds_trap() + } + handle_error(file, line, column, row_index, column_index, row_count, column_count) +} + + type_assertion_check :: proc "contextless" (ok: bool, file: string, line, column: i32, from, to: typeid) { if ok { return diff --git a/src/check_expr.cpp b/src/check_expr.cpp index a75334e6c..73e1a7e51 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -6367,8 +6367,7 @@ bool check_set_index_data(Operand *o, Type *t, bool indirection, i64 *max_count, *max_count = t->Matrix.column_count; if (indirection) { o->mode = Addressing_Variable; - } else if (o->mode != Addressing_Variable && - o->mode != Addressing_Constant) { + } else if (o->mode != Addressing_Variable) { o->mode = Addressing_Value; } o->type = alloc_type_array(t->Matrix.elem, t->Matrix.row_count); @@ -6672,7 +6671,68 @@ void check_promote_optional_ok(CheckerContext *c, Operand *x, Type **val_type_, void check_matrix_index_expr(CheckerContext *c, Operand *o, Ast *node, Type *type_hint) { - error(node, "TODO: matrix index expressions"); + ast_node(ie, MatrixIndexExpr, node); + + check_expr(c, o, ie->expr); + node->viral_state_flags |= ie->expr->viral_state_flags; + if (o->mode == Addressing_Invalid) { + o->expr = node; + return; + } + + Type *t = base_type(type_deref(o->type)); + bool is_ptr = is_type_pointer(o->type); + bool is_const = o->mode == Addressing_Constant; + + if (t->kind != Type_Matrix) { + gbString str = expr_to_string(o->expr); + gbString type_str = type_to_string(o->type); + defer (gb_string_free(str)); + defer (gb_string_free(type_str)); + if (is_const) { + error(o->expr, "Cannot use matrix indexing on constant '%s' of type '%s'", str, type_str); + } else { + error(o->expr, "Cannot use matrix indexing on '%s' of type '%s'", str, type_str); + } + o->mode = Addressing_Invalid; + o->expr = node; + return; + } + o->type = t->Matrix.elem; + if (is_ptr) { + o->mode = Addressing_Variable; + } else if (o->mode != Addressing_Variable) { + o->mode = Addressing_Value; + } + + if (ie->row_index == nullptr) { + gbString str = expr_to_string(o->expr); + error(o->expr, "Missing row index for '%s'", str); + gb_string_free(str); + o->mode = Addressing_Invalid; + o->expr = node; + return; + } + if (ie->column_index == nullptr) { + gbString str = expr_to_string(o->expr); + error(o->expr, "Missing column index for '%s'", str); + gb_string_free(str); + o->mode = Addressing_Invalid; + o->expr = node; + return; + } + + i64 row_count = t->Matrix.row_count; + i64 column_count = t->Matrix.column_count; + + i64 row_index = 0; + i64 column_index = 0; + bool row_ok = check_index_value(c, t, false, ie->row_index, row_count, &row_index, nullptr); + bool column_ok = check_index_value(c, t, false, ie->column_index, column_count, &column_index, nullptr); + + + gb_unused(row_ok); + gb_unused(column_ok); } diff --git a/src/checker.cpp b/src/checker.cpp index c0e6d47c0..23597167b 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2022,6 +2022,7 @@ void generate_minimum_dependency_set(Checker *c, Entity *start) { String bounds_check_entities[] = { // Bounds checking related procedures str_lit("bounds_check_error"), + str_lit("matrix_bounds_check_error"), str_lit("slice_expr_error_hi"), str_lit("slice_expr_error_lo_hi"), str_lit("multi_pointer_slice_expr_error"), diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 73ddad797..9041e7621 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -333,6 +333,7 @@ lbValue lb_emit_array_ep(lbProcedure *p, lbValue s, lbValue index); lbValue lb_emit_deep_field_gep(lbProcedure *p, lbValue e, Selection sel); lbValue lb_emit_deep_field_ev(lbProcedure *p, lbValue e, Selection sel); +lbValue lb_emit_matrix_ep(lbProcedure *p, lbValue s, lbValue row, lbValue column); lbValue lb_emit_matrix_epi(lbProcedure *p, lbValue s, isize row, isize column); lbValue lb_emit_matrix_ev(lbProcedure *p, lbValue s, isize row, isize column); diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index ed98c6845..bcbb77355 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1727,7 +1727,7 @@ lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { } if (is_type_matrix(dst) && !is_type_matrix(src)) { - GB_ASSERT(dst->Matrix.row_count == dst->Matrix.column_count); + GB_ASSERT_MSG(dst->Matrix.row_count == dst->Matrix.column_count, "%s <- %s", type_to_string(dst), type_to_string(src)); Type *elem = base_array_type(dst); lbValue e = lb_emit_conv(p, value, elem); @@ -2805,6 +2805,10 @@ lbValue lb_build_expr(lbProcedure *p, Ast *expr) { case_ast_node(ie, IndexExpr, expr); return lb_addr_load(p, lb_build_addr(p, expr)); case_end; + + case_ast_node(ie, MatrixIndexExpr, expr); + return lb_addr_load(p, lb_build_addr(p, expr)); + case_end; case_ast_node(ia, InlineAsmExpr, expr); Type *t = type_of_expr(expr); @@ -3304,6 +3308,25 @@ lbAddr lb_build_addr(lbProcedure *p, Ast *expr) { lbValue v = lb_emit_ptr_offset(p, elem, index); return lb_addr(v); } + + case Type_Matrix: { + lbValue matrix = {}; + matrix = lb_build_addr_ptr(p, ie->expr); + if (deref) { + matrix = lb_emit_load(p, matrix); + } + lbValue index = lb_build_expr(p, ie->index); + index = lb_emit_conv(p, index, t_int); + lbValue elem = lb_emit_matrix_ep(p, matrix, lb_const_int(p->module, t_int, 0), index); + elem = lb_emit_conv(p, elem, alloc_type_pointer(type_of_expr(expr))); + + auto index_tv = type_and_value_of_expr(ie->index); + if (index_tv.mode != Addressing_Constant) { + lbValue len = lb_const_int(p->module, t_int, t->Matrix.column_count); + lb_emit_bounds_check(p, ast_token(ie->index), index, len); + } + return lb_addr(elem); + } case Type_Basic: { // Basic_string @@ -3326,6 +3349,35 @@ lbAddr lb_build_addr(lbProcedure *p, Ast *expr) { } } case_end; + + case_ast_node(ie, MatrixIndexExpr, expr); + Type *t = base_type(type_of_expr(ie->expr)); + + bool deref = is_type_pointer(t); + t = base_type(type_deref(t)); + + lbValue m = {}; + m = lb_build_addr_ptr(p, ie->expr); + if (deref) { + m = lb_emit_load(p, m); + } + lbValue row_index = lb_build_expr(p, ie->row_index); + lbValue column_index = lb_build_expr(p, ie->column_index); + row_index = lb_emit_conv(p, row_index, t_int); + column_index = lb_emit_conv(p, column_index, t_int); + lbValue elem = lb_emit_matrix_ep(p, m, row_index, column_index); + + auto row_index_tv = type_and_value_of_expr(ie->row_index); + auto column_index_tv = type_and_value_of_expr(ie->column_index); + if (row_index_tv.mode != Addressing_Constant || column_index_tv.mode != Addressing_Constant) { + lbValue row_count = lb_const_int(p->module, t_int, t->Matrix.row_count); + lbValue column_count = lb_const_int(p->module, t_int, t->Matrix.column_count); + lb_emit_matrix_bounds_check(p, ast_token(ie->row_index), row_index, column_index, row_count, column_count); + } + return lb_addr(elem); + + + case_end; case_ast_node(se, SliceExpr, expr); diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 63a63349a..01221cad6 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -419,6 +419,36 @@ void lb_emit_bounds_check(lbProcedure *p, Token token, lbValue index, lbValue le lb_emit_runtime_call(p, "bounds_check_error", args); } +void lb_emit_matrix_bounds_check(lbProcedure *p, Token token, lbValue row_index, lbValue column_index, lbValue row_count, lbValue column_count) { + if (build_context.no_bounds_check) { + return; + } + if ((p->state_flags & StateFlag_no_bounds_check) != 0) { + return; + } + + row_index = lb_emit_conv(p, row_index, t_int); + column_index = lb_emit_conv(p, column_index, t_int); + row_count = lb_emit_conv(p, row_count, t_int); + column_count = lb_emit_conv(p, column_count, t_int); + + lbValue file = lb_find_or_add_entity_string(p->module, get_file_path_string(token.pos.file_id)); + lbValue line = lb_const_int(p->module, t_i32, token.pos.line); + lbValue column = lb_const_int(p->module, t_i32, token.pos.column); + + auto args = array_make(permanent_allocator(), 7); + args[0] = file; + args[1] = line; + args[2] = column; + args[3] = row_index; + args[4] = column_index; + args[5] = row_count; + args[6] = column_count; + + lb_emit_runtime_call(p, "matrix_bounds_check_error", args); +} + + void lb_emit_multi_pointer_slice_bounds_check(lbProcedure *p, Token token, lbValue low, lbValue high) { if (build_context.no_bounds_check) { return; diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 3971c0ca6..c7e9e1742 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1249,6 +1249,37 @@ lbValue lb_emit_matrix_epi(lbProcedure *p, lbValue s, isize row, isize column) { return res; } +lbValue lb_emit_matrix_ep(lbProcedure *p, lbValue s, lbValue row, lbValue column) { + Type *t = s.type; + GB_ASSERT(is_type_pointer(t)); + Type *mt = base_type(type_deref(t)); + GB_ASSERT_MSG(is_type_matrix(mt), "%s", type_to_string(mt)); + + Type *ptr = base_array_type(mt); + + LLVMValueRef stride_elems = lb_const_int(p->module, t_int, matrix_type_stride_in_elems(mt)).value; + + row = lb_emit_conv(p, row, t_int); + column = lb_emit_conv(p, column, t_int); + + LLVMValueRef index = LLVMBuildAdd(p->builder, row.value, LLVMBuildMul(p->builder, column.value, stride_elems, ""), ""); + + LLVMValueRef indices[2] = { + LLVMConstInt(lb_type(p->module, t_int), 0, false), + index, + }; + + lbValue res = {}; + if (lb_is_const(s)) { + res.value = LLVMConstGEP(s.value, indices, gb_count_of(indices)); + } else { + res.value = LLVMBuildGEP(p->builder, s.value, indices, gb_count_of(indices), ""); + } + res.type = alloc_type_pointer(ptr); + return res; +} + + lbValue lb_emit_matrix_ev(lbProcedure *p, lbValue s, isize row, isize column) { Type *st = base_type(s.type); GB_ASSERT_MSG(is_type_matrix(st), "%s", type_to_string(st)); diff --git a/src/types.cpp b/src/types.cpp index 8e64a10c1..ec094b4ff 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1726,6 +1726,8 @@ bool is_type_indexable(Type *t) { return true; case Type_RelativeSlice: return true; + case Type_Matrix: + return true; } return false; } @@ -1743,6 +1745,8 @@ bool is_type_sliceable(Type *t) { return false; case Type_RelativeSlice: return true; + case Type_Matrix: + return false; } return false; } -- cgit v1.2.3 From 306bdf8869f2c9676e73acbf477a302c08137087 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 25 Oct 2021 00:46:50 +0100 Subject: Update alignment rules for `matrix` types as a compromise to keep zero padding --- src/check_builtin.cpp | 4 +- src/check_type.cpp | 12 +-- src/llvm_backend_expr.cpp | 23 ++++- src/llvm_backend_general.cpp | 8 +- src/llvm_backend_utility.cpp | 2 +- src/types.cpp | 205 ++++++++++++++++++++++++------------------- 6 files changed, 147 insertions(+), 107 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 9b94be002..2373317c3 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -2083,8 +2083,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 } i64 max_count = xt->Array.count*yt->Array.count; - if (max_count > MAX_MATRIX_ELEMENT_COUNT) { - error(call, "Product of the array lengths exceed the maximum matrix element count, got %d, expected a maximum of %d", cast(int)max_count, MAX_MATRIX_ELEMENT_COUNT); + if (max_count > MATRIX_ELEMENT_COUNT_MAX) { + error(call, "Product of the array lengths exceed the maximum matrix element count, got %d, expected a maximum of %d", cast(int)max_count, MATRIX_ELEMENT_COUNT_MAX); return false; } diff --git a/src/check_type.cpp b/src/check_type.cpp index 21c8a9f19..813990020 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2226,21 +2226,21 @@ void check_matrix_type(CheckerContext *ctx, Type **type, Ast *node) { generic_column = column.type; } - if (row_count < MIN_MATRIX_ELEMENT_COUNT && generic_row == nullptr) { + if (row_count < MATRIX_ELEMENT_COUNT_MIN && generic_row == nullptr) { gbString s = expr_to_string(row.expr); - error(row.expr, "Invalid matrix row count, expected %d+ rows, got %s", MIN_MATRIX_ELEMENT_COUNT, s); + error(row.expr, "Invalid matrix row count, expected %d+ rows, got %s", MATRIX_ELEMENT_COUNT_MIN, s); gb_string_free(s); } - if (column_count < MIN_MATRIX_ELEMENT_COUNT && generic_column == nullptr) { + if (column_count < MATRIX_ELEMENT_COUNT_MIN && generic_column == nullptr) { gbString s = expr_to_string(column.expr); - error(column.expr, "Invalid matrix column count, expected %d+ rows, got %s", MIN_MATRIX_ELEMENT_COUNT, s); + error(column.expr, "Invalid matrix column count, expected %d+ rows, got %s", MATRIX_ELEMENT_COUNT_MIN, s); gb_string_free(s); } - if (row_count*column_count > MAX_MATRIX_ELEMENT_COUNT) { + if (row_count*column_count > MATRIX_ELEMENT_COUNT_MAX) { i64 element_count = row_count*column_count; - error(column.expr, "Matrix types are limited to a maximum of %d elements, got %lld", MAX_MATRIX_ELEMENT_COUNT, cast(long long)element_count); + error(column.expr, "Matrix types are limited to a maximum of %d elements, got %lld", MATRIX_ELEMENT_COUNT_MAX, cast(long long)element_count); } if (!is_type_valid_for_matrix_elems(elem)) { diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 9c114882e..fa2b0b084 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -511,10 +511,16 @@ LLVMValueRef lb_matrix_to_vector(lbProcedure *p, lbValue matrix) { unsigned total_count = cast(unsigned)matrix_type_total_internal_elems(mt); LLVMTypeRef total_matrix_type = LLVMVectorType(elem_type, total_count); +#if 1 LLVMValueRef ptr = lb_address_from_load_or_generate_local(p, matrix).value; LLVMValueRef matrix_vector_ptr = LLVMBuildPointerCast(p->builder, ptr, LLVMPointerType(total_matrix_type, 0), ""); LLVMValueRef matrix_vector = LLVMBuildLoad(p->builder, matrix_vector_ptr, ""); + LLVMSetAlignment(matrix_vector, cast(unsigned)type_align_of(mt)); return matrix_vector; +#else + LLVMValueRef matrix_vector = LLVMBuildBitCast(p->builder, matrix.value, total_matrix_type, ""); + return matrix_vector; +#endif } LLVMValueRef lb_matrix_trimmed_vector_mask(lbProcedure *p, Type *mt) { @@ -524,7 +530,6 @@ LLVMValueRef lb_matrix_trimmed_vector_mask(lbProcedure *p, Type *mt) { unsigned stride = cast(unsigned)matrix_type_stride_in_elems(mt); unsigned row_count = cast(unsigned)mt->Matrix.row_count; unsigned column_count = cast(unsigned)mt->Matrix.column_count; - unsigned mask_elems_index = 0; auto mask_elems = slice_make(permanent_allocator(), row_count*column_count); for (unsigned j = 0; j < column_count; j++) { @@ -540,7 +545,17 @@ LLVMValueRef lb_matrix_trimmed_vector_mask(lbProcedure *p, Type *mt) { LLVMValueRef lb_matrix_to_trimmed_vector(lbProcedure *p, lbValue m) { LLVMValueRef vector = lb_matrix_to_vector(p, m); - LLVMValueRef mask = lb_matrix_trimmed_vector_mask(p, m.type); + + Type *mt = base_type(m.type); + GB_ASSERT(mt->kind == Type_Matrix); + + unsigned stride = cast(unsigned)matrix_type_stride_in_elems(mt); + unsigned row_count = cast(unsigned)mt->Matrix.row_count; + if (stride == row_count) { + return vector; + } + + LLVMValueRef mask = lb_matrix_trimmed_vector_mask(p, mt); LLVMValueRef trimmed_vector = LLVMBuildShuffleVector(p->builder, vector, LLVMGetUndef(LLVMTypeOf(vector)), mask, ""); return trimmed_vector; } @@ -791,7 +806,7 @@ lbValue lb_emit_matrix_mul_vector(lbProcedure *p, lbValue lhs, lbValue rhs, Type for (unsigned row_index = 0; row_index < column_count; row_index++) { LLVMValueRef value = lb_emit_struct_ev(p, rhs, row_index).value; - LLVMValueRef row = llvm_splat(p, value, row_count); + LLVMValueRef row = llvm_vector_broadcast(p, value, row_count); v_rows[row_index] = row; } @@ -866,7 +881,7 @@ lbValue lb_emit_vector_mul_matrix(lbProcedure *p, lbValue lhs, lbValue rhs, Type for (unsigned column_index = 0; column_index < row_count; column_index++) { LLVMValueRef value = lb_emit_struct_ev(p, lhs, column_index).value; - LLVMValueRef row = llvm_splat(p, value, column_count); + LLVMValueRef row = llvm_vector_broadcast(p, value, column_count); v_rows[column_index] = row; } diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 01221cad6..7aa7c7cdd 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -512,8 +512,7 @@ void lb_emit_slice_bounds_check(lbProcedure *p, Token token, lbValue low, lbValu } } -bool lb_try_update_alignment(lbValue ptr, unsigned alignment) { - LLVMValueRef addr_ptr = ptr.value; +bool lb_try_update_alignment(LLVMValueRef addr_ptr, unsigned alignment) { if (LLVMIsAGlobalValue(addr_ptr) || LLVMIsAAllocaInst(addr_ptr) || LLVMIsALoadInst(addr_ptr)) { if (LLVMGetAlignment(addr_ptr) < alignment) { if (LLVMIsAAllocaInst(addr_ptr) || LLVMIsAGlobalValue(addr_ptr)) { @@ -525,6 +524,11 @@ bool lb_try_update_alignment(lbValue ptr, unsigned alignment) { return false; } +bool lb_try_update_alignment(lbValue ptr, unsigned alignment) { + return lb_try_update_alignment(ptr.value, alignment); +} + + bool lb_try_vector_cast(lbModule *m, lbValue ptr, LLVMTypeRef *vector_type_) { Type *array_type = base_type(type_deref(ptr.type)); GB_ASSERT(is_type_array_like(array_type)); diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 6754ce798..e458c0692 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1526,7 +1526,7 @@ LLVMValueRef llvm_mask_zero(lbModule *m, unsigned count) { return LLVMConstNull(LLVMVectorType(lb_type(m, t_u32), count)); } -LLVMValueRef llvm_splat(lbProcedure *p, LLVMValueRef value, unsigned count) { +LLVMValueRef llvm_vector_broadcast(lbProcedure *p, LLVMValueRef value, unsigned count) { GB_ASSERT(count > 0); if (LLVMIsConstant(value)) { LLVMValueRef single = LLVMConstVector(&value, 1); diff --git a/src/types.cpp b/src/types.cpp index 3abcebdfb..bfedb5381 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -360,8 +360,8 @@ enum TypeInfoFlag : u32 { enum : int { - MIN_MATRIX_ELEMENT_COUNT = 1, - MAX_MATRIX_ELEMENT_COUNT = 16, + MATRIX_ELEMENT_COUNT_MIN = 1, + MATRIX_ELEMENT_COUNT_MAX = 16, }; @@ -700,6 +700,74 @@ bool is_type_pointer(Type *t); bool is_type_slice(Type *t); bool is_type_integer(Type *t); bool type_set_offsets(Type *t); +Type *base_type(Type *t); + +i64 type_size_of_internal(Type *t, TypePath *path); +i64 type_align_of_internal(Type *t, TypePath *path); + + +// IMPORTANT TODO(bill): SHould this TypePath code be removed since type cycle checking is handled much earlier on? + +struct TypePath { + Array path; // Entity_TypeName; + bool failure; +}; + + +void type_path_init(TypePath *tp) { + tp->path.allocator = heap_allocator(); +} + +void type_path_free(TypePath *tp) { + array_free(&tp->path); +} + +void type_path_print_illegal_cycle(TypePath *tp, isize start_index) { + GB_ASSERT(tp != nullptr); + + GB_ASSERT(start_index < tp->path.count); + Entity *e = tp->path[start_index]; + GB_ASSERT(e != nullptr); + error(e->token, "Illegal type declaration cycle of `%.*s`", LIT(e->token.string)); + // NOTE(bill): Print cycle, if it's deep enough + for (isize j = start_index; j < tp->path.count; j++) { + Entity *e = tp->path[j]; + error(e->token, "\t%.*s refers to", LIT(e->token.string)); + } + // NOTE(bill): This will only print if the path count > 1 + error(e->token, "\t%.*s", LIT(e->token.string)); + tp->failure = true; + e->type->failure = true; + base_type(e->type)->failure = true; +} + +bool type_path_push(TypePath *tp, Type *t) { + GB_ASSERT(tp != nullptr); + if (t->kind != Type_Named) { + return false; + } + Entity *e = t->Named.type_name; + + for (isize i = 0; i < tp->path.count; i++) { + Entity *p = tp->path[i]; + if (p == e) { + type_path_print_illegal_cycle(tp, i); + } + } + + array_add(&tp->path, e); + return true; +} + +void type_path_pop(TypePath *tp) { + if (tp != nullptr && tp->path.count > 0) { + array_pop(&tp->path); + } +} + + +#define FAILURE_SIZE 0 +#define FAILURE_ALIGNMENT 0 void init_type_mutex(void) { mutex_init(&g_type_mutex); @@ -1251,6 +1319,42 @@ bool is_type_matrix(Type *t) { return t->kind == Type_Matrix; } +i64 matrix_align_of(Type *t, struct TypePath *tp) { + t = base_type(t); + GB_ASSERT(t->kind == Type_Matrix); + + Type *elem = t->Matrix.elem; + i64 row_count = gb_max(t->Matrix.row_count, 1); + + bool pop = type_path_push(tp, elem); + if (tp->failure) { + return FAILURE_ALIGNMENT; + } + + i64 elem_align = type_align_of_internal(elem, tp); + if (pop) type_path_pop(tp); + + i64 elem_size = type_size_of(elem); + + + // NOTE(bill, 2021-10-25): The alignment strategy here is to have zero padding + // It would be better for performance to pad each column so that each column + // could be maximally aligned but as a compromise, having no padding will be + // beneficial to third libraries that assume no padding + + i64 total_expected_size = row_count*t->Matrix.column_count*elem_size; + // i64 min_alignment = prev_pow2(elem_align * row_count); + i64 min_alignment = prev_pow2(total_expected_size); + while ((total_expected_size % min_alignment) != 0) { + min_alignment >>= 1; + } + GB_ASSERT(min_alignment >= elem_align); + + i64 align = gb_min(min_alignment, build_context.max_align); + return align; +} + + i64 matrix_type_stride_in_bytes(Type *t, struct TypePath *tp) { t = base_type(t); GB_ASSERT(t->kind == Type_Matrix); @@ -1266,21 +1370,16 @@ i64 matrix_type_stride_in_bytes(Type *t, struct TypePath *tp) { } else { elem_size = type_size_of(t->Matrix.elem); } - i64 stride_in_bytes = 0; + // NOTE(bill, 2021-10-25): The alignment strategy here is to have zero padding + // It would be better for performance to pad each column so that each column + // could be maximally aligned but as a compromise, having no padding will be + // beneficial to third libraries that assume no padding i64 row_count = t->Matrix.row_count; -#if 0 - if (row_count == 1) { - stride_in_bytes = elem_size; - } else { - i64 matrix_alignment = type_align_of(t); - stride_in_bytes = align_formula(elem_size*row_count, matrix_alignment); - } -#else stride_in_bytes = elem_size*row_count; -#endif + t->Matrix.stride_in_bytes = stride_in_bytes; return stride_in_bytes; } @@ -2969,71 +3068,6 @@ Slice struct_fields_index_by_increasing_offset(gbAllocator allocator, Type - -// IMPORTANT TODO(bill): SHould this TypePath code be removed since type cycle checking is handled much earlier on? - -struct TypePath { - Array path; // Entity_TypeName; - bool failure; -}; - - -void type_path_init(TypePath *tp) { - tp->path.allocator = heap_allocator(); -} - -void type_path_free(TypePath *tp) { - array_free(&tp->path); -} - -void type_path_print_illegal_cycle(TypePath *tp, isize start_index) { - GB_ASSERT(tp != nullptr); - - GB_ASSERT(start_index < tp->path.count); - Entity *e = tp->path[start_index]; - GB_ASSERT(e != nullptr); - error(e->token, "Illegal type declaration cycle of `%.*s`", LIT(e->token.string)); - // NOTE(bill): Print cycle, if it's deep enough - for (isize j = start_index; j < tp->path.count; j++) { - Entity *e = tp->path[j]; - error(e->token, "\t%.*s refers to", LIT(e->token.string)); - } - // NOTE(bill): This will only print if the path count > 1 - error(e->token, "\t%.*s", LIT(e->token.string)); - tp->failure = true; - e->type->failure = true; - base_type(e->type)->failure = true; -} - -bool type_path_push(TypePath *tp, Type *t) { - GB_ASSERT(tp != nullptr); - if (t->kind != Type_Named) { - return false; - } - Entity *e = t->Named.type_name; - - for (isize i = 0; i < tp->path.count; i++) { - Entity *p = tp->path[i]; - if (p == e) { - type_path_print_illegal_cycle(tp, i); - } - } - - array_add(&tp->path, e); - return true; -} - -void type_path_pop(TypePath *tp) { - if (tp != nullptr && tp->path.count > 0) { - array_pop(&tp->path); - } -} - - -#define FAILURE_SIZE 0 -#define FAILURE_ALIGNMENT 0 - - i64 type_size_of_internal (Type *t, TypePath *path); i64 type_align_of_internal(Type *t, TypePath *path); i64 type_size_of(Type *t); @@ -3260,21 +3294,8 @@ i64 type_align_of_internal(Type *t, TypePath *path) { return gb_clamp(next_pow2(type_size_of_internal(t, path)), 1, build_context.max_align); } - case Type_Matrix: { - Type *elem = t->Matrix.elem; - i64 row_count = gb_max(t->Matrix.row_count, 1); - - bool pop = type_path_push(path, elem); - if (path->failure) { - return FAILURE_ALIGNMENT; - } - // elem align is used here rather than size as it make a little more sense - i64 elem_align = type_align_of_internal(elem, path); - if (pop) type_path_pop(path); - - i64 align = gb_min(next_pow2(elem_align * row_count), build_context.max_align); - return align; - } + case Type_Matrix: + return matrix_align_of(t, path); case Type_RelativePointer: return type_align_of_internal(t->RelativePointer.base_integer, path); -- cgit v1.2.3 From 7d715fe113c6bffaf004fe21a6e9723913e38bb6 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 25 Oct 2021 16:05:22 +0100 Subject: Add `ODIN_LLVM_MINIMUM_VERSION_12` --- src/llvm_backend.cpp | 10 +++------- src/llvm_backend.hpp | 12 ++++++++++++ src/llvm_backend_general.cpp | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) (limited to 'src/llvm_backend_general.cpp') diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index a853a6224..4d1245c98 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -21,12 +21,6 @@ #include "llvm_backend_stmt.cpp" #include "llvm_backend_proc.cpp" -#if LLVM_VERSION_MAJOR < 11 -#error "LLVM Version 11 is the minimum required" -#elif LLVM_VERSION_MAJOR == 12 && !(LLVM_VERSION_MINOR > 0 || LLVM_VERSION_PATCH > 0) -#error "If LLVM Version 12.x.y is wanted, at least LLVM 12.0.1 is required" -#endif - void lb_add_foreign_library_path(lbModule *m, Entity *e) { if (e == nullptr) { @@ -1214,7 +1208,9 @@ void lb_generate_code(lbGenerator *gen) { // x86-64-v2: (close to Nehalem) CMPXCHG16B, LAHF-SAHF, POPCNT, SSE3, SSE4.1, SSE4.2, SSSE3 // x86-64-v3: (close to Haswell) AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE // x86-64-v4: AVX512F, AVX512BW, AVX512CD, AVX512DQ, AVX512VL - llvm_cpu = "x86-64-v2"; + if (ODIN_LLVM_MINIMUM_VERSION_12) { + llvm_cpu = "x86-64-v2"; + } } // GB_ASSERT_MSG(LLVMTargetHasAsmBackend(target)); diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 4aea88f47..9aa9920f2 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -30,6 +30,18 @@ #include #endif +#if LLVM_VERSION_MAJOR < 11 +#error "LLVM Version 11 is the minimum required" +#elif LLVM_VERSION_MAJOR == 12 && !(LLVM_VERSION_MINOR > 0 || LLVM_VERSION_PATCH > 0) +#error "If LLVM Version 12.x.y is wanted, at least LLVM 12.0.1 is required" +#endif + +#if LLVM_VERSION_MAJOR > 12 || (LLVM_VERSION_MAJOR == 12 && LLVM_VERSION_MINOR >= 0 && LLVM_VERSION_PATCH > 0) +#define ODIN_LLVM_MINIMUM_VERSION_12 1 +#else +#define ODIN_LLVM_MINIMUM_VERSION_12 0 +#endif + struct lbProcedure; struct lbValue { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 7aa7c7cdd..b1c1f924b 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -2065,7 +2065,7 @@ LLVMAttributeRef lb_create_enum_attribute_with_type(LLVMContextRef ctx, char con unsigned kind = 0; String s = make_string_c(name); - #if (LLVM_VERSION_MAJOR > 12 || (LLVM_VERSION_MAJOR == 12 && (LLVM_VERSION_MINOR > 0 || LLVM_VERSION_PATCH >= 1))) + #if ODIN_LLVM_MINIMUM_VERSION_12 kind = LLVMGetEnumAttributeKindForName(name, s.len); GB_ASSERT_MSG(kind != 0, "unknown attribute: %s", name); return LLVMCreateTypeAttribute(ctx, kind, type); -- cgit v1.2.3