head 1.1; branch 1.1.1; access; symbols netbsd-11-0-RC5:1.1.1.4 netbsd-11-0-RC4:1.1.1.4 netbsd-11-0-RC3:1.1.1.4 netbsd-11-0-RC2:1.1.1.4 netbsd-11-0-RC1:1.1.1.4 netbsd-11:1.1.1.4.0.4 netbsd-11-base:1.1.1.4 netbsd-10-1-RELEASE:1.1.1.4 netbsd-9-4-RELEASE:1.1.1.2 netbsd-10-0-RELEASE:1.1.1.4 netbsd-10-0-RC6:1.1.1.4 netbsd-10-0-RC5:1.1.1.4 netbsd-10-0-RC4:1.1.1.4 netbsd-10-0-RC3:1.1.1.4 netbsd-10-0-RC2:1.1.1.4 netbsd-10-0-RC1:1.1.1.4 netbsd-10:1.1.1.4.0.2 netbsd-10-base:1.1.1.4 netbsd-9-3-RELEASE:1.1.1.2 mesa-21-3-7:1.1.1.4 netbsd-9-2-RELEASE:1.1.1.2 netbsd-9-1-RELEASE:1.1.1.2 netbsd-9-0-RELEASE:1.1.1.2 netbsd-9-0-RC2:1.1.1.2 netbsd-9-0-RC1:1.1.1.2 mesalib-19-1-7:1.1.1.3 netbsd-9:1.1.1.2.0.2 netbsd-9-base:1.1.1.2 mesa-18-3-6:1.1.1.2 mesa-18-3-4:1.1.1.1 xorg:1.1.1; locks; strict; comment @// @; 1.1 date 2019.03.10.03.42.41; author mrg; state Exp; branches 1.1.1.1; next ; commitid r12jo1Nf3ebQKLeB; 1.1.1.1 date 2019.03.10.03.42.41; author mrg; state Exp; branches; next 1.1.1.2; commitid r12jo1Nf3ebQKLeB; 1.1.1.2 date 2019.06.01.07.41.01; author mrg; state Exp; branches; next 1.1.1.3; commitid HVa6uub5uYIfqspB; 1.1.1.3 date 2019.09.24.17.39.50; author maya; state Exp; branches; next 1.1.1.4; commitid KJXusGl8fi9AAhEB; 1.1.1.4 date 2022.05.09.01.23.37; author mrg; state Exp; branches; next ; commitid UEBs6hNk81DdQjDD; desc @@ 1.1 log @Initial revision @ text @/* * Copyright © 2010 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "compiler/glsl/ir.h" #include "brw_fs.h" #include "brw_fs_surface_builder.h" #include "brw_nir.h" #include "util/u_math.h" using namespace brw; using namespace brw::surface_access; void fs_visitor::emit_nir_code() { /* emit the arrays used for inputs and outputs - load/store intrinsics will * be converted to reads/writes of these arrays */ nir_setup_outputs(); nir_setup_uniforms(); nir_emit_system_values(); nir_emit_impl(nir_shader_get_entrypoint((nir_shader *)nir)); } void fs_visitor::nir_setup_outputs() { if (stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_FRAGMENT) return; unsigned vec4s[VARYING_SLOT_TESS_MAX] = { 0, }; /* Calculate the size of output registers in a separate pass, before * allocating them. With ARB_enhanced_layouts, multiple output variables * may occupy the same slot, but have different type sizes. */ nir_foreach_variable(var, &nir->outputs) { const int loc = var->data.driver_location; const unsigned var_vec4s = var->data.compact ? DIV_ROUND_UP(glsl_get_length(var->type), 4) : type_size_vec4(var->type); vec4s[loc] = MAX2(vec4s[loc], var_vec4s); } for (unsigned loc = 0; loc < ARRAY_SIZE(vec4s);) { if (vec4s[loc] == 0) { loc++; continue; } unsigned reg_size = vec4s[loc]; /* Check if there are any ranges that start within this range and extend * past it. If so, include them in this allocation. */ for (unsigned i = 1; i < reg_size; i++) reg_size = MAX2(vec4s[i + loc] + i, reg_size); fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * reg_size); for (unsigned i = 0; i < reg_size; i++) outputs[loc + i] = offset(reg, bld, 4 * i); loc += reg_size; } } void fs_visitor::nir_setup_uniforms() { /* Only the first compile gets to set up uniforms. */ if (push_constant_loc) { assert(pull_constant_loc); return; } uniforms = nir->num_uniforms / 4; if (stage == MESA_SHADER_COMPUTE) { /* Add a uniform for the thread local id. It must be the last uniform * on the list. */ assert(uniforms == prog_data->nr_params); uint32_t *param = brw_stage_prog_data_add_params(prog_data, 1); *param = BRW_PARAM_BUILTIN_SUBGROUP_ID; subgroup_id = fs_reg(UNIFORM, uniforms++, BRW_REGISTER_TYPE_UD); } } static bool emit_system_values_block(nir_block *block, fs_visitor *v) { fs_reg *reg; nir_foreach_instr(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_vertex_id: case nir_intrinsic_load_base_vertex: unreachable("should be lowered by nir_lower_system_values()."); case nir_intrinsic_load_vertex_id_zero_base: case nir_intrinsic_load_is_indexed_draw: case nir_intrinsic_load_first_vertex: case nir_intrinsic_load_instance_id: case nir_intrinsic_load_base_instance: case nir_intrinsic_load_draw_id: unreachable("should be lowered by brw_nir_lower_vs_inputs()."); case nir_intrinsic_load_invocation_id: if (v->stage == MESA_SHADER_TESS_CTRL) break; assert(v->stage == MESA_SHADER_GEOMETRY); reg = &v->nir_system_values[SYSTEM_VALUE_INVOCATION_ID]; if (reg->file == BAD_FILE) { const fs_builder abld = v->bld.annotate("gl_InvocationID", NULL); fs_reg g1(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD)); fs_reg iid = abld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.SHR(iid, g1, brw_imm_ud(27u)); *reg = iid; } break; case nir_intrinsic_load_sample_pos: assert(v->stage == MESA_SHADER_FRAGMENT); reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_POS]; if (reg->file == BAD_FILE) *reg = *v->emit_samplepos_setup(); break; case nir_intrinsic_load_sample_id: assert(v->stage == MESA_SHADER_FRAGMENT); reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_ID]; if (reg->file == BAD_FILE) *reg = *v->emit_sampleid_setup(); break; case nir_intrinsic_load_sample_mask_in: assert(v->stage == MESA_SHADER_FRAGMENT); assert(v->devinfo->gen >= 7); reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_MASK_IN]; if (reg->file == BAD_FILE) *reg = *v->emit_samplemaskin_setup(); break; case nir_intrinsic_load_work_group_id: assert(v->stage == MESA_SHADER_COMPUTE); reg = &v->nir_system_values[SYSTEM_VALUE_WORK_GROUP_ID]; if (reg->file == BAD_FILE) *reg = *v->emit_cs_work_group_id_setup(); break; case nir_intrinsic_load_helper_invocation: assert(v->stage == MESA_SHADER_FRAGMENT); reg = &v->nir_system_values[SYSTEM_VALUE_HELPER_INVOCATION]; if (reg->file == BAD_FILE) { const fs_builder abld = v->bld.annotate("gl_HelperInvocation", NULL); /* On Gen6+ (gl_HelperInvocation is only exposed on Gen7+) the * pixel mask is in g1.7 of the thread payload. * * We move the per-channel pixel enable bit to the low bit of each * channel by shifting the byte containing the pixel mask by the * vector immediate 0x76543210UV. * * The region of <1,8,0> reads only 1 byte (the pixel masks for * subspans 0 and 1) in SIMD8 and an additional byte (the pixel * masks for 2 and 3) in SIMD16. */ fs_reg shifted = abld.vgrf(BRW_REGISTER_TYPE_UW, 1); for (unsigned i = 0; i < DIV_ROUND_UP(v->dispatch_width, 16); i++) { const fs_builder hbld = abld.group(MIN2(16, v->dispatch_width), i); hbld.SHR(offset(shifted, hbld, i), stride(retype(brw_vec1_grf(1 + i, 7), BRW_REGISTER_TYPE_UB), 1, 8, 0), brw_imm_v(0x76543210)); } /* A set bit in the pixel mask means the channel is enabled, but * that is the opposite of gl_HelperInvocation so we need to invert * the mask. * * The negate source-modifier bit of logical instructions on Gen8+ * performs 1's complement negation, so we can use that instead of * a NOT instruction. */ fs_reg inverted = negate(shifted); if (v->devinfo->gen < 8) { inverted = abld.vgrf(BRW_REGISTER_TYPE_UW); abld.NOT(inverted, shifted); } /* We then resolve the 0/1 result to 0/~0 boolean values by ANDing * with 1 and negating. */ fs_reg anded = abld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.AND(anded, inverted, brw_imm_uw(1)); fs_reg dst = abld.vgrf(BRW_REGISTER_TYPE_D, 1); abld.MOV(dst, negate(retype(anded, BRW_REGISTER_TYPE_D))); *reg = dst; } break; default: break; } } return true; } void fs_visitor::nir_emit_system_values() { nir_system_values = ralloc_array(mem_ctx, fs_reg, SYSTEM_VALUE_MAX); for (unsigned i = 0; i < SYSTEM_VALUE_MAX; i++) { nir_system_values[i] = fs_reg(); } /* Always emit SUBGROUP_INVOCATION. Dead code will clean it up if we * never end up using it. */ { const fs_builder abld = bld.annotate("gl_SubgroupInvocation", NULL); fs_reg ® = nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]; reg = abld.vgrf(BRW_REGISTER_TYPE_UW); const fs_builder allbld8 = abld.group(8, 0).exec_all(); allbld8.MOV(reg, brw_imm_v(0x76543210)); if (dispatch_width > 8) allbld8.ADD(byte_offset(reg, 16), reg, brw_imm_uw(8u)); if (dispatch_width > 16) { const fs_builder allbld16 = abld.group(16, 0).exec_all(); allbld16.ADD(byte_offset(reg, 32), reg, brw_imm_uw(16u)); } } nir_function_impl *impl = nir_shader_get_entrypoint((nir_shader *)nir); nir_foreach_block(block, impl) emit_system_values_block(block, this); } /* * Returns a type based on a reference_type (word, float, half-float) and a * given bit_size. * * Reference BRW_REGISTER_TYPE are HF,F,DF,W,D,UW,UD. * * @@FIXME: 64-bit return types are always DF on integer types to maintain * compability with uses of DF previously to the introduction of int64 * support. */ static brw_reg_type brw_reg_type_from_bit_size(const unsigned bit_size, const brw_reg_type reference_type) { switch(reference_type) { case BRW_REGISTER_TYPE_HF: case BRW_REGISTER_TYPE_F: case BRW_REGISTER_TYPE_DF: switch(bit_size) { case 16: return BRW_REGISTER_TYPE_HF; case 32: return BRW_REGISTER_TYPE_F; case 64: return BRW_REGISTER_TYPE_DF; default: unreachable("Invalid bit size"); } case BRW_REGISTER_TYPE_B: case BRW_REGISTER_TYPE_W: case BRW_REGISTER_TYPE_D: case BRW_REGISTER_TYPE_Q: switch(bit_size) { case 8: return BRW_REGISTER_TYPE_B; case 16: return BRW_REGISTER_TYPE_W; case 32: return BRW_REGISTER_TYPE_D; case 64: return BRW_REGISTER_TYPE_Q; default: unreachable("Invalid bit size"); } case BRW_REGISTER_TYPE_UB: case BRW_REGISTER_TYPE_UW: case BRW_REGISTER_TYPE_UD: case BRW_REGISTER_TYPE_UQ: switch(bit_size) { case 8: return BRW_REGISTER_TYPE_UB; case 16: return BRW_REGISTER_TYPE_UW; case 32: return BRW_REGISTER_TYPE_UD; case 64: return BRW_REGISTER_TYPE_UQ; default: unreachable("Invalid bit size"); } default: unreachable("Unknown type"); } } void fs_visitor::nir_emit_impl(nir_function_impl *impl) { nir_locals = ralloc_array(mem_ctx, fs_reg, impl->reg_alloc); for (unsigned i = 0; i < impl->reg_alloc; i++) { nir_locals[i] = fs_reg(); } foreach_list_typed(nir_register, reg, node, &impl->registers) { unsigned array_elems = reg->num_array_elems == 0 ? 1 : reg->num_array_elems; unsigned size = array_elems * reg->num_components; const brw_reg_type reg_type = brw_reg_type_from_bit_size(reg->bit_size, BRW_REGISTER_TYPE_F); nir_locals[reg->index] = bld.vgrf(reg_type, size); } nir_ssa_values = reralloc(mem_ctx, nir_ssa_values, fs_reg, impl->ssa_alloc); nir_emit_cf_list(&impl->body); } void fs_visitor::nir_emit_cf_list(exec_list *list) { exec_list_validate(list); foreach_list_typed(nir_cf_node, node, node, list) { switch (node->type) { case nir_cf_node_if: nir_emit_if(nir_cf_node_as_if(node)); break; case nir_cf_node_loop: nir_emit_loop(nir_cf_node_as_loop(node)); break; case nir_cf_node_block: nir_emit_block(nir_cf_node_as_block(node)); break; default: unreachable("Invalid CFG node block"); } } } void fs_visitor::nir_emit_if(nir_if *if_stmt) { /* first, put the condition into f0 */ fs_inst *inst = bld.MOV(bld.null_reg_d(), retype(get_nir_src(if_stmt->condition), BRW_REGISTER_TYPE_D)); inst->conditional_mod = BRW_CONDITIONAL_NZ; bld.IF(BRW_PREDICATE_NORMAL); nir_emit_cf_list(&if_stmt->then_list); /* note: if the else is empty, dead CF elimination will remove it */ bld.emit(BRW_OPCODE_ELSE); nir_emit_cf_list(&if_stmt->else_list); bld.emit(BRW_OPCODE_ENDIF); if (devinfo->gen < 7) limit_dispatch_width(16, "Non-uniform control flow unsupported " "in SIMD32 mode."); } void fs_visitor::nir_emit_loop(nir_loop *loop) { bld.emit(BRW_OPCODE_DO); nir_emit_cf_list(&loop->body); bld.emit(BRW_OPCODE_WHILE); if (devinfo->gen < 7) limit_dispatch_width(16, "Non-uniform control flow unsupported " "in SIMD32 mode."); } void fs_visitor::nir_emit_block(nir_block *block) { nir_foreach_instr(instr, block) { nir_emit_instr(instr); } } void fs_visitor::nir_emit_instr(nir_instr *instr) { const fs_builder abld = bld.annotate(NULL, instr); switch (instr->type) { case nir_instr_type_alu: nir_emit_alu(abld, nir_instr_as_alu(instr)); break; case nir_instr_type_deref: /* Derefs can exist for images but they do nothing */ break; case nir_instr_type_intrinsic: switch (stage) { case MESA_SHADER_VERTEX: nir_emit_vs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_TESS_CTRL: nir_emit_tcs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_TESS_EVAL: nir_emit_tes_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_GEOMETRY: nir_emit_gs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_FRAGMENT: nir_emit_fs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; case MESA_SHADER_COMPUTE: nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; default: unreachable("unsupported shader stage"); } break; case nir_instr_type_tex: nir_emit_texture(abld, nir_instr_as_tex(instr)); break; case nir_instr_type_load_const: nir_emit_load_const(abld, nir_instr_as_load_const(instr)); break; case nir_instr_type_ssa_undef: /* We create a new VGRF for undefs on every use (by handling * them in get_nir_src()), rather than for each definition. * This helps register coalescing eliminate MOVs from undef. */ break; case nir_instr_type_jump: nir_emit_jump(abld, nir_instr_as_jump(instr)); break; default: unreachable("unknown instruction type"); } } /** * Recognizes a parent instruction of nir_op_extract_* and changes the type to * match instr. */ bool fs_visitor::optimize_extract_to_float(nir_alu_instr *instr, const fs_reg &result) { if (!instr->src[0].src.is_ssa || !instr->src[0].src.ssa->parent_instr) return false; if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu) return false; nir_alu_instr *src0 = nir_instr_as_alu(instr->src[0].src.ssa->parent_instr); if (src0->op != nir_op_extract_u8 && src0->op != nir_op_extract_u16 && src0->op != nir_op_extract_i8 && src0->op != nir_op_extract_i16) return false; nir_const_value *element = nir_src_as_const_value(src0->src[1].src); assert(element != NULL); /* Element type to extract.*/ const brw_reg_type type = brw_int_type( src0->op == nir_op_extract_u16 || src0->op == nir_op_extract_i16 ? 2 : 1, src0->op == nir_op_extract_i16 || src0->op == nir_op_extract_i8); fs_reg op0 = get_nir_src(src0->src[0].src); op0.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[src0->op].input_types[0] | nir_src_bit_size(src0->src[0].src))); op0 = offset(op0, bld, src0->src[0].swizzle[0]); set_saturate(instr->dest.saturate, bld.MOV(result, subscript(op0, type, element->u32[0]))); return true; } bool fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr, const fs_reg &result) { if (!instr->src[0].src.is_ssa || instr->src[0].src.ssa->parent_instr->type != nir_instr_type_intrinsic) return false; nir_intrinsic_instr *src0 = nir_instr_as_intrinsic(instr->src[0].src.ssa->parent_instr); if (src0->intrinsic != nir_intrinsic_load_front_face) return false; nir_const_value *value1 = nir_src_as_const_value(instr->src[1].src); if (!value1 || fabsf(value1->f32[0]) != 1.0f) return false; nir_const_value *value2 = nir_src_as_const_value(instr->src[2].src); if (!value2 || fabsf(value2->f32[0]) != 1.0f) return false; fs_reg tmp = vgrf(glsl_type::int_type); if (devinfo->gen >= 6) { /* Bit 15 of g0.0 is 0 if the polygon is front facing. */ fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W)); /* For (gl_FrontFacing ? 1.0 : -1.0), emit: * * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D * * and negate g0.0<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0). * * This negation looks like it's safe in practice, because bits 0:4 will * surely be TRIANGLES */ if (value1->f32[0] == -1.0f) { g0.negate = true; } bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1), g0, brw_imm_uw(0x3f80)); } else { /* Bit 31 of g1.6 is 0 if the polygon is front facing. */ fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D)); /* For (gl_FrontFacing ? 1.0 : -1.0), emit: * * or(8) tmp<1>D g1.6<0,1,0>D 0x3f800000D * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D * * and negate g1.6<0,1,0>D for (gl_FrontFacing ? -1.0 : 1.0). * * This negation looks like it's safe in practice, because bits 0:4 will * surely be TRIANGLES */ if (value1->f32[0] == -1.0f) { g1_6.negate = true; } bld.OR(tmp, g1_6, brw_imm_d(0x3f800000)); } bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, brw_imm_d(0xbf800000)); return true; } static void emit_find_msb_using_lzd(const fs_builder &bld, const fs_reg &result, const fs_reg &src, bool is_signed) { fs_inst *inst; fs_reg temp = src; if (is_signed) { /* LZD of an absolute value source almost always does the right * thing. There are two problem values: * * * 0x80000000. Since abs(0x80000000) == 0x80000000, LZD returns * 0. However, findMSB(int(0x80000000)) == 30. * * * 0xffffffff. Since abs(0xffffffff) == 1, LZD returns * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says: * * For a value of zero or negative one, -1 will be returned. * * * Negative powers of two. LZD(abs(-(1<src[0].negate = true; } static brw_rnd_mode brw_rnd_mode_from_nir_op (const nir_op op) { switch (op) { case nir_op_f2f16_rtz: return BRW_RND_MODE_RTZ; case nir_op_f2f16_rtne: return BRW_RND_MODE_RTNE; default: unreachable("Operation doesn't support rounding mode"); } } void fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr) { struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key; fs_inst *inst; fs_reg result = get_nir_dest(instr->dest.dest); result.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[instr->op].output_type | nir_dest_bit_size(instr->dest.dest))); fs_reg op[4]; for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { op[i] = get_nir_src(instr->src[i].src); op[i].type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[instr->op].input_types[i] | nir_src_bit_size(instr->src[i].src))); op[i].abs = instr->src[i].abs; op[i].negate = instr->src[i].negate; } /* We get a bunch of mov's out of the from_ssa pass and they may still * be vectorized. We'll handle them as a special-case. We'll also * handle vecN here because it's basically the same thing. */ switch (instr->op) { case nir_op_imov: case nir_op_fmov: case nir_op_vec2: case nir_op_vec3: case nir_op_vec4: { fs_reg temp = result; bool need_extra_copy = false; for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { if (!instr->src[i].src.is_ssa && instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) { need_extra_copy = true; temp = bld.vgrf(result.type, 4); break; } } for (unsigned i = 0; i < 4; i++) { if (!(instr->dest.write_mask & (1 << i))) continue; if (instr->op == nir_op_imov || instr->op == nir_op_fmov) { inst = bld.MOV(offset(temp, bld, i), offset(op[0], bld, instr->src[0].swizzle[i])); } else { inst = bld.MOV(offset(temp, bld, i), offset(op[i], bld, instr->src[i].swizzle[0])); } inst->saturate = instr->dest.saturate; } /* In this case the source and destination registers were the same, * so we need to insert an extra set of moves in order to deal with * any swizzling. */ if (need_extra_copy) { for (unsigned i = 0; i < 4; i++) { if (!(instr->dest.write_mask & (1 << i))) continue; bld.MOV(offset(result, bld, i), offset(temp, bld, i)); } } return; } default: break; } /* At this point, we have dealt with any instruction that operates on * more than a single channel. Therefore, we can just adjust the source * and destination registers for that channel and emit the instruction. */ unsigned channel = 0; if (nir_op_infos[instr->op].output_size == 0) { /* Since NIR is doing the scalarizing for us, we should only ever see * vectorized operations with a single channel. */ assert(util_bitcount(instr->dest.write_mask) == 1); channel = ffs(instr->dest.write_mask) - 1; result = offset(result, bld, channel); } for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { assert(nir_op_infos[instr->op].input_sizes[i] < 2); op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]); } switch (instr->op) { case nir_op_i2f32: case nir_op_u2f32: if (optimize_extract_to_float(instr, result)) return; inst = bld.MOV(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_f2f16_rtne: case nir_op_f2f16_rtz: bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(brw_rnd_mode_from_nir_op(instr->op))); /* fallthrough */ /* In theory, it would be better to use BRW_OPCODE_F32TO16. Depending * on the HW gen, it is a special hw opcode or just a MOV, and * brw_F32TO16 (at brw_eu_emit) would do the work to chose. * * But if we want to use that opcode, we need to provide support on * different optimizations and lowerings. As right now HF support is * only for gen8+, it will be better to use directly the MOV, and use * BRW_OPCODE_F32TO16 when/if we work for HF support on gen7. */ case nir_op_f2f16: inst = bld.MOV(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_b2i: case nir_op_b2f: op[0].type = BRW_REGISTER_TYPE_D; op[0].negate = !op[0].negate; /* fallthrough */ case nir_op_f2f64: case nir_op_f2i64: case nir_op_f2u64: case nir_op_i2f64: case nir_op_i2i64: case nir_op_u2f64: case nir_op_u2u64: /* CHV PRM, vol07, 3D Media GPGPU Engine, Register Region Restrictions: * * "When source or destination is 64b (...), regioning in Align1 * must follow these rules: * * 1. Source and destination horizontal stride must be aligned to * the same qword. * (...)" * * This means that conversions from bit-sizes smaller than 64-bit to * 64-bit need to have the source data elements aligned to 64-bit. * This restriction does not apply to BDW and later. */ if (nir_dest_bit_size(instr->dest.dest) == 64 && nir_src_bit_size(instr->src[0].src) < 64 && (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) { fs_reg tmp = bld.vgrf(result.type, 1); tmp = subscript(tmp, op[0].type, 0); inst = bld.MOV(tmp, op[0]); inst = bld.MOV(result, tmp); inst->saturate = instr->dest.saturate; break; } /* fallthrough */ case nir_op_f2f32: case nir_op_f2i32: case nir_op_f2u32: case nir_op_f2i16: case nir_op_f2u16: case nir_op_i2i32: case nir_op_u2u32: case nir_op_i2i16: case nir_op_u2u16: case nir_op_i2f16: case nir_op_u2f16: case nir_op_i2i8: case nir_op_u2u8: inst = bld.MOV(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fsign: { assert(!instr->dest.saturate); if (op[0].abs) { /* Straightforward since the source can be assumed to be either * strictly >= 0 or strictly <= 0 depending on the setting of the * negate flag. */ set_condmod(BRW_CONDITIONAL_NZ, bld.MOV(result, op[0])); inst = (op[0].negate) ? bld.MOV(result, brw_imm_f(-1.0f)) : bld.MOV(result, brw_imm_f(1.0f)); set_predicate(BRW_PREDICATE_NORMAL, inst); } else if (type_sz(op[0].type) < 8) { /* AND(val, 0x80000000) gives the sign bit. * * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not * zero. */ bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ); fs_reg result_int = retype(result, BRW_REGISTER_TYPE_UD); op[0].type = BRW_REGISTER_TYPE_UD; result.type = BRW_REGISTER_TYPE_UD; bld.AND(result_int, op[0], brw_imm_ud(0x80000000u)); inst = bld.OR(result_int, result_int, brw_imm_ud(0x3f800000u)); inst->predicate = BRW_PREDICATE_NORMAL; } else { /* For doubles we do the same but we need to consider: * * - 2-src instructions can't operate with 64-bit immediates * - The sign is encoded in the high 32-bit of each DF * - We need to produce a DF result. */ fs_reg zero = vgrf(glsl_type::double_type); bld.MOV(zero, setup_imm_df(bld, 0.0)); bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ); bld.MOV(result, zero); fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1); bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1), brw_imm_ud(0x80000000u)); set_predicate(BRW_PREDICATE_NORMAL, bld.OR(r, r, brw_imm_ud(0x3ff00000u))); } break; } case nir_op_isign: { /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1). * -> non-negative val generates 0x00000000. * Predicated OR sets 1 if val is positive. */ uint32_t bit_size = nir_dest_bit_size(instr->dest.dest); assert(bit_size == 32 || bit_size == 16); fs_reg zero = bit_size == 32 ? brw_imm_d(0) : brw_imm_w(0); fs_reg one = bit_size == 32 ? brw_imm_d(1) : brw_imm_w(1); fs_reg shift = bit_size == 32 ? brw_imm_d(31) : brw_imm_w(15); bld.CMP(bld.null_reg_d(), op[0], zero, BRW_CONDITIONAL_G); bld.ASR(result, op[0], shift); inst = bld.OR(result, result, one); inst->predicate = BRW_PREDICATE_NORMAL; break; } case nir_op_frcp: inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fexp2: inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_flog2: inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fsin: inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fcos: inst = bld.emit(SHADER_OPCODE_COS, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fddx: if (fs_key->high_quality_derivatives) { inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]); } else { inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]); } inst->saturate = instr->dest.saturate; break; case nir_op_fddx_fine: inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fddx_coarse: inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fddy: if (fs_key->high_quality_derivatives) { inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]); } else { inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]); } inst->saturate = instr->dest.saturate; break; case nir_op_fddy_fine: inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fddy_coarse: inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_iadd: case nir_op_fadd: inst = bld.ADD(result, op[0], op[1]); inst->saturate = instr->dest.saturate; break; case nir_op_fmul: inst = bld.MUL(result, op[0], op[1]); inst->saturate = instr->dest.saturate; break; case nir_op_imul: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.MUL(result, op[0], op[1]); break; case nir_op_imul_high: case nir_op_umul_high: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]); break; case nir_op_idiv: case nir_op_udiv: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]); break; case nir_op_uadd_carry: unreachable("Should have been lowered by carry_to_arith()."); case nir_op_usub_borrow: unreachable("Should have been lowered by borrow_to_arith()."); case nir_op_umod: case nir_op_irem: /* According to the sign table for INT DIV in the Ivy Bridge PRM, it * appears that our hardware just does the right thing for signed * remainder. */ assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]); break; case nir_op_imod: { /* Get a regular C-style remainder. If a % b == 0, set the predicate. */ bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]); /* Math instructions don't support conditional mod */ inst = bld.MOV(bld.null_reg_d(), result); inst->conditional_mod = BRW_CONDITIONAL_NZ; /* Now, we need to determine if signs of the sources are different. * When we XOR the sources, the top bit is 0 if they are the same and 1 * if they are different. We can then use a conditional modifier to * turn that into a predicate. This leads us to an XOR.l instruction. * * Technically, according to the PRM, you're not allowed to use .l on a * XOR instruction. However, emperical experiments and Curro's reading * of the simulator source both indicate that it's safe. */ fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D); inst = bld.XOR(tmp, op[0], op[1]); inst->predicate = BRW_PREDICATE_NORMAL; inst->conditional_mod = BRW_CONDITIONAL_L; /* If the result of the initial remainder operation is non-zero and the * two sources have different signs, add in a copy of op[1] to get the * final integer modulus value. */ inst = bld.ADD(result, result, op[1]); inst->predicate = BRW_PREDICATE_NORMAL; break; } case nir_op_flt: case nir_op_fge: case nir_op_feq: case nir_op_fne: { fs_reg dest = result; const uint32_t bit_size = nir_src_bit_size(instr->src[0].src); if (bit_size != 32) dest = bld.vgrf(op[0].type, 1); brw_conditional_mod cond; switch (instr->op) { case nir_op_flt: cond = BRW_CONDITIONAL_L; break; case nir_op_fge: cond = BRW_CONDITIONAL_GE; break; case nir_op_feq: cond = BRW_CONDITIONAL_Z; break; case nir_op_fne: cond = BRW_CONDITIONAL_NZ; break; default: unreachable("bad opcode"); } bld.CMP(dest, op[0], op[1], cond); if (bit_size > 32) { bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0)); } else if(bit_size < 32) { /* When we convert the result to 32-bit we need to be careful and do * it as a signed conversion to get sign extension (for 32-bit true) */ const brw_reg_type src_type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D); bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type)); } break; } case nir_op_ilt: case nir_op_ult: case nir_op_ige: case nir_op_uge: case nir_op_ieq: case nir_op_ine: { fs_reg dest = result; const uint32_t bit_size = nir_src_bit_size(instr->src[0].src); if (bit_size != 32) dest = bld.vgrf(op[0].type, 1); brw_conditional_mod cond; switch (instr->op) { case nir_op_ilt: case nir_op_ult: cond = BRW_CONDITIONAL_L; break; case nir_op_ige: case nir_op_uge: cond = BRW_CONDITIONAL_GE; break; case nir_op_ieq: cond = BRW_CONDITIONAL_Z; break; case nir_op_ine: cond = BRW_CONDITIONAL_NZ; break; default: unreachable("bad opcode"); } bld.CMP(dest, op[0], op[1], cond); if (bit_size > 32) { bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0)); } else if (bit_size < 32) { /* When we convert the result to 32-bit we need to be careful and do * it as a signed conversion to get sign extension (for 32-bit true) */ const brw_reg_type src_type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D); bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type)); } break; } case nir_op_inot: if (devinfo->gen >= 8) { op[0] = resolve_source_modifiers(op[0]); } bld.NOT(result, op[0]); break; case nir_op_ixor: if (devinfo->gen >= 8) { op[0] = resolve_source_modifiers(op[0]); op[1] = resolve_source_modifiers(op[1]); } bld.XOR(result, op[0], op[1]); break; case nir_op_ior: if (devinfo->gen >= 8) { op[0] = resolve_source_modifiers(op[0]); op[1] = resolve_source_modifiers(op[1]); } bld.OR(result, op[0], op[1]); break; case nir_op_iand: if (devinfo->gen >= 8) { op[0] = resolve_source_modifiers(op[0]); op[1] = resolve_source_modifiers(op[1]); } bld.AND(result, op[0], op[1]); break; case nir_op_fdot2: case nir_op_fdot3: case nir_op_fdot4: case nir_op_ball_fequal2: case nir_op_ball_iequal2: case nir_op_ball_fequal3: case nir_op_ball_iequal3: case nir_op_ball_fequal4: case nir_op_ball_iequal4: case nir_op_bany_fnequal2: case nir_op_bany_inequal2: case nir_op_bany_fnequal3: case nir_op_bany_inequal3: case nir_op_bany_fnequal4: case nir_op_bany_inequal4: unreachable("Lowered by nir_lower_alu_reductions"); case nir_op_fnoise1_1: case nir_op_fnoise1_2: case nir_op_fnoise1_3: case nir_op_fnoise1_4: case nir_op_fnoise2_1: case nir_op_fnoise2_2: case nir_op_fnoise2_3: case nir_op_fnoise2_4: case nir_op_fnoise3_1: case nir_op_fnoise3_2: case nir_op_fnoise3_3: case nir_op_fnoise3_4: case nir_op_fnoise4_1: case nir_op_fnoise4_2: case nir_op_fnoise4_3: case nir_op_fnoise4_4: unreachable("not reached: should be handled by lower_noise"); case nir_op_ldexp: unreachable("not reached: should be handled by ldexp_to_arith()"); case nir_op_fsqrt: inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_frsq: inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_i2b: case nir_op_f2b: { uint32_t bit_size = nir_src_bit_size(instr->src[0].src); if (bit_size == 64) { /* two-argument instructions can't take 64-bit immediates */ fs_reg zero; fs_reg tmp; if (instr->op == nir_op_f2b) { zero = vgrf(glsl_type::double_type); tmp = vgrf(glsl_type::double_type); bld.MOV(zero, setup_imm_df(bld, 0.0)); } else { zero = vgrf(glsl_type::int64_t_type); tmp = vgrf(glsl_type::int64_t_type); bld.MOV(zero, brw_imm_q(0)); } /* A SIMD16 execution needs to be split in two instructions, so use * a vgrf instead of the flag register as dst so instruction splitting * works */ bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ); bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0)); } else { fs_reg zero; if (bit_size == 32) { zero = instr->op == nir_op_f2b ? brw_imm_f(0.0f) : brw_imm_d(0); } else { assert(bit_size == 16); zero = instr->op == nir_op_f2b ? retype(brw_imm_w(0), BRW_REGISTER_TYPE_HF) : brw_imm_w(0); } bld.CMP(result, op[0], zero, BRW_CONDITIONAL_NZ); } break; } case nir_op_ftrunc: inst = bld.RNDZ(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fceil: { op[0].negate = !op[0].negate; fs_reg temp = vgrf(glsl_type::float_type); bld.RNDD(temp, op[0]); temp.negate = true; inst = bld.MOV(result, temp); inst->saturate = instr->dest.saturate; break; } case nir_op_ffloor: inst = bld.RNDD(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_ffract: inst = bld.FRC(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fround_even: inst = bld.RNDE(result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_fquantize2f16: { fs_reg tmp16 = bld.vgrf(BRW_REGISTER_TYPE_D); fs_reg tmp32 = bld.vgrf(BRW_REGISTER_TYPE_F); fs_reg zero = bld.vgrf(BRW_REGISTER_TYPE_F); /* The destination stride must be at least as big as the source stride. */ tmp16.type = BRW_REGISTER_TYPE_W; tmp16.stride = 2; /* Check for denormal */ fs_reg abs_src0 = op[0]; abs_src0.abs = true; bld.CMP(bld.null_reg_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)), BRW_CONDITIONAL_L); /* Get the appropriately signed zero */ bld.AND(retype(zero, BRW_REGISTER_TYPE_UD), retype(op[0], BRW_REGISTER_TYPE_UD), brw_imm_ud(0x80000000)); /* Do the actual F32 -> F16 -> F32 conversion */ bld.emit(BRW_OPCODE_F32TO16, tmp16, op[0]); bld.emit(BRW_OPCODE_F16TO32, tmp32, tmp16); /* Select that or zero based on normal status */ inst = bld.SEL(result, zero, tmp32); inst->predicate = BRW_PREDICATE_NORMAL; inst->saturate = instr->dest.saturate; break; } case nir_op_imin: case nir_op_umin: case nir_op_fmin: inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_L); inst->saturate = instr->dest.saturate; break; case nir_op_imax: case nir_op_umax: case nir_op_fmax: inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_GE); inst->saturate = instr->dest.saturate; break; case nir_op_pack_snorm_2x16: case nir_op_pack_snorm_4x8: case nir_op_pack_unorm_2x16: case nir_op_pack_unorm_4x8: case nir_op_unpack_snorm_2x16: case nir_op_unpack_snorm_4x8: case nir_op_unpack_unorm_2x16: case nir_op_unpack_unorm_4x8: case nir_op_unpack_half_2x16: case nir_op_pack_half_2x16: unreachable("not reached: should be handled by lower_packing_builtins"); case nir_op_unpack_half_2x16_split_x: inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_unpack_half_2x16_split_y: inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y, result, op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_pack_64_2x32_split: case nir_op_pack_32_2x16_split: bld.emit(FS_OPCODE_PACK, result, op[0], op[1]); break; case nir_op_unpack_64_2x32_split_x: case nir_op_unpack_64_2x32_split_y: { if (instr->op == nir_op_unpack_64_2x32_split_x) bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 0)); else bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 1)); break; } case nir_op_unpack_32_2x16_split_x: case nir_op_unpack_32_2x16_split_y: { if (instr->op == nir_op_unpack_32_2x16_split_x) bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0)); else bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1)); break; } case nir_op_fpow: inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]); inst->saturate = instr->dest.saturate; break; case nir_op_bitfield_reverse: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.BFREV(result, op[0]); break; case nir_op_bit_count: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.CBIT(result, op[0]); break; case nir_op_ufind_msb: { assert(nir_dest_bit_size(instr->dest.dest) < 64); emit_find_msb_using_lzd(bld, result, op[0], false); break; } case nir_op_ifind_msb: { assert(nir_dest_bit_size(instr->dest.dest) < 64); if (devinfo->gen < 7) { emit_find_msb_using_lzd(bld, result, op[0], true); } else { bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]); /* FBH counts from the MSB side, while GLSL's findMSB() wants the * count from the LSB side. If FBH didn't return an error * (0xFFFFFFFF), then subtract the result from 31 to convert the MSB * count into an LSB count. */ bld.CMP(bld.null_reg_d(), result, brw_imm_d(-1), BRW_CONDITIONAL_NZ); inst = bld.ADD(result, result, brw_imm_d(31)); inst->predicate = BRW_PREDICATE_NORMAL; inst->src[0].negate = true; } break; } case nir_op_find_lsb: assert(nir_dest_bit_size(instr->dest.dest) < 64); if (devinfo->gen < 7) { fs_reg temp = vgrf(glsl_type::int_type); /* (x & -x) generates a value that consists of only the LSB of x. * For all powers of 2, findMSB(y) == findLSB(y). */ fs_reg src = retype(op[0], BRW_REGISTER_TYPE_D); fs_reg negated_src = src; /* One must be negated, and the other must be non-negated. It * doesn't matter which is which. */ negated_src.negate = true; src.negate = false; bld.AND(temp, src, negated_src); emit_find_msb_using_lzd(bld, result, temp, false); } else { bld.FBL(result, op[0]); } break; case nir_op_ubitfield_extract: case nir_op_ibitfield_extract: unreachable("should have been lowered"); case nir_op_ubfe: case nir_op_ibfe: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.BFE(result, op[2], op[1], op[0]); break; case nir_op_bfm: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.BFI1(result, op[0], op[1]); break; case nir_op_bfi: assert(nir_dest_bit_size(instr->dest.dest) < 64); bld.BFI2(result, op[0], op[1], op[2]); break; case nir_op_bitfield_insert: unreachable("not reached: should have been lowered"); case nir_op_ishl: case nir_op_ishr: case nir_op_ushr: { fs_reg shift_count = op[1]; if (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo)) { if (op[1].file == VGRF && (result.type == BRW_REGISTER_TYPE_Q || result.type == BRW_REGISTER_TYPE_UQ)) { shift_count = fs_reg(VGRF, alloc.allocate(dispatch_width / 4), BRW_REGISTER_TYPE_UD); shift_count.stride = 2; bld.MOV(shift_count, op[1]); } } switch (instr->op) { case nir_op_ishl: bld.SHL(result, op[0], shift_count); break; case nir_op_ishr: bld.ASR(result, op[0], shift_count); break; case nir_op_ushr: bld.SHR(result, op[0], shift_count); break; default: unreachable("not reached"); } break; } case nir_op_pack_half_2x16_split: bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]); break; case nir_op_ffma: inst = bld.MAD(result, op[2], op[1], op[0]); inst->saturate = instr->dest.saturate; break; case nir_op_flrp: inst = bld.LRP(result, op[0], op[1], op[2]); inst->saturate = instr->dest.saturate; break; case nir_op_bcsel: if (optimize_frontfacing_ternary(instr, result)) return; bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ); inst = bld.SEL(result, op[1], op[2]); inst->predicate = BRW_PREDICATE_NORMAL; break; case nir_op_extract_u8: case nir_op_extract_i8: { nir_const_value *byte = nir_src_as_const_value(instr->src[1].src); assert(byte != NULL); /* The PRMs say: * * BDW+ * There is no direct conversion from B/UB to Q/UQ or Q/UQ to B/UB. * Use two instructions and a word or DWord intermediate integer type. */ if (nir_dest_bit_size(instr->dest.dest) == 64) { const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i8); if (instr->op == nir_op_extract_i8) { /* If we need to sign extend, extract to a word first */ fs_reg w_temp = bld.vgrf(BRW_REGISTER_TYPE_W); bld.MOV(w_temp, subscript(op[0], type, byte->u32[0])); bld.MOV(result, w_temp); } else { /* Otherwise use an AND with 0xff and a word type */ bld.AND(result, subscript(op[0], type, byte->u32[0] / 2), brw_imm_uw(0xff)); } } else { const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8); bld.MOV(result, subscript(op[0], type, byte->u32[0])); } break; } case nir_op_extract_u16: case nir_op_extract_i16: { const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16); nir_const_value *word = nir_src_as_const_value(instr->src[1].src); assert(word != NULL); bld.MOV(result, subscript(op[0], type, word->u32[0])); break; } default: unreachable("unhandled instruction"); } /* If we need to do a boolean resolve, replace the result with -(x & 1) * to sign extend the low bit to 0/~0 */ if (devinfo->gen <= 5 && (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) { fs_reg masked = vgrf(glsl_type::int_type); bld.AND(masked, result, brw_imm_d(1)); masked.negate = true; bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked); } } void fs_visitor::nir_emit_load_const(const fs_builder &bld, nir_load_const_instr *instr) { const brw_reg_type reg_type = brw_reg_type_from_bit_size(instr->def.bit_size, BRW_REGISTER_TYPE_D); fs_reg reg = bld.vgrf(reg_type, instr->def.num_components); switch (instr->def.bit_size) { case 8: for (unsigned i = 0; i < instr->def.num_components; i++) bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value.i8[i])); break; case 16: for (unsigned i = 0; i < instr->def.num_components; i++) bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value.i16[i])); break; case 32: for (unsigned i = 0; i < instr->def.num_components; i++) bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value.i32[i])); break; case 64: assert(devinfo->gen >= 7); if (devinfo->gen == 7) { /* We don't get 64-bit integer types until gen8 */ for (unsigned i = 0; i < instr->def.num_components; i++) { bld.MOV(retype(offset(reg, bld, i), BRW_REGISTER_TYPE_DF), setup_imm_df(bld, instr->value.f64[i])); } } else { for (unsigned i = 0; i < instr->def.num_components; i++) bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value.i64[i])); } break; default: unreachable("Invalid bit size"); } nir_ssa_values[instr->def.index] = reg; } fs_reg fs_visitor::get_nir_src(const nir_src &src) { fs_reg reg; if (src.is_ssa) { if (src.ssa->parent_instr->type == nir_instr_type_ssa_undef) { const brw_reg_type reg_type = brw_reg_type_from_bit_size(src.ssa->bit_size, BRW_REGISTER_TYPE_D); reg = bld.vgrf(reg_type, src.ssa->num_components); } else { reg = nir_ssa_values[src.ssa->index]; } } else { /* We don't handle indirects on locals */ assert(src.reg.indirect == NULL); reg = offset(nir_locals[src.reg.reg->index], bld, src.reg.base_offset * src.reg.reg->num_components); } if (nir_src_bit_size(src) == 64 && devinfo->gen == 7) { /* The only 64-bit type available on gen7 is DF, so use that. */ reg.type = BRW_REGISTER_TYPE_DF; } else { /* To avoid floating-point denorm flushing problems, set the type by * default to an integer type - instructions that need floating point * semantics will set this to F if they need to */ reg.type = brw_reg_type_from_bit_size(nir_src_bit_size(src), BRW_REGISTER_TYPE_D); } return reg; } /** * Return an IMM for constants; otherwise call get_nir_src() as normal. * * This function should not be called on any value which may be 64 bits. * We could theoretically support 64-bit on gen8+ but we choose not to * because it wouldn't work in general (no gen7 support) and there are * enough restrictions in 64-bit immediates that you can't take the return * value and treat it the same as the result of get_nir_src(). */ fs_reg fs_visitor::get_nir_src_imm(const nir_src &src) { nir_const_value *val = nir_src_as_const_value(src); assert(nir_src_bit_size(src) == 32); return val ? fs_reg(brw_imm_d(val->i32[0])) : get_nir_src(src); } fs_reg fs_visitor::get_nir_dest(const nir_dest &dest) { if (dest.is_ssa) { const brw_reg_type reg_type = brw_reg_type_from_bit_size(dest.ssa.bit_size, dest.ssa.bit_size == 8 ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_F); nir_ssa_values[dest.ssa.index] = bld.vgrf(reg_type, dest.ssa.num_components); return nir_ssa_values[dest.ssa.index]; } else { /* We don't handle indirects on locals */ assert(dest.reg.indirect == NULL); return offset(nir_locals[dest.reg.reg->index], bld, dest.reg.base_offset * dest.reg.reg->num_components); } } void fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst, unsigned wr_mask) { for (unsigned i = 0; i < 4; i++) { if (!((wr_mask >> i) & 1)) continue; fs_inst *new_inst = new(mem_ctx) fs_inst(inst); new_inst->dst = offset(new_inst->dst, bld, i); for (unsigned j = 0; j < new_inst->sources; j++) if (new_inst->src[j].file == VGRF) new_inst->src[j] = offset(new_inst->src[j], bld, i); bld.emit(new_inst); } } static fs_inst * emit_pixel_interpolater_send(const fs_builder &bld, enum opcode opcode, const fs_reg &dst, const fs_reg &src, const fs_reg &desc, glsl_interp_mode interpolation) { struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(bld.shader->stage_prog_data); fs_inst *inst = bld.emit(opcode, dst, src, desc); /* 2 floats per slot returned */ inst->size_written = 2 * dst.component_size(inst->exec_size); inst->pi_noperspective = interpolation == INTERP_MODE_NOPERSPECTIVE; wm_prog_data->pulls_bary = true; return inst; } /** * Computes 1 << x, given a D/UD register containing some value x. */ static fs_reg intexp2(const fs_builder &bld, const fs_reg &x) { assert(x.type == BRW_REGISTER_TYPE_UD || x.type == BRW_REGISTER_TYPE_D); fs_reg result = bld.vgrf(x.type, 1); fs_reg one = bld.vgrf(x.type, 1); bld.MOV(one, retype(brw_imm_d(1), one.type)); bld.SHL(result, one, x); return result; } void fs_visitor::emit_gs_end_primitive(const nir_src &vertex_count_nir_src) { assert(stage == MESA_SHADER_GEOMETRY); struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data); if (gs_compile->control_data_header_size_bits == 0) return; /* We can only do EndPrimitive() functionality when the control data * consists of cut bits. Fortunately, the only time it isn't is when the * output type is points, in which case EndPrimitive() is a no-op. */ if (gs_prog_data->control_data_format != GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) { return; } /* Cut bits use one bit per vertex. */ assert(gs_compile->control_data_bits_per_vertex == 1); fs_reg vertex_count = get_nir_src(vertex_count_nir_src); vertex_count.type = BRW_REGISTER_TYPE_UD; /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting * vertex n, 0 otherwise. So all we need to do here is mark bit * (vertex_count - 1) % 32 in the cut_bits register to indicate that * EndPrimitive() was called after emitting vertex (vertex_count - 1); * vec4_gs_visitor::emit_control_data_bits() will take care of the rest. * * Note that if EndPrimitive() is called before emitting any vertices, this * will cause us to set bit 31 of the control_data_bits register to 1. * That's fine because: * * - If max_vertices < 32, then vertex number 31 (zero-based) will never be * output, so the hardware will ignore cut bit 31. * * - If max_vertices == 32, then vertex number 31 is guaranteed to be the * last vertex, so setting cut bit 31 has no effect (since the primitive * is automatically ended when the GS terminates). * * - If max_vertices > 32, then the ir_emit_vertex visitor will reset the * control_data_bits register to 0 when the first vertex is emitted. */ const fs_builder abld = bld.annotate("end primitive"); /* control_data_bits |= 1 << ((vertex_count - 1) % 32) */ fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu)); fs_reg mask = intexp2(abld, prev_count); /* Note: we're relying on the fact that the GEN SHL instruction only pays * attention to the lower 5 bits of its second source argument, so on this * architecture, 1 << (vertex_count - 1) is equivalent to 1 << * ((vertex_count - 1) % 32). */ abld.OR(this->control_data_bits, this->control_data_bits, mask); } void fs_visitor::emit_gs_control_data_bits(const fs_reg &vertex_count) { assert(stage == MESA_SHADER_GEOMETRY); assert(gs_compile->control_data_bits_per_vertex != 0); struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data); const fs_builder abld = bld.annotate("emit control data bits"); const fs_builder fwa_bld = bld.exec_all(); /* We use a single UD register to accumulate control data bits (32 bits * for each of the SIMD8 channels). So we need to write a DWord (32 bits) * at a time. * * Unfortunately, the URB_WRITE_SIMD8 message uses 128-bit (OWord) offsets. * We have select a 128-bit group via the Global and Per-Slot Offsets, then * use the Channel Mask phase to enable/disable which DWord within that * group to write. (Remember, different SIMD8 channels may have emitted * different numbers of vertices, so we may need per-slot offsets.) * * Channel masking presents an annoying problem: we may have to replicate * the data up to 4 times: * * Msg = Handles, Per-Slot Offsets, Channel Masks, Data, Data, Data, Data. * * To avoid penalizing shaders that emit a small number of vertices, we * can avoid these sometimes: if the size of the control data header is * <= 128 bits, then there is only 1 OWord. All SIMD8 channels will land * land in the same 128-bit group, so we can skip per-slot offsets. * * Similarly, if the control data header is <= 32 bits, there is only one * DWord, so we can skip channel masks. */ enum opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8; fs_reg channel_mask, per_slot_offset; if (gs_compile->control_data_header_size_bits > 32) { opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED; channel_mask = vgrf(glsl_type::uint_type); } if (gs_compile->control_data_header_size_bits > 128) { opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT; per_slot_offset = vgrf(glsl_type::uint_type); } /* Figure out which DWord we're trying to write to using the formula: * * dword_index = (vertex_count - 1) * bits_per_vertex / 32 * * Since bits_per_vertex is a power of two, and is known at compile * time, this can be optimized to: * * dword_index = (vertex_count - 1) >> (6 - log2(bits_per_vertex)) */ if (opcode != SHADER_OPCODE_URB_WRITE_SIMD8) { fs_reg dword_index = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu)); unsigned log2_bits_per_vertex = util_last_bit(gs_compile->control_data_bits_per_vertex); abld.SHR(dword_index, prev_count, brw_imm_ud(6u - log2_bits_per_vertex)); if (per_slot_offset.file != BAD_FILE) { /* Set the per-slot offset to dword_index / 4, so that we'll write to * the appropriate OWord within the control data header. */ abld.SHR(per_slot_offset, dword_index, brw_imm_ud(2u)); } /* Set the channel masks to 1 << (dword_index % 4), so that we'll * write to the appropriate DWORD within the OWORD. */ fs_reg channel = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fwa_bld.AND(channel, dword_index, brw_imm_ud(3u)); channel_mask = intexp2(fwa_bld, channel); /* Then the channel masks need to be in bits 23:16. */ fwa_bld.SHL(channel_mask, channel_mask, brw_imm_ud(16u)); } /* Store the control data bits in the message payload and send it. */ int mlen = 2; if (channel_mask.file != BAD_FILE) mlen += 4; /* channel masks, plus 3 extra copies of the data */ if (per_slot_offset.file != BAD_FILE) mlen++; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen); fs_reg *sources = ralloc_array(mem_ctx, fs_reg, mlen); int i = 0; sources[i++] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD)); if (per_slot_offset.file != BAD_FILE) sources[i++] = per_slot_offset; if (channel_mask.file != BAD_FILE) sources[i++] = channel_mask; while (i < mlen) { sources[i++] = this->control_data_bits; } abld.LOAD_PAYLOAD(payload, sources, mlen, mlen); fs_inst *inst = abld.emit(opcode, reg_undef, payload); inst->mlen = mlen; /* We need to increment Global Offset by 256-bits to make room for * Broadwell's extra "Vertex Count" payload at the beginning of the * URB entry. Since this is an OWord message, Global Offset is counted * in 128-bit units, so we must set it to 2. */ if (gs_prog_data->static_vertex_count == -1) inst->offset = 2; } void fs_visitor::set_gs_stream_control_data_bits(const fs_reg &vertex_count, unsigned stream_id) { /* control_data_bits |= stream_id << ((2 * (vertex_count - 1)) % 32) */ /* Note: we are calling this *before* increasing vertex_count, so * this->vertex_count == vertex_count - 1 in the formula above. */ /* Stream mode uses 2 bits per vertex */ assert(gs_compile->control_data_bits_per_vertex == 2); /* Must be a valid stream */ assert(stream_id < MAX_VERTEX_STREAMS); /* Control data bits are initialized to 0 so we don't have to set any * bits when sending vertices to stream 0. */ if (stream_id == 0) return; const fs_builder abld = bld.annotate("set stream control data bits", NULL); /* reg::sid = stream_id */ fs_reg sid = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.MOV(sid, brw_imm_ud(stream_id)); /* reg:shift_count = 2 * (vertex_count - 1) */ fs_reg shift_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.SHL(shift_count, vertex_count, brw_imm_ud(1u)); /* Note: we're relying on the fact that the GEN SHL instruction only pays * attention to the lower 5 bits of its second source argument, so on this * architecture, stream_id << 2 * (vertex_count - 1) is equivalent to * stream_id << ((2 * (vertex_count - 1)) % 32). */ fs_reg mask = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); abld.SHL(mask, sid, shift_count); abld.OR(this->control_data_bits, this->control_data_bits, mask); } void fs_visitor::emit_gs_vertex(const nir_src &vertex_count_nir_src, unsigned stream_id) { assert(stage == MESA_SHADER_GEOMETRY); struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data); fs_reg vertex_count = get_nir_src(vertex_count_nir_src); vertex_count.type = BRW_REGISTER_TYPE_UD; /* Haswell and later hardware ignores the "Render Stream Select" bits * from the 3DSTATE_STREAMOUT packet when the SOL stage is disabled, * and instead sends all primitives down the pipeline for rasterization. * If the SOL stage is enabled, "Render Stream Select" is honored and * primitives bound to non-zero streams are discarded after stream output. * * Since the only purpose of primives sent to non-zero streams is to * be recorded by transform feedback, we can simply discard all geometry * bound to these streams when transform feedback is disabled. */ if (stream_id > 0 && !nir->info.has_transform_feedback_varyings) return; /* If we're outputting 32 control data bits or less, then we can wait * until the shader is over to output them all. Otherwise we need to * output them as we go. Now is the time to do it, since we're about to * output the vertex_count'th vertex, so it's guaranteed that the * control data bits associated with the (vertex_count - 1)th vertex are * correct. */ if (gs_compile->control_data_header_size_bits > 32) { const fs_builder abld = bld.annotate("emit vertex: emit control data bits"); /* Only emit control data bits if we've finished accumulating a batch * of 32 bits. This is the case when: * * (vertex_count * bits_per_vertex) % 32 == 0 * * (in other words, when the last 5 bits of vertex_count * * bits_per_vertex are 0). Assuming bits_per_vertex == 2^n for some * integer n (which is always the case, since bits_per_vertex is * always 1 or 2), this is equivalent to requiring that the last 5-n * bits of vertex_count are 0: * * vertex_count & (2^(5-n) - 1) == 0 * * 2^(5-n) == 2^5 / 2^n == 32 / bits_per_vertex, so this is * equivalent to: * * vertex_count & (32 / bits_per_vertex - 1) == 0 * * TODO: If vertex_count is an immediate, we could do some of this math * at compile time... */ fs_inst *inst = abld.AND(bld.null_reg_d(), vertex_count, brw_imm_ud(32u / gs_compile->control_data_bits_per_vertex - 1u)); inst->conditional_mod = BRW_CONDITIONAL_Z; abld.IF(BRW_PREDICATE_NORMAL); /* If vertex_count is 0, then no control data bits have been * accumulated yet, so we can skip emitting them. */ abld.CMP(bld.null_reg_d(), vertex_count, brw_imm_ud(0u), BRW_CONDITIONAL_NEQ); abld.IF(BRW_PREDICATE_NORMAL); emit_gs_control_data_bits(vertex_count); abld.emit(BRW_OPCODE_ENDIF); /* Reset control_data_bits to 0 so we can start accumulating a new * batch. * * Note: in the case where vertex_count == 0, this neutralizes the * effect of any call to EndPrimitive() that the shader may have * made before outputting its first vertex. */ inst = abld.MOV(this->control_data_bits, brw_imm_ud(0u)); inst->force_writemask_all = true; abld.emit(BRW_OPCODE_ENDIF); } emit_urb_writes(vertex_count); /* In stream mode we have to set control data bits for all vertices * unless we have disabled control data bits completely (which we do * do for GL_POINTS outputs that don't use streams). */ if (gs_compile->control_data_header_size_bits > 0 && gs_prog_data->control_data_format == GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) { set_gs_stream_control_data_bits(vertex_count, stream_id); } } void fs_visitor::emit_gs_input_load(const fs_reg &dst, const nir_src &vertex_src, unsigned base_offset, const nir_src &offset_src, unsigned num_components, unsigned first_component) { struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data); nir_const_value *vertex_const = nir_src_as_const_value(vertex_src); nir_const_value *offset_const = nir_src_as_const_value(offset_src); const unsigned push_reg_count = gs_prog_data->base.urb_read_length * 8; /* TODO: figure out push input layout for invocations == 1 */ /* TODO: make this work with 64-bit inputs */ if (gs_prog_data->invocations == 1 && type_sz(dst.type) <= 4 && offset_const != NULL && vertex_const != NULL && 4 * (base_offset + offset_const->u32[0]) < push_reg_count) { int imm_offset = (base_offset + offset_const->u32[0]) * 4 + vertex_const->u32[0] * push_reg_count; for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), fs_reg(ATTR, imm_offset + i + first_component, dst.type)); } return; } /* Resort to the pull model. Ensure the VUE handles are provided. */ assert(gs_prog_data->base.include_vue_handles); unsigned first_icp_handle = gs_prog_data->include_primitive_id ? 3 : 2; fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); if (gs_prog_data->invocations == 1) { if (vertex_const) { /* The vertex index is constant; just select the proper URB handle. */ icp_handle = retype(brw_vec8_grf(first_icp_handle + vertex_const->i32[0], 0), BRW_REGISTER_TYPE_UD); } else { /* The vertex index is non-constant. We need to use indirect * addressing to fetch the proper URB handle. * * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0> * indicating that channel should read the handle from * DWord . We convert that to bytes by multiplying by 4. * * Next, we convert the vertex index to bytes by multiplying * by 32 (shifting by 5), and add the two together. This is * the final indirect byte offset. */ fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1); fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */ bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210))); /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */ bld.SHL(channel_offsets, sequence, brw_imm_ud(2u)); /* Convert vertex_index to bytes (multiply by 32) */ bld.SHL(vertex_offset_bytes, retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(5u)); bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets); /* Use first_icp_handle as the base offset. There is one register * of URB handles per vertex, so inform the register allocator that * we might read up to nir->info.gs.vertices_in registers. */ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle, retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type), fs_reg(icp_offset_bytes), brw_imm_ud(nir->info.gs.vertices_in * REG_SIZE)); } } else { assert(gs_prog_data->invocations > 1); if (vertex_const) { assert(devinfo->gen >= 9 || vertex_const->i32[0] <= 5); bld.MOV(icp_handle, retype(brw_vec1_grf(first_icp_handle + vertex_const->i32[0] / 8, vertex_const->i32[0] % 8), BRW_REGISTER_TYPE_UD)); } else { /* The vertex index is non-constant. We need to use indirect * addressing to fetch the proper URB handle. * */ fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); /* Convert vertex_index to bytes (multiply by 4) */ bld.SHL(icp_offset_bytes, retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(2u)); /* Use first_icp_handle as the base offset. There is one DWord * of URB handles per vertex, so inform the register allocator that * we might read up to ceil(nir->info.gs.vertices_in / 8) registers. */ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle, retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type), fs_reg(icp_offset_bytes), brw_imm_ud(DIV_ROUND_UP(nir->info.gs.vertices_in, 8) * REG_SIZE)); } } fs_inst *inst; fs_reg tmp_dst = dst; fs_reg indirect_offset = get_nir_src(offset_src); unsigned num_iterations = 1; unsigned orig_num_components = num_components; if (type_sz(dst.type) == 8) { if (num_components > 2) { num_iterations = 2; num_components = 2; } fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type); tmp_dst = tmp; first_component = first_component / 2; } for (unsigned iter = 0; iter < num_iterations; iter++) { if (offset_const) { /* Constant indexing - use global offset. */ if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle); inst->size_written = read_components * tmp.component_size(inst->exec_size); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(tmp_dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp_dst, icp_handle); inst->size_written = num_components * tmp_dst.component_size(inst->exec_size); } inst->offset = base_offset + offset_const->u32[0]; inst->mlen = 1; } else { /* Indirect indexing - use per-slot offsets as well. */ const fs_reg srcs[] = { icp_handle, indirect_offset }; unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); inst->size_written = read_components * tmp.component_size(inst->exec_size); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(tmp_dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp_dst, payload); inst->size_written = num_components * tmp_dst.component_size(inst->exec_size); } inst->offset = base_offset; inst->mlen = 2; } if (type_sz(dst.type) == 8) { shuffle_from_32bit_read(bld, offset(dst, bld, iter * 2), retype(tmp_dst, BRW_REGISTER_TYPE_D), 0, num_components); } if (num_iterations > 1) { num_components = orig_num_components - 2; if(offset_const) { base_offset++; } else { fs_reg new_indirect = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.ADD(new_indirect, indirect_offset, brw_imm_ud(1u)); indirect_offset = new_indirect; } } } } fs_reg fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr) { nir_src *offset_src = nir_get_io_offset_src(instr); nir_const_value *const_value = nir_src_as_const_value(*offset_src); if (const_value) { /* The only constant offset we should find is 0. brw_nir.c's * add_const_offset_to_base() will fold other constant offsets * into instr->const_index[0]. */ assert(const_value->u32[0] == 0); return fs_reg(); } return get_nir_src(*offset_src); } static void do_untyped_vector_read(const fs_builder &bld, const fs_reg dest, const fs_reg surf_index, const fs_reg offset_reg, unsigned num_components) { if (type_sz(dest.type) <= 2) { assert(dest.stride == 1); boolean is_const_offset = offset_reg.file == BRW_IMMEDIATE_VALUE; if (is_const_offset) { uint32_t start = offset_reg.ud & ~3; uint32_t end = offset_reg.ud + num_components * type_sz(dest.type); end = ALIGN(end, 4); assert (end - start <= 16); /* At this point we have 16-bit component/s that have constant * offset aligned to 4-bytes that can be read with untyped_reads. * untyped_read message requires 32-bit aligned offsets. */ unsigned first_component = (offset_reg.ud & 3) / type_sz(dest.type); unsigned num_components_32bit = (end - start) / 4; fs_reg read_result = emit_untyped_read(bld, surf_index, brw_imm_ud(start), 1 /* dims */, num_components_32bit, BRW_PREDICATE_NONE); shuffle_from_32bit_read(bld, dest, read_result, first_component, num_components); } else { fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD); for (unsigned i = 0; i < num_components; i++) { if (i == 0) { bld.MOV(read_offset, offset_reg); } else { bld.ADD(read_offset, offset_reg, brw_imm_ud(i * type_sz(dest.type))); } /* Non constant offsets are not guaranteed to be aligned 32-bits * so they are read using one byte_scattered_read message * for each component. */ fs_reg read_result = emit_byte_scattered_read(bld, surf_index, read_offset, 1 /* dims */, 1, type_sz(dest.type) * 8 /* bit_size */, BRW_PREDICATE_NONE); bld.MOV(offset(dest, bld, i), subscript (read_result, dest.type, 0)); } } } else if (type_sz(dest.type) == 4) { fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg, 1 /* dims */, num_components, BRW_PREDICATE_NONE); read_result.type = dest.type; for (unsigned i = 0; i < num_components; i++) bld.MOV(offset(dest, bld, i), offset(read_result, bld, i)); } else if (type_sz(dest.type) == 8) { /* Reading a dvec, so we need to: * * 1. Multiply num_components by 2, to account for the fact that we * need to read 64-bit components. * 2. Shuffle the result of the load to form valid 64-bit elements * 3. Emit a second load (for components z/w) if needed. */ fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(read_offset, offset_reg); int iters = num_components <= 2 ? 1 : 2; /* Load the dvec, the first iteration loads components x/y, the second * iteration, if needed, loads components z/w */ for (int it = 0; it < iters; it++) { /* Compute number of components to read in this iteration */ int iter_components = MIN2(2, num_components); num_components -= iter_components; /* Read. Since this message reads 32-bit components, we need to * read twice as many components. */ fs_reg read_result = emit_untyped_read(bld, surf_index, read_offset, 1 /* dims */, iter_components * 2, BRW_PREDICATE_NONE); /* Shuffle the 32-bit load result into valid 64-bit data */ shuffle_from_32bit_read(bld, offset(dest, bld, it * 2), read_result, 0, iter_components); bld.ADD(read_offset, read_offset, brw_imm_ud(16)); } } else { unreachable("Unsupported type"); } } void fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_VERTEX); fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_vertex_id: case nir_intrinsic_load_base_vertex: unreachable("should be lowered by nir_lower_system_values()"); case nir_intrinsic_load_input: { fs_reg src = fs_reg(ATTR, nir_intrinsic_base(instr) * 4, dest.type); unsigned first_component = nir_intrinsic_component(instr); unsigned num_components = instr->num_components; nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); assert(const_offset && "Indirect input loads not allowed"); src = offset(src, bld, const_offset->u32[0]); if (type_sz(dest.type) == 8) first_component /= 2; /* For 16-bit support maybe a temporary will be needed to copy from * the ATTR file. */ shuffle_from_32bit_read(bld, dest, retype(src, BRW_REGISTER_TYPE_D), first_component, num_components); break; } case nir_intrinsic_load_vertex_id_zero_base: case nir_intrinsic_load_instance_id: case nir_intrinsic_load_base_instance: case nir_intrinsic_load_draw_id: case nir_intrinsic_load_first_vertex: case nir_intrinsic_load_is_indexed_draw: unreachable("lowered by brw_nir_lower_vs_inputs"); default: nir_emit_intrinsic(bld, instr); break; } } void fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_TESS_CTRL); struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key; struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data); fs_reg dst; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dst = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: bld.MOV(dst, fs_reg(brw_vec1_grf(0, 1))); break; case nir_intrinsic_load_invocation_id: bld.MOV(retype(dst, invocation_id.type), invocation_id); break; case nir_intrinsic_load_patch_vertices_in: bld.MOV(retype(dst, BRW_REGISTER_TYPE_D), brw_imm_d(tcs_key->input_vertices)); break; case nir_intrinsic_barrier: { if (tcs_prog_data->instances == 1) break; fs_reg m0 = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg m0_2 = component(m0, 2); const fs_builder chanbld = bld.exec_all().group(1, 0); /* Zero the message header */ bld.exec_all().MOV(m0, brw_imm_ud(0u)); /* Copy "Barrier ID" from r0.2, bits 16:13 */ chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(INTEL_MASK(16, 13))); /* Shift it up to bits 27:24. */ chanbld.SHL(m0_2, m0_2, brw_imm_ud(11)); /* Set the Barrier Count and the enable bit */ chanbld.OR(m0_2, m0_2, brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15))); bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0); break; } case nir_intrinsic_load_input: unreachable("nir_lower_io should never give us these."); break; case nir_intrinsic_load_per_vertex_input: { fs_reg indirect_offset = get_indirect_offset(instr); unsigned imm_offset = instr->const_index[0]; const nir_src &vertex_src = instr->src[0]; nir_const_value *vertex_const = nir_src_as_const_value(vertex_src); fs_inst *inst; fs_reg icp_handle; if (vertex_const) { /* Emit a MOV to resolve <0,1,0> regioning. */ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.MOV(icp_handle, retype(brw_vec1_grf(1 + (vertex_const->i32[0] >> 3), vertex_const->i32[0] & 7), BRW_REGISTER_TYPE_UD)); } else if (tcs_prog_data->instances == 1 && vertex_src.is_ssa && vertex_src.ssa->parent_instr->type == nir_instr_type_intrinsic && nir_instr_as_intrinsic(vertex_src.ssa->parent_instr)->intrinsic == nir_intrinsic_load_invocation_id) { /* For the common case of only 1 instance, an array index of * gl_InvocationID means reading g1. Skip all the indirect work. */ icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD); } else { /* The vertex index is non-constant. We need to use indirect * addressing to fetch the proper URB handle. */ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); /* Each ICP handle is a single DWord (4 bytes) */ fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.SHL(vertex_offset_bytes, retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(2u)); /* Start at g1. We might read up to 4 registers. */ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle, retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes, brw_imm_ud(4 * REG_SIZE)); } /* We can only read two double components with each URB read, so * we send two read messages in that case, each one loading up to * two double components. */ unsigned num_iterations = 1; unsigned num_components = instr->num_components; unsigned first_component = nir_intrinsic_component(instr); fs_reg orig_dst = dst; if (type_sz(dst.type) == 8) { first_component = first_component / 2; if (instr->num_components > 2) { num_iterations = 2; num_components = 2; } fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type); dst = tmp; } for (unsigned iter = 0; iter < num_iterations; iter++) { if (indirect_offset.file == BAD_FILE) { /* Constant indexing - use global offset. */ if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle); } inst->offset = imm_offset; inst->mlen = 1; } else { /* Indirect indexing - use per-slot offsets as well. */ const fs_reg srcs[] = { icp_handle, indirect_offset }; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload); } inst->offset = imm_offset; inst->mlen = 2; } inst->size_written = (num_components + first_component) * inst->dst.component_size(inst->exec_size); /* If we are reading 64-bit data using 32-bit read messages we need * build proper 64-bit data elements by shuffling the low and high * 32-bit components around like we do for other things like UBOs * or SSBOs. */ if (type_sz(dst.type) == 8) { shuffle_from_32bit_read(bld, offset(orig_dst, bld, iter * 2), retype(dst, BRW_REGISTER_TYPE_D), 0, num_components); } /* Copy the temporary to the destination to deal with writemasking. * * Also attempt to deal with gl_PointSize being in the .w component. */ if (inst->offset == 0 && indirect_offset.file == BAD_FILE) { assert(type_sz(dst.type) < 8); inst->dst = bld.vgrf(dst.type, 4); inst->size_written = 4 * REG_SIZE; bld.MOV(dst, offset(inst->dst, bld, 3)); } /* If we are loading double data and we need a second read message * adjust the write offset */ if (num_iterations > 1) { num_components = instr->num_components - 2; imm_offset++; } } break; } case nir_intrinsic_load_output: case nir_intrinsic_load_per_vertex_output: { fs_reg indirect_offset = get_indirect_offset(instr); unsigned imm_offset = instr->const_index[0]; unsigned first_component = nir_intrinsic_component(instr); fs_inst *inst; if (indirect_offset.file == BAD_FILE) { /* Replicate the patch handle to all enabled channels */ fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.MOV(patch_handle, retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)); { if (first_component != 0) { unsigned read_components = instr->num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, patch_handle); inst->size_written = read_components * REG_SIZE; for (unsigned i = 0; i < instr->num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, patch_handle); inst->size_written = instr->num_components * REG_SIZE; } inst->offset = imm_offset; inst->mlen = 1; } } else { /* Indirect indexing - use per-slot offsets as well. */ const fs_reg srcs[] = { retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD), indirect_offset }; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { unsigned read_components = instr->num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); inst->size_written = read_components * REG_SIZE; for (unsigned i = 0; i < instr->num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload); inst->size_written = instr->num_components * REG_SIZE; } inst->offset = imm_offset; inst->mlen = 2; } break; } case nir_intrinsic_store_output: case nir_intrinsic_store_per_vertex_output: { fs_reg value = get_nir_src(instr->src[0]); bool is_64bit = (instr->src[0].is_ssa ? instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size) == 64; fs_reg indirect_offset = get_indirect_offset(instr); unsigned imm_offset = instr->const_index[0]; unsigned mask = instr->const_index[1]; unsigned header_regs = 0; fs_reg srcs[7]; srcs[header_regs++] = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD); if (indirect_offset.file != BAD_FILE) { srcs[header_regs++] = indirect_offset; } if (mask == 0) break; unsigned num_components = util_last_bit(mask); enum opcode opcode; /* We can only pack two 64-bit components in a single message, so send * 2 messages if we have more components */ unsigned num_iterations = 1; unsigned iter_components = num_components; unsigned first_component = nir_intrinsic_component(instr); if (is_64bit) { first_component = first_component / 2; if (instr->num_components > 2) { num_iterations = 2; iter_components = 2; } } mask = mask << first_component; for (unsigned iter = 0; iter < num_iterations; iter++) { if (!is_64bit && mask != WRITEMASK_XYZW) { srcs[header_regs++] = brw_imm_ud(mask << 16); opcode = indirect_offset.file != BAD_FILE ? SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT : SHADER_OPCODE_URB_WRITE_SIMD8_MASKED; } else if (is_64bit && ((mask & WRITEMASK_XY) != WRITEMASK_XY)) { /* Expand the 64-bit mask to 32-bit channels. We only handle * two channels in each iteration, so we only care about X/Y. */ unsigned mask32 = 0; if (mask & WRITEMASK_X) mask32 |= WRITEMASK_XY; if (mask & WRITEMASK_Y) mask32 |= WRITEMASK_ZW; /* If the mask does not include any of the channels X or Y there * is nothing to do in this iteration. Move on to the next couple * of 64-bit channels. */ if (!mask32) { mask >>= 2; imm_offset++; continue; } srcs[header_regs++] = brw_imm_ud(mask32 << 16); opcode = indirect_offset.file != BAD_FILE ? SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT : SHADER_OPCODE_URB_WRITE_SIMD8_MASKED; } else { opcode = indirect_offset.file != BAD_FILE ? SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT : SHADER_OPCODE_URB_WRITE_SIMD8; } for (unsigned i = 0; i < iter_components; i++) { if (!(mask & (1 << (i + first_component)))) continue; if (!is_64bit) { srcs[header_regs + i + first_component] = offset(value, bld, i); } else { /* We need to shuffle the 64-bit data to match the layout * expected by our 32-bit URB write messages. We use a temporary * for that. */ unsigned channel = iter * 2 + i; fs_reg dest = shuffle_for_32bit_write(bld, value, channel, 1); srcs[header_regs + (i + first_component) * 2] = dest; srcs[header_regs + (i + first_component) * 2 + 1] = offset(dest, bld, 1); } } unsigned mlen = header_regs + (is_64bit ? 2 * iter_components : iter_components) + (is_64bit ? 2 * first_component : first_component); fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen); bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs); fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload); inst->offset = imm_offset; inst->mlen = mlen; /* If this is a 64-bit attribute, select the next two 64-bit channels * to be handled in the next iteration. */ if (is_64bit) { mask >>= 2; imm_offset++; } } break; } default: nir_emit_intrinsic(bld, instr); break; } } void fs_visitor::nir_emit_tes_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_TESS_EVAL); struct brw_tes_prog_data *tes_prog_data = brw_tes_prog_data(prog_data); fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: bld.MOV(dest, fs_reg(brw_vec1_grf(0, 1))); break; case nir_intrinsic_load_tess_coord: /* gl_TessCoord is part of the payload in g1-3 */ for (unsigned i = 0; i < 3; i++) { bld.MOV(offset(dest, bld, i), fs_reg(brw_vec8_grf(1 + i, 0))); } break; case nir_intrinsic_load_input: case nir_intrinsic_load_per_vertex_input: { fs_reg indirect_offset = get_indirect_offset(instr); unsigned imm_offset = instr->const_index[0]; unsigned first_component = nir_intrinsic_component(instr); if (type_sz(dest.type) == 8) { first_component = first_component / 2; } fs_inst *inst; if (indirect_offset.file == BAD_FILE) { /* Arbitrarily only push up to 32 vec4 slots worth of data, * which is 16 registers (since each holds 2 vec4 slots). */ unsigned slot_count = 1; if (type_sz(dest.type) == 8 && instr->num_components > 2) slot_count++; const unsigned max_push_slots = 32; if (imm_offset + slot_count <= max_push_slots) { fs_reg src = fs_reg(ATTR, imm_offset / 2, dest.type); for (int i = 0; i < instr->num_components; i++) { unsigned comp = 16 / type_sz(dest.type) * (imm_offset % 2) + i + first_component; bld.MOV(offset(dest, bld, i), component(src, comp)); } tes_prog_data->base.urb_read_length = MAX2(tes_prog_data->base.urb_read_length, DIV_ROUND_UP(imm_offset + slot_count, 2)); } else { /* Replicate the patch handle to all enabled channels */ const fs_reg srcs[] = { retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD) }; fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.LOAD_PAYLOAD(patch_handle, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { unsigned read_components = instr->num_components + first_component; fs_reg tmp = bld.vgrf(dest.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, patch_handle); inst->size_written = read_components * REG_SIZE; for (unsigned i = 0; i < instr->num_components; i++) { bld.MOV(offset(dest, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dest, patch_handle); inst->size_written = instr->num_components * REG_SIZE; } inst->mlen = 1; inst->offset = imm_offset; } } else { /* Indirect indexing - use per-slot offsets as well. */ /* We can only read two double components with each URB read, so * we send two read messages in that case, each one loading up to * two double components. */ unsigned num_iterations = 1; unsigned num_components = instr->num_components; fs_reg orig_dest = dest; if (type_sz(dest.type) == 8) { if (instr->num_components > 2) { num_iterations = 2; num_components = 2; } fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dest.type); dest = tmp; } for (unsigned iter = 0; iter < num_iterations; iter++) { const fs_reg srcs[] = { retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD), indirect_offset }; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dest.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dest, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest, payload); } inst->mlen = 2; inst->offset = imm_offset; inst->size_written = (num_components + first_component) * inst->dst.component_size(inst->exec_size); /* If we are reading 64-bit data using 32-bit read messages we need * build proper 64-bit data elements by shuffling the low and high * 32-bit components around like we do for other things like UBOs * or SSBOs. */ if (type_sz(dest.type) == 8) { shuffle_from_32bit_read(bld, offset(orig_dest, bld, iter * 2), retype(dest, BRW_REGISTER_TYPE_D), 0, num_components); } /* If we are loading double data and we need a second read message * adjust the offset */ if (num_iterations > 1) { num_components = instr->num_components - 2; imm_offset++; } } } break; } default: nir_emit_intrinsic(bld, instr); break; } } void fs_visitor::nir_emit_gs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_GEOMETRY); fs_reg indirect_offset; fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: assert(stage == MESA_SHADER_GEOMETRY); assert(brw_gs_prog_data(prog_data)->include_primitive_id); bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), retype(fs_reg(brw_vec8_grf(2, 0)), BRW_REGISTER_TYPE_UD)); break; case nir_intrinsic_load_input: unreachable("load_input intrinsics are invalid for the GS stage"); case nir_intrinsic_load_per_vertex_input: emit_gs_input_load(dest, instr->src[0], instr->const_index[0], instr->src[1], instr->num_components, nir_intrinsic_component(instr)); break; case nir_intrinsic_emit_vertex_with_counter: emit_gs_vertex(instr->src[0], instr->const_index[0]); break; case nir_intrinsic_end_primitive_with_counter: emit_gs_end_primitive(instr->src[0]); break; case nir_intrinsic_set_vertex_count: bld.MOV(this->final_gs_vertex_count, get_nir_src(instr->src[0])); break; case nir_intrinsic_load_invocation_id: { fs_reg val = nir_system_values[SYSTEM_VALUE_INVOCATION_ID]; assert(val.file != BAD_FILE); dest.type = val.type; bld.MOV(dest, val); break; } default: nir_emit_intrinsic(bld, instr); break; } } /** * Fetch the current render target layer index. */ static fs_reg fetch_render_target_array_index(const fs_builder &bld) { if (bld.shader->devinfo->gen >= 6) { /* The render target array index is provided in the thread payload as * bits 26:16 of r0.0. */ const fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.AND(idx, brw_uw1_reg(BRW_GENERAL_REGISTER_FILE, 0, 1), brw_imm_uw(0x7ff)); return idx; } else { /* Pre-SNB we only ever render into the first layer of the framebuffer * since layered rendering is not implemented. */ return brw_imm_ud(0); } } /** * Fake non-coherent framebuffer read implemented using TXF to fetch from the * framebuffer at the current fragment coordinates and sample index. */ fs_inst * fs_visitor::emit_non_coherent_fb_read(const fs_builder &bld, const fs_reg &dst, unsigned target) { const struct gen_device_info *devinfo = bld.shader->devinfo; assert(bld.shader->stage == MESA_SHADER_FRAGMENT); const brw_wm_prog_key *wm_key = reinterpret_cast(key); assert(!wm_key->coherent_fb_fetch); const struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(stage_prog_data); /* Calculate the surface index relative to the start of the texture binding * table block, since that's what the texturing messages expect. */ const unsigned surface = target + wm_prog_data->binding_table.render_target_read_start - wm_prog_data->base.binding_table.texture_start; brw_mark_surface_used( bld.shader->stage_prog_data, wm_prog_data->binding_table.render_target_read_start + target); /* Calculate the fragment coordinates. */ const fs_reg coords = bld.vgrf(BRW_REGISTER_TYPE_UD, 3); bld.MOV(offset(coords, bld, 0), pixel_x); bld.MOV(offset(coords, bld, 1), pixel_y); bld.MOV(offset(coords, bld, 2), fetch_render_target_array_index(bld)); /* Calculate the sample index and MCS payload when multisampling. Luckily * the MCS fetch message behaves deterministically for UMS surfaces, so it * shouldn't be necessary to recompile based on whether the framebuffer is * CMS or UMS. */ if (wm_key->multisample_fbo && nir_system_values[SYSTEM_VALUE_SAMPLE_ID].file == BAD_FILE) nir_system_values[SYSTEM_VALUE_SAMPLE_ID] = *emit_sampleid_setup(); const fs_reg sample = nir_system_values[SYSTEM_VALUE_SAMPLE_ID]; const fs_reg mcs = wm_key->multisample_fbo ? emit_mcs_fetch(coords, 3, brw_imm_ud(surface)) : fs_reg(); /* Use either a normal or a CMS texel fetch message depending on whether * the framebuffer is single or multisample. On SKL+ use the wide CMS * message just in case the framebuffer uses 16x multisampling, it should * be equivalent to the normal CMS fetch for lower multisampling modes. */ const opcode op = !wm_key->multisample_fbo ? SHADER_OPCODE_TXF_LOGICAL : devinfo->gen >= 9 ? SHADER_OPCODE_TXF_CMS_W_LOGICAL : SHADER_OPCODE_TXF_CMS_LOGICAL; /* Emit the instruction. */ const fs_reg srcs[] = { coords, fs_reg(), brw_imm_ud(0), fs_reg(), sample, mcs, brw_imm_ud(surface), brw_imm_ud(0), fs_reg(), brw_imm_ud(3), brw_imm_ud(0) }; STATIC_ASSERT(ARRAY_SIZE(srcs) == TEX_LOGICAL_NUM_SRCS); fs_inst *inst = bld.emit(op, dst, srcs, ARRAY_SIZE(srcs)); inst->size_written = 4 * inst->dst.component_size(inst->exec_size); return inst; } /** * Actual coherent framebuffer read implemented using the native render target * read message. Requires SKL+. */ static fs_inst * emit_coherent_fb_read(const fs_builder &bld, const fs_reg &dst, unsigned target) { assert(bld.shader->devinfo->gen >= 9); fs_inst *inst = bld.emit(FS_OPCODE_FB_READ_LOGICAL, dst); inst->target = target; inst->size_written = 4 * inst->dst.component_size(inst->exec_size); return inst; } static fs_reg alloc_temporary(const fs_builder &bld, unsigned size, fs_reg *regs, unsigned n) { if (n && regs[0].file != BAD_FILE) { return regs[0]; } else { const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, size); for (unsigned i = 0; i < n; i++) regs[i] = tmp; return tmp; } } static fs_reg alloc_frag_output(fs_visitor *v, unsigned location) { assert(v->stage == MESA_SHADER_FRAGMENT); const brw_wm_prog_key *const key = reinterpret_cast(v->key); const unsigned l = GET_FIELD(location, BRW_NIR_FRAG_OUTPUT_LOCATION); const unsigned i = GET_FIELD(location, BRW_NIR_FRAG_OUTPUT_INDEX); if (i > 0 || (key->force_dual_color_blend && l == FRAG_RESULT_DATA1)) return alloc_temporary(v->bld, 4, &v->dual_src_output, 1); else if (l == FRAG_RESULT_COLOR) return alloc_temporary(v->bld, 4, v->outputs, MAX2(key->nr_color_regions, 1)); else if (l == FRAG_RESULT_DEPTH) return alloc_temporary(v->bld, 1, &v->frag_depth, 1); else if (l == FRAG_RESULT_STENCIL) return alloc_temporary(v->bld, 1, &v->frag_stencil, 1); else if (l == FRAG_RESULT_SAMPLE_MASK) return alloc_temporary(v->bld, 1, &v->sample_mask, 1); else if (l >= FRAG_RESULT_DATA0 && l < FRAG_RESULT_DATA0 + BRW_MAX_DRAW_BUFFERS) return alloc_temporary(v->bld, 4, &v->outputs[l - FRAG_RESULT_DATA0], 1); else unreachable("Invalid location"); } void fs_visitor::nir_emit_fs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_FRAGMENT); fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_front_face: bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), *emit_frontfacing_interpolation()); break; case nir_intrinsic_load_sample_pos: { fs_reg sample_pos = nir_system_values[SYSTEM_VALUE_SAMPLE_POS]; assert(sample_pos.file != BAD_FILE); dest.type = sample_pos.type; bld.MOV(dest, sample_pos); bld.MOV(offset(dest, bld, 1), offset(sample_pos, bld, 1)); break; } case nir_intrinsic_load_layer_id: dest.type = BRW_REGISTER_TYPE_UD; bld.MOV(dest, fetch_render_target_array_index(bld)); break; case nir_intrinsic_load_helper_invocation: case nir_intrinsic_load_sample_mask_in: case nir_intrinsic_load_sample_id: { gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic); fs_reg val = nir_system_values[sv]; assert(val.file != BAD_FILE); dest.type = val.type; bld.MOV(dest, val); break; } case nir_intrinsic_store_output: { const fs_reg src = get_nir_src(instr->src[0]); const nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]); assert(const_offset && "Indirect output stores not allowed"); const unsigned location = nir_intrinsic_base(instr) + SET_FIELD(const_offset->u32[0], BRW_NIR_FRAG_OUTPUT_LOCATION); const fs_reg new_dest = retype(alloc_frag_output(this, location), src.type); for (unsigned j = 0; j < instr->num_components; j++) bld.MOV(offset(new_dest, bld, nir_intrinsic_component(instr) + j), offset(src, bld, j)); break; } case nir_intrinsic_load_output: { const unsigned l = GET_FIELD(nir_intrinsic_base(instr), BRW_NIR_FRAG_OUTPUT_LOCATION); assert(l >= FRAG_RESULT_DATA0); nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); assert(const_offset && "Indirect output loads not allowed"); const unsigned target = l - FRAG_RESULT_DATA0 + const_offset->u32[0]; const fs_reg tmp = bld.vgrf(dest.type, 4); if (reinterpret_cast(key)->coherent_fb_fetch) emit_coherent_fb_read(bld, tmp, target); else emit_non_coherent_fb_read(bld, tmp, target); for (unsigned j = 0; j < instr->num_components; j++) { bld.MOV(offset(dest, bld, j), offset(tmp, bld, nir_intrinsic_component(instr) + j)); } break; } case nir_intrinsic_discard: case nir_intrinsic_discard_if: { /* We track our discarded pixels in f0.1. By predicating on it, we can * update just the flag bits that aren't yet discarded. If there's no * condition, we emit a CMP of g0 != g0, so all currently executing * channels will get turned off. */ fs_inst *cmp; if (instr->intrinsic == nir_intrinsic_discard_if) { cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_Z); } else { fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UW)); cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ); } cmp->predicate = BRW_PREDICATE_NORMAL; cmp->flag_subreg = 1; if (devinfo->gen >= 6) { emit_discard_jump(); } limit_dispatch_width(16, "Fragment discard not implemented in SIMD32 mode."); break; } case nir_intrinsic_load_input: { /* load_input is only used for flat inputs */ unsigned base = nir_intrinsic_base(instr); unsigned comp = nir_intrinsic_component(instr); unsigned num_components = instr->num_components; fs_reg orig_dest = dest; enum brw_reg_type type = dest.type; /* Special case fields in the VUE header */ if (base == VARYING_SLOT_LAYER) comp = 1; else if (base == VARYING_SLOT_VIEWPORT) comp = 2; if (nir_dest_bit_size(instr->dest) == 64) { /* const_index is in 32-bit type size units that could not be aligned * with DF. We need to read the double vector as if it was a float * vector of twice the number of components to fetch the right data. */ type = BRW_REGISTER_TYPE_F; num_components *= 2; dest = bld.vgrf(type, num_components); } for (unsigned int i = 0; i < num_components; i++) { bld.MOV(offset(retype(dest, type), bld, i), retype(component(interp_reg(base, comp + i), 3), type)); } if (nir_dest_bit_size(instr->dest) == 64) { shuffle_from_32bit_read(bld, orig_dest, dest, 0, instr->num_components); } break; } case nir_intrinsic_load_barycentric_pixel: case nir_intrinsic_load_barycentric_centroid: case nir_intrinsic_load_barycentric_sample: /* Do nothing - load_interpolated_input handling will handle it later. */ break; case nir_intrinsic_load_barycentric_at_sample: { const glsl_interp_mode interpolation = (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr); nir_const_value *const_sample = nir_src_as_const_value(instr->src[0]); if (const_sample) { unsigned msg_data = const_sample->i32[0] << 4; emit_pixel_interpolater_send(bld, FS_OPCODE_INTERPOLATE_AT_SAMPLE, dest, fs_reg(), /* src */ brw_imm_ud(msg_data), interpolation); } else { const fs_reg sample_src = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD); if (nir_src_is_dynamically_uniform(instr->src[0])) { const fs_reg sample_id = bld.emit_uniformize(sample_src); const fs_reg msg_data = vgrf(glsl_type::uint_type); bld.exec_all().group(1, 0) .SHL(msg_data, sample_id, brw_imm_ud(4u)); emit_pixel_interpolater_send(bld, FS_OPCODE_INTERPOLATE_AT_SAMPLE, dest, fs_reg(), /* src */ msg_data, interpolation); } else { /* Make a loop that sends a message to the pixel interpolater * for the sample number in each live channel. If there are * multiple channels with the same sample number then these * will be handled simultaneously with a single interation of * the loop. */ bld.emit(BRW_OPCODE_DO); /* Get the next live sample number into sample_id_reg */ const fs_reg sample_id = bld.emit_uniformize(sample_src); /* Set the flag register so that we can perform the send * message on all channels that have the same sample number */ bld.CMP(bld.null_reg_ud(), sample_src, sample_id, BRW_CONDITIONAL_EQ); const fs_reg msg_data = vgrf(glsl_type::uint_type); bld.exec_all().group(1, 0) .SHL(msg_data, sample_id, brw_imm_ud(4u)); fs_inst *inst = emit_pixel_interpolater_send(bld, FS_OPCODE_INTERPOLATE_AT_SAMPLE, dest, fs_reg(), /* src */ component(msg_data, 0), interpolation); set_predicate(BRW_PREDICATE_NORMAL, inst); /* Continue the loop if there are any live channels left */ set_predicate_inv(BRW_PREDICATE_NORMAL, true, /* inverse */ bld.emit(BRW_OPCODE_WHILE)); } } break; } case nir_intrinsic_load_barycentric_at_offset: { const glsl_interp_mode interpolation = (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr); nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); if (const_offset) { unsigned off_x = MIN2((int)(const_offset->f32[0] * 16), 7) & 0xf; unsigned off_y = MIN2((int)(const_offset->f32[1] * 16), 7) & 0xf; emit_pixel_interpolater_send(bld, FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET, dest, fs_reg(), /* src */ brw_imm_ud(off_x | (off_y << 4)), interpolation); } else { fs_reg src = vgrf(glsl_type::ivec2_type); fs_reg offset_src = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F); for (int i = 0; i < 2; i++) { fs_reg temp = vgrf(glsl_type::float_type); bld.MUL(temp, offset(offset_src, bld, i), brw_imm_f(16.0f)); fs_reg itemp = vgrf(glsl_type::int_type); /* float to int */ bld.MOV(itemp, temp); /* Clamp the upper end of the range to +7/16. * ARB_gpu_shader5 requires that we support a maximum offset * of +0.5, which isn't representable in a S0.4 value -- if * we didn't clamp it, we'd end up with -8/16, which is the * opposite of what the shader author wanted. * * This is legal due to ARB_gpu_shader5's quantization * rules: * * "Not all values of may be supported; x and y * offsets may be rounded to fixed-point values with the * number of fraction bits given by the * implementation-dependent constant * FRAGMENT_INTERPOLATION_OFFSET_BITS" */ set_condmod(BRW_CONDITIONAL_L, bld.SEL(offset(src, bld, i), itemp, brw_imm_d(7))); } const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET; emit_pixel_interpolater_send(bld, opcode, dest, src, brw_imm_ud(0u), interpolation); } break; } case nir_intrinsic_load_interpolated_input: { if (nir_intrinsic_base(instr) == VARYING_SLOT_POS) { emit_fragcoord_interpolation(dest); break; } assert(instr->src[0].ssa && instr->src[0].ssa->parent_instr->type == nir_instr_type_intrinsic); nir_intrinsic_instr *bary_intrinsic = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr); nir_intrinsic_op bary_intrin = bary_intrinsic->intrinsic; enum glsl_interp_mode interp_mode = (enum glsl_interp_mode) nir_intrinsic_interp_mode(bary_intrinsic); fs_reg dst_xy; if (bary_intrin == nir_intrinsic_load_barycentric_at_offset || bary_intrin == nir_intrinsic_load_barycentric_at_sample) { /* Use the result of the PI message */ dst_xy = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F); } else { /* Use the delta_xy values computed from the payload */ enum brw_barycentric_mode bary = brw_barycentric_mode(interp_mode, bary_intrin); dst_xy = this->delta_xy[bary]; } for (unsigned int i = 0; i < instr->num_components; i++) { fs_reg interp = component(interp_reg(nir_intrinsic_base(instr), nir_intrinsic_component(instr) + i), 0); interp.type = BRW_REGISTER_TYPE_F; dest.type = BRW_REGISTER_TYPE_F; if (devinfo->gen < 6 && interp_mode == INTERP_MODE_SMOOTH) { fs_reg tmp = vgrf(glsl_type::float_type); bld.emit(FS_OPCODE_LINTERP, tmp, dst_xy, interp); bld.MUL(offset(dest, bld, i), tmp, this->pixel_w); } else { bld.emit(FS_OPCODE_LINTERP, offset(dest, bld, i), dst_xy, interp); } } break; } default: nir_emit_intrinsic(bld, instr); break; } } static int get_op_for_atomic_add(nir_intrinsic_instr *instr, unsigned src) { const nir_const_value *const val = nir_src_as_const_value(instr->src[src]); if (val != NULL) { if (val->i32[0] == 1) return BRW_AOP_INC; else if (val->i32[0] == -1) return BRW_AOP_DEC; } return BRW_AOP_ADD; } void fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_COMPUTE); struct brw_cs_prog_data *cs_prog_data = brw_cs_prog_data(prog_data); fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_barrier: emit_barrier(); cs_prog_data->uses_barrier = true; break; case nir_intrinsic_load_subgroup_id: bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), subgroup_id); break; case nir_intrinsic_load_local_invocation_id: case nir_intrinsic_load_work_group_id: { gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic); fs_reg val = nir_system_values[sv]; assert(val.file != BAD_FILE); dest.type = val.type; for (unsigned i = 0; i < 3; i++) bld.MOV(offset(dest, bld, i), offset(val, bld, i)); break; } case nir_intrinsic_load_num_work_groups: { const unsigned surface = cs_prog_data->binding_table.work_groups_start; cs_prog_data->uses_num_work_groups = true; fs_reg surf_index = brw_imm_ud(surface); brw_mark_surface_used(prog_data, surface); /* Read the 3 GLuint components of gl_NumWorkGroups */ for (unsigned i = 0; i < 3; i++) { fs_reg read_result = emit_untyped_read(bld, surf_index, brw_imm_ud(i << 2), 1 /* dims */, 1 /* size */, BRW_PREDICATE_NONE); read_result.type = dest.type; bld.MOV(dest, read_result); dest = offset(dest, bld, 1); } break; } case nir_intrinsic_shared_atomic_add: nir_emit_shared_atomic(bld, get_op_for_atomic_add(instr, 1), instr); break; case nir_intrinsic_shared_atomic_imin: nir_emit_shared_atomic(bld, BRW_AOP_IMIN, instr); break; case nir_intrinsic_shared_atomic_umin: nir_emit_shared_atomic(bld, BRW_AOP_UMIN, instr); break; case nir_intrinsic_shared_atomic_imax: nir_emit_shared_atomic(bld, BRW_AOP_IMAX, instr); break; case nir_intrinsic_shared_atomic_umax: nir_emit_shared_atomic(bld, BRW_AOP_UMAX, instr); break; case nir_intrinsic_shared_atomic_and: nir_emit_shared_atomic(bld, BRW_AOP_AND, instr); break; case nir_intrinsic_shared_atomic_or: nir_emit_shared_atomic(bld, BRW_AOP_OR, instr); break; case nir_intrinsic_shared_atomic_xor: nir_emit_shared_atomic(bld, BRW_AOP_XOR, instr); break; case nir_intrinsic_shared_atomic_exchange: nir_emit_shared_atomic(bld, BRW_AOP_MOV, instr); break; case nir_intrinsic_shared_atomic_comp_swap: nir_emit_shared_atomic(bld, BRW_AOP_CMPWR, instr); break; case nir_intrinsic_shared_atomic_fmin: nir_emit_shared_atomic_float(bld, BRW_AOP_FMIN, instr); break; case nir_intrinsic_shared_atomic_fmax: nir_emit_shared_atomic_float(bld, BRW_AOP_FMAX, instr); break; case nir_intrinsic_shared_atomic_fcomp_swap: nir_emit_shared_atomic_float(bld, BRW_AOP_FCMPWR, instr); break; case nir_intrinsic_load_shared: { assert(devinfo->gen >= 7); fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM); /* Get the offset to read from */ fs_reg offset_reg; nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); if (const_offset) { offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]); } else { offset_reg = vgrf(glsl_type::uint_type); bld.ADD(offset_reg, retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), brw_imm_ud(instr->const_index[0])); } /* Read the vector */ do_untyped_vector_read(bld, dest, surf_index, offset_reg, instr->num_components); break; } case nir_intrinsic_store_shared: { assert(devinfo->gen >= 7); /* Block index */ fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM); /* Value */ fs_reg val_reg = get_nir_src(instr->src[0]); /* Writemask */ unsigned writemask = instr->const_index[1]; /* get_nir_src() retypes to integer. Be wary of 64-bit types though * since the untyped writes below operate in units of 32-bits, which * means that we need to write twice as many components each time. * Also, we have to suffle 64-bit data to be in the appropriate layout * expected by our 32-bit write messages. */ unsigned type_size = 4; if (nir_src_bit_size(instr->src[0]) == 64) { type_size = 8; val_reg = shuffle_for_32bit_write(bld, val_reg, 0, instr->num_components); } unsigned type_slots = type_size / 4; /* Combine groups of consecutive enabled channels in one write * message. We use ffs to find the first enabled channel and then ffs on * the bit-inverse, down-shifted writemask to determine the length of * the block of enabled bits. */ while (writemask) { unsigned first_component = ffs(writemask) - 1; unsigned length = ffs(~(writemask >> first_component)) - 1; /* We can't write more than 2 64-bit components at once. Limit the * length of the write to what we can do and let the next iteration * handle the rest */ if (type_size > 4) length = MIN2(2, length); fs_reg offset_reg; nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]); if (const_offset) { offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0] + type_size * first_component); } else { offset_reg = vgrf(glsl_type::uint_type); bld.ADD(offset_reg, retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD), brw_imm_ud(instr->const_index[0] + type_size * first_component)); } emit_untyped_write(bld, surf_index, offset_reg, offset(val_reg, bld, first_component * type_slots), 1 /* dims */, length * type_slots, BRW_PREDICATE_NONE); /* Clear the bits in the writemask that we just wrote, then try * again to see if more channels are left. */ writemask &= (15 << (first_component + length)); } break; } default: nir_emit_intrinsic(bld, instr); break; } } static fs_reg brw_nir_reduction_op_identity(const fs_builder &bld, nir_op op, brw_reg_type type) { nir_const_value value = nir_alu_binop_identity(op, type_sz(type) * 8); switch (type_sz(type)) { case 2: assert(type != BRW_REGISTER_TYPE_HF); return retype(brw_imm_uw(value.u16[0]), type); case 4: return retype(brw_imm_ud(value.u32[0]), type); case 8: if (type == BRW_REGISTER_TYPE_DF) return setup_imm_df(bld, value.f64[0]); else return retype(brw_imm_u64(value.u64[0]), type); default: unreachable("Invalid type size"); } } static opcode brw_op_for_nir_reduction_op(nir_op op) { switch (op) { case nir_op_iadd: return BRW_OPCODE_ADD; case nir_op_fadd: return BRW_OPCODE_ADD; case nir_op_imul: return BRW_OPCODE_MUL; case nir_op_fmul: return BRW_OPCODE_MUL; case nir_op_imin: return BRW_OPCODE_SEL; case nir_op_umin: return BRW_OPCODE_SEL; case nir_op_fmin: return BRW_OPCODE_SEL; case nir_op_imax: return BRW_OPCODE_SEL; case nir_op_umax: return BRW_OPCODE_SEL; case nir_op_fmax: return BRW_OPCODE_SEL; case nir_op_iand: return BRW_OPCODE_AND; case nir_op_ior: return BRW_OPCODE_OR; case nir_op_ixor: return BRW_OPCODE_XOR; default: unreachable("Invalid reduction operation"); } } static brw_conditional_mod brw_cond_mod_for_nir_reduction_op(nir_op op) { switch (op) { case nir_op_iadd: return BRW_CONDITIONAL_NONE; case nir_op_fadd: return BRW_CONDITIONAL_NONE; case nir_op_imul: return BRW_CONDITIONAL_NONE; case nir_op_fmul: return BRW_CONDITIONAL_NONE; case nir_op_imin: return BRW_CONDITIONAL_L; case nir_op_umin: return BRW_CONDITIONAL_L; case nir_op_fmin: return BRW_CONDITIONAL_L; case nir_op_imax: return BRW_CONDITIONAL_GE; case nir_op_umax: return BRW_CONDITIONAL_GE; case nir_op_fmax: return BRW_CONDITIONAL_GE; case nir_op_iand: return BRW_CONDITIONAL_NONE; case nir_op_ior: return BRW_CONDITIONAL_NONE; case nir_op_ixor: return BRW_CONDITIONAL_NONE; default: unreachable("Invalid reduction operation"); } } fs_reg fs_visitor::get_nir_image_intrinsic_image(const brw::fs_builder &bld, nir_intrinsic_instr *instr) { fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD); if (stage_prog_data->binding_table.image_start > 0) { if (image.file == BRW_IMMEDIATE_VALUE) { image.d += stage_prog_data->binding_table.image_start; } else { bld.ADD(image, image, brw_imm_d(stage_prog_data->binding_table.image_start)); } } return bld.emit_uniformize(image); } static unsigned image_intrinsic_coord_components(nir_intrinsic_instr *instr) { switch (nir_intrinsic_image_dim(instr)) { case GLSL_SAMPLER_DIM_1D: return 1 + nir_intrinsic_image_array(instr); case GLSL_SAMPLER_DIM_2D: case GLSL_SAMPLER_DIM_RECT: return 2 + nir_intrinsic_image_array(instr); case GLSL_SAMPLER_DIM_3D: case GLSL_SAMPLER_DIM_CUBE: return 3; case GLSL_SAMPLER_DIM_BUF: return 1; case GLSL_SAMPLER_DIM_MS: return 2 + nir_intrinsic_image_array(instr); default: unreachable("Invalid image dimension"); } } void fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_min: case nir_intrinsic_image_atomic_max: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: case nir_intrinsic_image_atomic_exchange: case nir_intrinsic_image_atomic_comp_swap: { if (stage == MESA_SHADER_FRAGMENT && instr->intrinsic != nir_intrinsic_image_load) brw_wm_prog_data(prog_data)->has_side_effects = true; /* Get some metadata from the image intrinsic. */ const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic]; const unsigned dims = image_intrinsic_coord_components(instr); const GLenum format = nir_intrinsic_format(instr); const unsigned dest_components = nir_intrinsic_dest_components(instr); /* Get the arguments of the image intrinsic. */ const fs_reg image = get_nir_image_intrinsic_image(bld, instr); const fs_reg coords = retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD); fs_reg tmp; /* Emit an image load, store or atomic op. */ if (instr->intrinsic == nir_intrinsic_image_load) { tmp = emit_typed_read(bld, image, coords, dims, instr->num_components); } else if (instr->intrinsic == nir_intrinsic_image_store) { const fs_reg src0 = get_nir_src(instr->src[3]); emit_typed_write(bld, image, coords, src0, dims, instr->num_components); } else { int op; unsigned num_srcs = info->num_srcs; switch (instr->intrinsic) { case nir_intrinsic_image_atomic_add: assert(num_srcs == 4); op = get_op_for_atomic_add(instr, 3); if (op != BRW_AOP_ADD) num_srcs = 3; break; case nir_intrinsic_image_atomic_min: assert(format == GL_R32UI || format == GL_R32I); op = (format == GL_R32I) ? BRW_AOP_IMIN : BRW_AOP_UMIN; break; case nir_intrinsic_image_atomic_max: assert(format == GL_R32UI || format == GL_R32I); op = (format == GL_R32I) ? BRW_AOP_IMAX : BRW_AOP_UMAX; break; case nir_intrinsic_image_atomic_and: op = BRW_AOP_AND; break; case nir_intrinsic_image_atomic_or: op = BRW_AOP_OR; break; case nir_intrinsic_image_atomic_xor: op = BRW_AOP_XOR; break; case nir_intrinsic_image_atomic_exchange: op = BRW_AOP_MOV; break; case nir_intrinsic_image_atomic_comp_swap: op = BRW_AOP_CMPWR; break; default: unreachable("Not reachable."); } const fs_reg src0 = (num_srcs >= 4 ? get_nir_src(instr->src[3]) : fs_reg()); const fs_reg src1 = (num_srcs >= 5 ? get_nir_src(instr->src[4]) : fs_reg()); tmp = emit_typed_atomic(bld, image, coords, src0, src1, dims, 1, op); } /* Assign the result. */ for (unsigned c = 0; c < dest_components; ++c) { bld.MOV(offset(retype(dest, tmp.type), bld, c), offset(tmp, bld, c)); } break; } case nir_intrinsic_image_size: { /* Unlike the [un]typed load and store opcodes, the TXS that this turns * into will handle the binding table index for us in the geneerator. */ fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD); image = bld.emit_uniformize(image); /* Since the image size is always uniform, we can just emit a SIMD8 * query instruction and splat the result out. */ const fs_builder ubld = bld.exec_all().group(8, 0); /* The LOD also serves as the message payload */ fs_reg lod = ubld.vgrf(BRW_REGISTER_TYPE_UD); ubld.MOV(lod, brw_imm_ud(0)); fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4); fs_inst *inst = ubld.emit(SHADER_OPCODE_IMAGE_SIZE, tmp, lod, image); inst->mlen = 1; inst->size_written = 4 * REG_SIZE; for (unsigned c = 0; c < instr->dest.ssa.num_components; ++c) { if (c == 2 && nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_CUBE) { bld.emit(SHADER_OPCODE_INT_QUOTIENT, offset(retype(dest, tmp.type), bld, c), component(offset(tmp, ubld, c), 0), brw_imm_ud(6)); } else { bld.MOV(offset(retype(dest, tmp.type), bld, c), component(offset(tmp, ubld, c), 0)); } } break; } case nir_intrinsic_image_load_raw_intel: { const fs_reg image = get_nir_image_intrinsic_image(bld, instr); const fs_reg addr = retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD); fs_reg tmp = emit_untyped_read(bld, image, addr, 1, instr->num_components); for (unsigned c = 0; c < instr->num_components; ++c) { bld.MOV(offset(retype(dest, tmp.type), bld, c), offset(tmp, bld, c)); } break; } case nir_intrinsic_image_store_raw_intel: { const fs_reg image = get_nir_image_intrinsic_image(bld, instr); const fs_reg addr = retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD); const fs_reg data = retype(get_nir_src(instr->src[2]), BRW_REGISTER_TYPE_UD); brw_wm_prog_data(prog_data)->has_side_effects = true; emit_untyped_write(bld, image, addr, data, 1, instr->num_components); break; } case nir_intrinsic_group_memory_barrier: case nir_intrinsic_memory_barrier_shared: case nir_intrinsic_memory_barrier_atomic_counter: case nir_intrinsic_memory_barrier_buffer: case nir_intrinsic_memory_barrier_image: case nir_intrinsic_memory_barrier: { const fs_builder ubld = bld.group(8, 0); const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2); ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp) ->size_written = 2 * REG_SIZE; break; } case nir_intrinsic_shader_clock: { /* We cannot do anything if there is an event, so ignore it for now */ const fs_reg shader_clock = get_timestamp(bld); const fs_reg srcs[] = { component(shader_clock, 0), component(shader_clock, 1) }; bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0); break; } case nir_intrinsic_image_samples: /* The driver does not support multi-sampled images. */ bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(1)); break; case nir_intrinsic_load_uniform: { /* Offsets are in bytes but they should always aligned to * the type size */ assert(instr->const_index[0] % 4 == 0 || instr->const_index[0] % type_sz(dest.type) == 0); fs_reg src(UNIFORM, instr->const_index[0] / 4, dest.type); nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); if (const_offset) { assert(const_offset->u32[0] % type_sz(dest.type) == 0); /* For 16-bit types we add the module of the const_index[0] * offset to access to not 32-bit aligned element */ src.offset = const_offset->u32[0] + instr->const_index[0] % 4; for (unsigned j = 0; j < instr->num_components; j++) { bld.MOV(offset(dest, bld, j), offset(src, bld, j)); } } else { fs_reg indirect = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD); /* We need to pass a size to the MOV_INDIRECT but we don't want it to * go past the end of the uniform. In order to keep the n'th * component from running past, we subtract off the size of all but * one component of the vector. */ assert(instr->const_index[1] >= instr->num_components * (int) type_sz(dest.type)); unsigned read_size = instr->const_index[1] - (instr->num_components - 1) * type_sz(dest.type); bool supports_64bit_indirects = !devinfo->is_cherryview && !gen_device_info_is_9lp(devinfo); if (type_sz(dest.type) != 8 || supports_64bit_indirects) { for (unsigned j = 0; j < instr->num_components; j++) { bld.emit(SHADER_OPCODE_MOV_INDIRECT, offset(dest, bld, j), offset(src, bld, j), indirect, brw_imm_ud(read_size)); } } else { const unsigned num_mov_indirects = type_sz(dest.type) / type_sz(BRW_REGISTER_TYPE_UD); /* We read a little bit less per MOV INDIRECT, as they are now * 32-bits ones instead of 64-bit. Fix read_size then. */ const unsigned read_size_32bit = read_size - (num_mov_indirects - 1) * type_sz(BRW_REGISTER_TYPE_UD); for (unsigned j = 0; j < instr->num_components; j++) { for (unsigned i = 0; i < num_mov_indirects; i++) { bld.emit(SHADER_OPCODE_MOV_INDIRECT, subscript(offset(dest, bld, j), BRW_REGISTER_TYPE_UD, i), subscript(offset(src, bld, j), BRW_REGISTER_TYPE_UD, i), indirect, brw_imm_ud(read_size_32bit)); } } } } break; } case nir_intrinsic_load_ubo: { nir_const_value *const_index = nir_src_as_const_value(instr->src[0]); fs_reg surf_index; if (const_index) { const unsigned index = stage_prog_data->binding_table.ubo_start + const_index->u32[0]; surf_index = brw_imm_ud(index); brw_mark_surface_used(prog_data, index); } else { /* The block index is not a constant. Evaluate the index expression * per-channel and add the base UBO index; we have to select a value * from any live channel. */ surf_index = vgrf(glsl_type::uint_type); bld.ADD(surf_index, get_nir_src(instr->src[0]), brw_imm_ud(stage_prog_data->binding_table.ubo_start)); surf_index = bld.emit_uniformize(surf_index); /* Assume this may touch any UBO. It would be nice to provide * a tighter bound, but the array information is already lowered away. */ brw_mark_surface_used(prog_data, stage_prog_data->binding_table.ubo_start + nir->info.num_ubos - 1); } nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]); if (const_offset == NULL) { fs_reg base_offset = retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD); for (int i = 0; i < instr->num_components; i++) VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index, base_offset, i * type_sz(dest.type)); } else { /* Even if we are loading doubles, a pull constant load will load * a 32-bit vec4, so should only reserve vgrf space for that. If we * need to load a full dvec4 we will have to emit 2 loads. This is * similar to demote_pull_constants(), except that in that case we * see individual accesses to each component of the vector and then * we let CSE deal with duplicate loads. Here we see a vector access * and we have to split it if necessary. */ const unsigned type_size = type_sz(dest.type); /* See if we've selected this as a push constant candidate */ if (const_index) { const unsigned ubo_block = const_index->u32[0]; const unsigned offset_256b = const_offset->u32[0] / 32; fs_reg push_reg; for (int i = 0; i < 4; i++) { const struct brw_ubo_range *range = &prog_data->ubo_ranges[i]; if (range->block == ubo_block && offset_256b >= range->start && offset_256b < range->start + range->length) { push_reg = fs_reg(UNIFORM, UBO_START + i, dest.type); push_reg.offset = const_offset->u32[0] - 32 * range->start; break; } } if (push_reg.file != BAD_FILE) { for (unsigned i = 0; i < instr->num_components; i++) { bld.MOV(offset(dest, bld, i), byte_offset(push_reg, i * type_size)); } break; } } const unsigned block_sz = 64; /* Fetch one cacheline at a time. */ const fs_builder ubld = bld.exec_all().group(block_sz / 4, 0); const fs_reg packed_consts = ubld.vgrf(BRW_REGISTER_TYPE_UD); for (unsigned c = 0; c < instr->num_components;) { const unsigned base = const_offset->u32[0] + c * type_size; /* Number of usable components in the next block-aligned load. */ const unsigned count = MIN2(instr->num_components - c, (block_sz - base % block_sz) / type_size); ubld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD, packed_consts, surf_index, brw_imm_ud(base & ~(block_sz - 1))); const fs_reg consts = retype(byte_offset(packed_consts, base & (block_sz - 1)), dest.type); for (unsigned d = 0; d < count; d++) bld.MOV(offset(dest, bld, c + d), component(consts, d)); c += count; } } break; } case nir_intrinsic_load_ssbo: { assert(devinfo->gen >= 7); nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[0]); fs_reg surf_index; if (const_uniform_block) { unsigned index = stage_prog_data->binding_table.ssbo_start + const_uniform_block->u32[0]; surf_index = brw_imm_ud(index); brw_mark_surface_used(prog_data, index); } else { surf_index = vgrf(glsl_type::uint_type); bld.ADD(surf_index, get_nir_src(instr->src[0]), brw_imm_ud(stage_prog_data->binding_table.ssbo_start)); /* Assume this may touch any UBO. It would be nice to provide * a tighter bound, but the array information is already lowered away. */ brw_mark_surface_used(prog_data, stage_prog_data->binding_table.ssbo_start + nir->info.num_ssbos - 1); } fs_reg offset_reg; nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]); if (const_offset) { offset_reg = brw_imm_ud(const_offset->u32[0]); } else { offset_reg = retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD); } /* Read the vector */ do_untyped_vector_read(bld, dest, surf_index, offset_reg, instr->num_components); break; } case nir_intrinsic_store_ssbo: { assert(devinfo->gen >= 7); if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; /* Block index */ fs_reg surf_index; nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[1]); if (const_uniform_block) { unsigned index = stage_prog_data->binding_table.ssbo_start + const_uniform_block->u32[0]; surf_index = brw_imm_ud(index); brw_mark_surface_used(prog_data, index); } else { surf_index = vgrf(glsl_type::uint_type); bld.ADD(surf_index, get_nir_src(instr->src[1]), brw_imm_ud(stage_prog_data->binding_table.ssbo_start)); brw_mark_surface_used(prog_data, stage_prog_data->binding_table.ssbo_start + nir->info.num_ssbos - 1); } /* Value */ fs_reg val_reg = get_nir_src(instr->src[0]); /* Writemask */ unsigned writemask = instr->const_index[0]; /* get_nir_src() retypes to integer. Be wary of 64-bit types though * since the untyped writes below operate in units of 32-bits, which * means that we need to write twice as many components each time. * Also, we have to suffle 64-bit data to be in the appropriate layout * expected by our 32-bit write messages. */ unsigned bit_size = nir_src_bit_size(instr->src[0]); unsigned type_size = bit_size / 8; /* Combine groups of consecutive enabled channels in one write * message. We use ffs to find the first enabled channel and then ffs on * the bit-inverse, down-shifted writemask to determine the num_components * of the block of enabled bits. */ while (writemask) { unsigned first_component = ffs(writemask) - 1; unsigned num_components = ffs(~(writemask >> first_component)) - 1; fs_reg write_src = offset(val_reg, bld, first_component); nir_const_value *const_offset = nir_src_as_const_value(instr->src[2]); if (type_size > 4) { /* We can't write more than 2 64-bit components at once. Limit * the num_components of the write to what we can do and let the next * iteration handle the rest. */ num_components = MIN2(2, num_components); write_src = shuffle_for_32bit_write(bld, write_src, 0, num_components); } else if (type_size < 4) { /* For 16-bit types we pack two consecutive values into a 32-bit * word and use an untyped write message. For single values or not * 32-bit-aligned we need to use byte-scattered writes because * untyped writes works with 32-bit components with 32-bit * alignment. byte_scattered_write messages only support one * 16-bit component at a time. As VK_KHR_relaxed_block_layout * could be enabled we can not guarantee that not constant offsets * to be 32-bit aligned for 16-bit types. For example an array, of * 16-bit vec3 with array element stride of 6. * * In the case of 32-bit aligned constant offsets if there is * a 3-components vector we submit one untyped-write message * of 32-bit (first two components), and one byte-scattered * write message (the last component). */ if ( !const_offset || ((const_offset->u32[0] + type_size * first_component) % 4)) { /* If we use a .yz writemask we also need to emit 2 * byte-scattered write messages because of y-component not * being aligned to 32-bit. */ num_components = 1; } else if (num_components * type_size > 4 && (num_components * type_size % 4)) { /* If the pending components size is not a multiple of 4 bytes * we left the not aligned components for following emits of * length == 1 with byte_scattered_write. */ num_components -= (num_components * type_size % 4) / type_size; } else if (num_components * type_size < 4) { num_components = 1; } /* For num_components == 1 we are also shuffling the component * because byte scattered writes of 16-bit need values to be dword * aligned. Shuffling only one component would be the same as * striding it. */ write_src = shuffle_for_32bit_write(bld, write_src, 0, num_components); } fs_reg offset_reg; if (const_offset) { offset_reg = brw_imm_ud(const_offset->u32[0] + type_size * first_component); } else { offset_reg = vgrf(glsl_type::uint_type); bld.ADD(offset_reg, retype(get_nir_src(instr->src[2]), BRW_REGISTER_TYPE_UD), brw_imm_ud(type_size * first_component)); } if (type_size < 4 && num_components == 1) { /* Untyped Surface messages have a fixed 32-bit size, so we need * to rely on byte scattered in order to write 16-bit elements. * The byte_scattered_write message needs that every written 16-bit * type to be aligned 32-bits (stride=2). */ emit_byte_scattered_write(bld, surf_index, offset_reg, write_src, 1 /* dims */, bit_size, BRW_PREDICATE_NONE); } else { assert(num_components * type_size <= 16); assert((num_components * type_size) % 4 == 0); assert(offset_reg.file != BRW_IMMEDIATE_VALUE || offset_reg.ud % 4 == 0); unsigned num_slots = (num_components * type_size) / 4; emit_untyped_write(bld, surf_index, offset_reg, write_src, 1 /* dims */, num_slots, BRW_PREDICATE_NONE); } /* Clear the bits in the writemask that we just wrote, then try * again to see if more channels are left. */ writemask &= (15 << (first_component + num_components)); } break; } case nir_intrinsic_store_output: { fs_reg src = get_nir_src(instr->src[0]); nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]); assert(const_offset && "Indirect output stores not allowed"); unsigned num_components = instr->num_components; unsigned first_component = nir_intrinsic_component(instr); if (nir_src_bit_size(instr->src[0]) == 64) { src = shuffle_for_32bit_write(bld, src, 0, num_components); num_components *= 2; } fs_reg new_dest = retype(offset(outputs[instr->const_index[0]], bld, 4 * const_offset->u32[0]), src.type); for (unsigned j = 0; j < num_components; j++) { bld.MOV(offset(new_dest, bld, j + first_component), offset(src, bld, j)); } break; } case nir_intrinsic_ssbo_atomic_add: nir_emit_ssbo_atomic(bld, get_op_for_atomic_add(instr, 2), instr); break; case nir_intrinsic_ssbo_atomic_imin: nir_emit_ssbo_atomic(bld, BRW_AOP_IMIN, instr); break; case nir_intrinsic_ssbo_atomic_umin: nir_emit_ssbo_atomic(bld, BRW_AOP_UMIN, instr); break; case nir_intrinsic_ssbo_atomic_imax: nir_emit_ssbo_atomic(bld, BRW_AOP_IMAX, instr); break; case nir_intrinsic_ssbo_atomic_umax: nir_emit_ssbo_atomic(bld, BRW_AOP_UMAX, instr); break; case nir_intrinsic_ssbo_atomic_and: nir_emit_ssbo_atomic(bld, BRW_AOP_AND, instr); break; case nir_intrinsic_ssbo_atomic_or: nir_emit_ssbo_atomic(bld, BRW_AOP_OR, instr); break; case nir_intrinsic_ssbo_atomic_xor: nir_emit_ssbo_atomic(bld, BRW_AOP_XOR, instr); break; case nir_intrinsic_ssbo_atomic_exchange: nir_emit_ssbo_atomic(bld, BRW_AOP_MOV, instr); break; case nir_intrinsic_ssbo_atomic_comp_swap: nir_emit_ssbo_atomic(bld, BRW_AOP_CMPWR, instr); break; case nir_intrinsic_ssbo_atomic_fmin: nir_emit_ssbo_atomic_float(bld, BRW_AOP_FMIN, instr); break; case nir_intrinsic_ssbo_atomic_fmax: nir_emit_ssbo_atomic_float(bld, BRW_AOP_FMAX, instr); break; case nir_intrinsic_ssbo_atomic_fcomp_swap: nir_emit_ssbo_atomic_float(bld, BRW_AOP_FCMPWR, instr); break; case nir_intrinsic_get_buffer_size: { nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[0]); unsigned ssbo_index = const_uniform_block ? const_uniform_block->u32[0] : 0; /* A resinfo's sampler message is used to get the buffer size. The * SIMD8's writeback message consists of four registers and SIMD16's * writeback message consists of 8 destination registers (two per each * component). Because we are only interested on the first channel of * the first returned component, where resinfo returns the buffer size * for SURFTYPE_BUFFER, we can just use the SIMD8 variant regardless of * the dispatch width. */ const fs_builder ubld = bld.exec_all().group(8, 0); fs_reg src_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD); fs_reg ret_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4); /* Set LOD = 0 */ ubld.MOV(src_payload, brw_imm_d(0)); const unsigned index = prog_data->binding_table.ssbo_start + ssbo_index; fs_inst *inst = ubld.emit(SHADER_OPCODE_GET_BUFFER_SIZE, ret_payload, src_payload, brw_imm_ud(index)); inst->header_size = 0; inst->mlen = 1; inst->size_written = 4 * REG_SIZE; /* SKL PRM, vol07, 3D Media GPGPU Engine, Bounds Checking and Faulting: * * "Out-of-bounds checking is always performed at a DWord granularity. If * any part of the DWord is out-of-bounds then the whole DWord is * considered out-of-bounds." * * This implies that types with size smaller than 4-bytes need to be * padded if they don't complete the last dword of the buffer. But as we * need to maintain the original size we need to reverse the padding * calculation to return the correct size to know the number of elements * of an unsized array. As we stored in the last two bits of the surface * size the needed padding for the buffer, we calculate here the * original buffer_size reversing the surface_size calculation: * * surface_size = isl_align(buffer_size, 4) + * (isl_align(buffer_size) - buffer_size) * * buffer_size = surface_size & ~3 - surface_size & 3 */ fs_reg size_aligned4 = ubld.vgrf(BRW_REGISTER_TYPE_UD); fs_reg size_padding = ubld.vgrf(BRW_REGISTER_TYPE_UD); fs_reg buffer_size = ubld.vgrf(BRW_REGISTER_TYPE_UD); ubld.AND(size_padding, ret_payload, brw_imm_ud(3)); ubld.AND(size_aligned4, ret_payload, brw_imm_ud(~3)); ubld.ADD(buffer_size, size_aligned4, negate(size_padding)); bld.MOV(retype(dest, ret_payload.type), component(buffer_size, 0)); brw_mark_surface_used(prog_data, index); break; } case nir_intrinsic_load_subgroup_invocation: bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]); break; case nir_intrinsic_load_subgroup_eq_mask: case nir_intrinsic_load_subgroup_ge_mask: case nir_intrinsic_load_subgroup_gt_mask: case nir_intrinsic_load_subgroup_le_mask: case nir_intrinsic_load_subgroup_lt_mask: unreachable("not reached"); case nir_intrinsic_vote_any: { const fs_builder ubld = bld.exec_all().group(1, 0); /* The any/all predicates do not consider channel enables. To prevent * dead channels from affecting the result, we initialize the flag with * with the identity value for the logical operation. */ if (dispatch_width == 32) { /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */ ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD), brw_imm_ud(0)); } else { ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0)); } bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); /* For some reason, the any/all predicates don't work properly with * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H * doesn't read the correct subset of the flag register and you end up * getting garbage in the second half. Work around this by using a pair * of 1-wide MOVs and scattering the result. */ fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D); ubld.MOV(res1, brw_imm_d(0)); set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ANY8H : dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ANY16H : BRW_PREDICATE_ALIGN1_ANY32H, ubld.MOV(res1, brw_imm_d(-1))); bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0)); break; } case nir_intrinsic_vote_all: { const fs_builder ubld = bld.exec_all().group(1, 0); /* The any/all predicates do not consider channel enables. To prevent * dead channels from affecting the result, we initialize the flag with * with the identity value for the logical operation. */ if (dispatch_width == 32) { /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */ ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD), brw_imm_ud(0xffffffff)); } else { ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0xffff)); } bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ); /* For some reason, the any/all predicates don't work properly with * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H * doesn't read the correct subset of the flag register and you end up * getting garbage in the second half. Work around this by using a pair * of 1-wide MOVs and scattering the result. */ fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D); ubld.MOV(res1, brw_imm_d(0)); set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ALL8H : dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ALL16H : BRW_PREDICATE_ALIGN1_ALL32H, ubld.MOV(res1, brw_imm_d(-1))); bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0)); break; } case nir_intrinsic_vote_feq: case nir_intrinsic_vote_ieq: { fs_reg value = get_nir_src(instr->src[0]); if (instr->intrinsic == nir_intrinsic_vote_feq) { const unsigned bit_size = nir_src_bit_size(instr->src[0]); value.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_F); } fs_reg uniformized = bld.emit_uniformize(value); const fs_builder ubld = bld.exec_all().group(1, 0); /* The any/all predicates do not consider channel enables. To prevent * dead channels from affecting the result, we initialize the flag with * with the identity value for the logical operation. */ if (dispatch_width == 32) { /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */ ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD), brw_imm_ud(0xffffffff)); } else { ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0xffff)); } bld.CMP(bld.null_reg_d(), value, uniformized, BRW_CONDITIONAL_Z); /* For some reason, the any/all predicates don't work properly with * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H * doesn't read the correct subset of the flag register and you end up * getting garbage in the second half. Work around this by using a pair * of 1-wide MOVs and scattering the result. */ fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D); ubld.MOV(res1, brw_imm_d(0)); set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ALL8H : dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ALL16H : BRW_PREDICATE_ALIGN1_ALL32H, ubld.MOV(res1, brw_imm_d(-1))); bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0)); break; } case nir_intrinsic_ballot: { const fs_reg value = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD); struct brw_reg flag = brw_flag_reg(0, 0); /* FIXME: For SIMD32 programs, this causes us to stomp on f0.1 as well * as f0.0. This is a problem for fragment programs as we currently use * f0.1 for discards. Fortunately, we don't support SIMD32 fragment * programs yet so this isn't a problem. When we do, something will * have to change. */ if (dispatch_width == 32) flag.type = BRW_REGISTER_TYPE_UD; bld.exec_all().group(1, 0).MOV(flag, brw_imm_ud(0u)); bld.CMP(bld.null_reg_ud(), value, brw_imm_ud(0u), BRW_CONDITIONAL_NZ); if (instr->dest.ssa.bit_size > 32) { dest.type = BRW_REGISTER_TYPE_UQ; } else { dest.type = BRW_REGISTER_TYPE_UD; } bld.MOV(dest, flag); break; } case nir_intrinsic_read_invocation: { const fs_reg value = get_nir_src(instr->src[0]); const fs_reg invocation = get_nir_src(instr->src[1]); fs_reg tmp = bld.vgrf(value.type); bld.exec_all().emit(SHADER_OPCODE_BROADCAST, tmp, value, bld.emit_uniformize(invocation)); bld.MOV(retype(dest, value.type), fs_reg(component(tmp, 0))); break; } case nir_intrinsic_read_first_invocation: { const fs_reg value = get_nir_src(instr->src[0]); bld.MOV(retype(dest, value.type), bld.emit_uniformize(value)); break; } case nir_intrinsic_shuffle: { const fs_reg value = get_nir_src(instr->src[0]); const fs_reg index = get_nir_src(instr->src[1]); bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, index); break; } case nir_intrinsic_first_invocation: { fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.exec_all().emit(SHADER_OPCODE_FIND_LIVE_CHANNEL, tmp); bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), fs_reg(component(tmp, 0))); break; } case nir_intrinsic_quad_broadcast: { const fs_reg value = get_nir_src(instr->src[0]); nir_const_value *index = nir_src_as_const_value(instr->src[1]); assert(nir_src_bit_size(instr->src[1]) == 32); bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, retype(dest, value.type), value, brw_imm_ud(index->u32[0]), brw_imm_ud(4)); break; } case nir_intrinsic_quad_swap_horizontal: { const fs_reg value = get_nir_src(instr->src[0]); const fs_reg tmp = bld.vgrf(value.type); const fs_builder ubld = bld.exec_all().group(dispatch_width / 2, 0); const fs_reg src_left = horiz_stride(value, 2); const fs_reg src_right = horiz_stride(horiz_offset(value, 1), 2); const fs_reg tmp_left = horiz_stride(tmp, 2); const fs_reg tmp_right = horiz_stride(horiz_offset(tmp, 1), 2); /* From the Cherryview PRM Vol. 7, "Register Region Restrictiosn": * * "When source or destination datatype is 64b or operation is * integer DWord multiply, regioning in Align1 must follow * these rules: * * [...] * * 3. Source and Destination offset must be the same, except * the case of scalar source." * * In order to work around this, we have to emit two 32-bit MOVs instead * of a single 64-bit MOV to do the shuffle. */ if (type_sz(value.type) > 4 && (devinfo->is_cherryview || gen_device_info_is_9lp(devinfo))) { ubld.MOV(subscript(tmp_left, BRW_REGISTER_TYPE_D, 0), subscript(src_right, BRW_REGISTER_TYPE_D, 0)); ubld.MOV(subscript(tmp_left, BRW_REGISTER_TYPE_D, 1), subscript(src_right, BRW_REGISTER_TYPE_D, 1)); ubld.MOV(subscript(tmp_right, BRW_REGISTER_TYPE_D, 0), subscript(src_left, BRW_REGISTER_TYPE_D, 0)); ubld.MOV(subscript(tmp_right, BRW_REGISTER_TYPE_D, 1), subscript(src_left, BRW_REGISTER_TYPE_D, 1)); } else { ubld.MOV(tmp_left, src_right); ubld.MOV(tmp_right, src_left); } bld.MOV(retype(dest, value.type), tmp); break; } case nir_intrinsic_quad_swap_vertical: { const fs_reg value = get_nir_src(instr->src[0]); if (nir_src_bit_size(instr->src[0]) == 32) { /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */ const fs_reg tmp = bld.vgrf(value.type); const fs_builder ubld = bld.exec_all(); ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value, brw_imm_ud(BRW_SWIZZLE4(2,3,0,1))); bld.MOV(retype(dest, value.type), tmp); } else { /* For larger data types, we have to either emit dispatch_width many * MOVs or else fall back to doing indirects. */ fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W); bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION], brw_imm_w(0x2)); bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx); } break; } case nir_intrinsic_quad_swap_diagonal: { const fs_reg value = get_nir_src(instr->src[0]); if (nir_src_bit_size(instr->src[0]) == 32) { /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */ const fs_reg tmp = bld.vgrf(value.type); const fs_builder ubld = bld.exec_all(); ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value, brw_imm_ud(BRW_SWIZZLE4(3,2,1,0))); bld.MOV(retype(dest, value.type), tmp); } else { /* For larger data types, we have to either emit dispatch_width many * MOVs or else fall back to doing indirects. */ fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W); bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION], brw_imm_w(0x3)); bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx); } break; } case nir_intrinsic_reduce: { fs_reg src = get_nir_src(instr->src[0]); nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr); unsigned cluster_size = nir_intrinsic_cluster_size(instr); if (cluster_size == 0 || cluster_size > dispatch_width) cluster_size = dispatch_width; /* Figure out the source type */ src.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[redop].input_types[0] | nir_src_bit_size(instr->src[0]))); fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type); opcode brw_op = brw_op_for_nir_reduction_op(redop); brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop); /* Set up a register for all of our scratching around and initialize it * to reduction operation's identity value. */ fs_reg scan = bld.vgrf(src.type); bld.exec_all().emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity); bld.emit_scan(brw_op, scan, cluster_size, cond_mod); dest.type = src.type; if (cluster_size * type_sz(src.type) >= REG_SIZE * 2) { /* In this case, CLUSTER_BROADCAST instruction isn't needed because * the distance between clusters is at least 2 GRFs. In this case, * we don't need the weird striding of the CLUSTER_BROADCAST * instruction and can just do regular MOVs. */ assert((cluster_size * type_sz(src.type)) % (REG_SIZE * 2) == 0); const unsigned groups = (dispatch_width * type_sz(src.type)) / (REG_SIZE * 2); const unsigned group_size = dispatch_width / groups; for (unsigned i = 0; i < groups; i++) { const unsigned cluster = (i * group_size) / cluster_size; const unsigned comp = cluster * cluster_size + (cluster_size - 1); bld.group(group_size, i).MOV(horiz_offset(dest, i * group_size), component(scan, comp)); } } else { bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, dest, scan, brw_imm_ud(cluster_size - 1), brw_imm_ud(cluster_size)); } break; } case nir_intrinsic_inclusive_scan: case nir_intrinsic_exclusive_scan: { fs_reg src = get_nir_src(instr->src[0]); nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr); /* Figure out the source type */ src.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_op_infos[redop].input_types[0] | nir_src_bit_size(instr->src[0]))); fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type); opcode brw_op = brw_op_for_nir_reduction_op(redop); brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop); /* Set up a register for all of our scratching around and initialize it * to reduction operation's identity value. */ fs_reg scan = bld.vgrf(src.type); const fs_builder allbld = bld.exec_all(); allbld.emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity); if (instr->intrinsic == nir_intrinsic_exclusive_scan) { /* Exclusive scan is a bit harder because we have to do an annoying * shift of the contents before we can begin. To make things worse, * we can't do this with a normal stride; we have to use indirects. */ fs_reg shifted = bld.vgrf(src.type); fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W); allbld.ADD(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION], brw_imm_w(-1)); allbld.emit(SHADER_OPCODE_SHUFFLE, shifted, scan, idx); allbld.group(1, 0).MOV(component(shifted, 0), identity); scan = shifted; } bld.emit_scan(brw_op, scan, dispatch_width, cond_mod); bld.MOV(retype(dest, src.type), scan); break; } case nir_intrinsic_begin_invocation_interlock: { const fs_builder ubld = bld.group(8, 0); const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2); ubld.emit(SHADER_OPCODE_INTERLOCK, tmp)->size_written = 2 * REG_SIZE; break; } case nir_intrinsic_end_invocation_interlock: { /* We don't need to do anything here */ break; } default: unreachable("unknown intrinsic"); } } void fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); fs_reg surface; nir_const_value *const_surface = nir_src_as_const_value(instr->src[0]); if (const_surface) { unsigned surf_index = stage_prog_data->binding_table.ssbo_start + const_surface->u32[0]; surface = brw_imm_ud(surf_index); brw_mark_surface_used(prog_data, surf_index); } else { surface = vgrf(glsl_type::uint_type); bld.ADD(surface, get_nir_src(instr->src[0]), brw_imm_ud(stage_prog_data->binding_table.ssbo_start)); /* Assume this may touch any SSBO. This is the same we do for other * UBO/SSBO accesses with non-constant surface. */ brw_mark_surface_used(prog_data, stage_prog_data->binding_table.ssbo_start + nir->info.num_ssbos - 1); } fs_reg offset = get_nir_src(instr->src[1]); fs_reg data1; if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC) data1 = get_nir_src(instr->src[2]); fs_reg data2; if (op == BRW_AOP_CMPWR) data2 = get_nir_src(instr->src[3]); /* Emit the actual atomic operation */ fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset, data1, data2, 1 /* dims */, 1 /* rsize */, op, BRW_PREDICATE_NONE); dest.type = atomic_result.type; bld.MOV(dest, atomic_result); } void fs_visitor::nir_emit_ssbo_atomic_float(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); fs_reg surface; nir_const_value *const_surface = nir_src_as_const_value(instr->src[0]); if (const_surface) { unsigned surf_index = stage_prog_data->binding_table.ssbo_start + const_surface->u32[0]; surface = brw_imm_ud(surf_index); brw_mark_surface_used(prog_data, surf_index); } else { surface = vgrf(glsl_type::uint_type); bld.ADD(surface, get_nir_src(instr->src[0]), brw_imm_ud(stage_prog_data->binding_table.ssbo_start)); /* Assume this may touch any SSBO. This is the same we do for other * UBO/SSBO accesses with non-constant surface. */ brw_mark_surface_used(prog_data, stage_prog_data->binding_table.ssbo_start + nir->info.num_ssbos - 1); } fs_reg offset = get_nir_src(instr->src[1]); fs_reg data1 = get_nir_src(instr->src[2]); fs_reg data2; if (op == BRW_AOP_FCMPWR) data2 = get_nir_src(instr->src[3]); /* Emit the actual atomic operation */ fs_reg atomic_result = emit_untyped_atomic_float(bld, surface, offset, data1, data2, 1 /* dims */, 1 /* rsize */, op, BRW_PREDICATE_NONE); dest.type = atomic_result.type; bld.MOV(dest, atomic_result); } void fs_visitor::nir_emit_shared_atomic(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); fs_reg surface = brw_imm_ud(GEN7_BTI_SLM); fs_reg offset; fs_reg data1; if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC) data1 = get_nir_src(instr->src[1]); fs_reg data2; if (op == BRW_AOP_CMPWR) data2 = get_nir_src(instr->src[2]); /* Get the offset */ nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); if (const_offset) { offset = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]); } else { offset = vgrf(glsl_type::uint_type); bld.ADD(offset, retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), brw_imm_ud(instr->const_index[0])); } /* Emit the actual atomic operation operation */ fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset, data1, data2, 1 /* dims */, 1 /* rsize */, op, BRW_PREDICATE_NONE); dest.type = atomic_result.type; bld.MOV(dest, atomic_result); } void fs_visitor::nir_emit_shared_atomic_float(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); fs_reg surface = brw_imm_ud(GEN7_BTI_SLM); fs_reg offset; fs_reg data1 = get_nir_src(instr->src[1]); fs_reg data2; if (op == BRW_AOP_FCMPWR) data2 = get_nir_src(instr->src[2]); /* Get the offset */ nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]); if (const_offset) { offset = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]); } else { offset = vgrf(glsl_type::uint_type); bld.ADD(offset, retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), brw_imm_ud(instr->const_index[0])); } /* Emit the actual atomic operation operation */ fs_reg atomic_result = emit_untyped_atomic_float(bld, surface, offset, data1, data2, 1 /* dims */, 1 /* rsize */, op, BRW_PREDICATE_NONE); dest.type = atomic_result.type; bld.MOV(dest, atomic_result); } void fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr) { unsigned texture = instr->texture_index; unsigned sampler = instr->sampler_index; fs_reg srcs[TEX_LOGICAL_NUM_SRCS]; srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture); srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(sampler); int lod_components = 0; /* The hardware requires a LOD for buffer textures */ if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_d(0); uint32_t header_bits = 0; for (unsigned i = 0; i < instr->num_srcs; i++) { fs_reg src = get_nir_src(instr->src[i].src); switch (instr->src[i].src_type) { case nir_tex_src_bias: srcs[TEX_LOGICAL_SRC_LOD] = retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); break; case nir_tex_src_comparator: srcs[TEX_LOGICAL_SRC_SHADOW_C] = retype(src, BRW_REGISTER_TYPE_F); break; case nir_tex_src_coord: switch (instr->op) { case nir_texop_txf: case nir_texop_txf_ms: case nir_texop_txf_ms_mcs: case nir_texop_samples_identical: srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_D); break; default: srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_F); break; } break; case nir_tex_src_ddx: srcs[TEX_LOGICAL_SRC_LOD] = retype(src, BRW_REGISTER_TYPE_F); lod_components = nir_tex_instr_src_size(instr, i); break; case nir_tex_src_ddy: srcs[TEX_LOGICAL_SRC_LOD2] = retype(src, BRW_REGISTER_TYPE_F); break; case nir_tex_src_lod: switch (instr->op) { case nir_texop_txs: srcs[TEX_LOGICAL_SRC_LOD] = retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_UD); break; case nir_texop_txf: srcs[TEX_LOGICAL_SRC_LOD] = retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_D); break; default: srcs[TEX_LOGICAL_SRC_LOD] = retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); break; } break; case nir_tex_src_ms_index: srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = retype(src, BRW_REGISTER_TYPE_UD); break; case nir_tex_src_offset: { nir_const_value *const_offset = nir_src_as_const_value(instr->src[i].src); unsigned offset_bits = 0; if (const_offset && brw_texture_offset(const_offset->i32, nir_tex_instr_src_size(instr, i), &offset_bits)) { header_bits |= offset_bits; } else { srcs[TEX_LOGICAL_SRC_TG4_OFFSET] = retype(src, BRW_REGISTER_TYPE_D); } break; } case nir_tex_src_projector: unreachable("should be lowered"); case nir_tex_src_texture_offset: { /* Figure out the highest possible texture index and mark it as used */ uint32_t max_used = texture + instr->texture_array_size - 1; if (instr->op == nir_texop_tg4 && devinfo->gen < 8) { max_used += stage_prog_data->binding_table.gather_texture_start; } else { max_used += stage_prog_data->binding_table.texture_start; } brw_mark_surface_used(prog_data, max_used); /* Emit code to evaluate the actual indexing expression */ fs_reg tmp = vgrf(glsl_type::uint_type); bld.ADD(tmp, src, brw_imm_ud(texture)); srcs[TEX_LOGICAL_SRC_SURFACE] = bld.emit_uniformize(tmp); break; } case nir_tex_src_sampler_offset: { /* Emit code to evaluate the actual indexing expression */ fs_reg tmp = vgrf(glsl_type::uint_type); bld.ADD(tmp, src, brw_imm_ud(sampler)); srcs[TEX_LOGICAL_SRC_SAMPLER] = bld.emit_uniformize(tmp); break; } case nir_tex_src_ms_mcs: assert(instr->op == nir_texop_txf_ms); srcs[TEX_LOGICAL_SRC_MCS] = retype(src, BRW_REGISTER_TYPE_D); break; case nir_tex_src_plane: { nir_const_value *const_plane = nir_src_as_const_value(instr->src[i].src); const uint32_t plane = const_plane->u32[0]; const uint32_t texture_index = instr->texture_index + stage_prog_data->binding_table.plane_start[plane] - stage_prog_data->binding_table.texture_start; srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture_index); break; } default: unreachable("unknown texture source"); } } if (srcs[TEX_LOGICAL_SRC_MCS].file == BAD_FILE && (instr->op == nir_texop_txf_ms || instr->op == nir_texop_samples_identical)) { if (devinfo->gen >= 7 && key_tex->compressed_multisample_layout_mask & (1 << texture)) { srcs[TEX_LOGICAL_SRC_MCS] = emit_mcs_fetch(srcs[TEX_LOGICAL_SRC_COORDINATE], instr->coord_components, srcs[TEX_LOGICAL_SRC_SURFACE]); } else { srcs[TEX_LOGICAL_SRC_MCS] = brw_imm_ud(0u); } } srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(instr->coord_components); srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(lod_components); enum opcode opcode; switch (instr->op) { case nir_texop_tex: opcode = (stage == MESA_SHADER_FRAGMENT ? SHADER_OPCODE_TEX_LOGICAL : SHADER_OPCODE_TXL_LOGICAL); break; case nir_texop_txb: opcode = FS_OPCODE_TXB_LOGICAL; break; case nir_texop_txl: opcode = SHADER_OPCODE_TXL_LOGICAL; break; case nir_texop_txd: opcode = SHADER_OPCODE_TXD_LOGICAL; break; case nir_texop_txf: opcode = SHADER_OPCODE_TXF_LOGICAL; break; case nir_texop_txf_ms: if ((key_tex->msaa_16 & (1 << sampler))) opcode = SHADER_OPCODE_TXF_CMS_W_LOGICAL; else opcode = SHADER_OPCODE_TXF_CMS_LOGICAL; break; case nir_texop_txf_ms_mcs: opcode = SHADER_OPCODE_TXF_MCS_LOGICAL; break; case nir_texop_query_levels: case nir_texop_txs: opcode = SHADER_OPCODE_TXS_LOGICAL; break; case nir_texop_lod: opcode = SHADER_OPCODE_LOD_LOGICAL; break; case nir_texop_tg4: if (srcs[TEX_LOGICAL_SRC_TG4_OFFSET].file != BAD_FILE) opcode = SHADER_OPCODE_TG4_OFFSET_LOGICAL; else opcode = SHADER_OPCODE_TG4_LOGICAL; break; case nir_texop_texture_samples: opcode = SHADER_OPCODE_SAMPLEINFO_LOGICAL; break; case nir_texop_samples_identical: { fs_reg dst = retype(get_nir_dest(instr->dest), BRW_REGISTER_TYPE_D); /* If mcs is an immediate value, it means there is no MCS. In that case * just return false. */ if (srcs[TEX_LOGICAL_SRC_MCS].file == BRW_IMMEDIATE_VALUE) { bld.MOV(dst, brw_imm_ud(0u)); } else if ((key_tex->msaa_16 & (1 << sampler))) { fs_reg tmp = vgrf(glsl_type::uint_type); bld.OR(tmp, srcs[TEX_LOGICAL_SRC_MCS], offset(srcs[TEX_LOGICAL_SRC_MCS], bld, 1)); bld.CMP(dst, tmp, brw_imm_ud(0u), BRW_CONDITIONAL_EQ); } else { bld.CMP(dst, srcs[TEX_LOGICAL_SRC_MCS], brw_imm_ud(0u), BRW_CONDITIONAL_EQ); } return; } default: unreachable("unknown texture opcode"); } if (instr->op == nir_texop_tg4) { if (instr->component == 1 && key_tex->gather_channel_quirk_mask & (1 << texture)) { /* gather4 sampler is broken for green channel on RG32F -- * we must ask for blue instead. */ header_bits |= 2 << 16; } else { header_bits |= instr->component << 16; } } fs_reg dst = bld.vgrf(brw_type_for_nir_type(devinfo, instr->dest_type), 4); fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs)); inst->offset = header_bits; const unsigned dest_size = nir_tex_instr_dest_size(instr); if (devinfo->gen >= 9 && instr->op != nir_texop_tg4 && instr->op != nir_texop_query_levels) { unsigned write_mask = instr->dest.is_ssa ? nir_ssa_def_components_read(&instr->dest.ssa): (1 << dest_size) - 1; assert(write_mask != 0); /* dead code should have been eliminated */ inst->size_written = util_last_bit(write_mask) * inst->dst.component_size(inst->exec_size); } else { inst->size_written = 4 * inst->dst.component_size(inst->exec_size); } if (srcs[TEX_LOGICAL_SRC_SHADOW_C].file != BAD_FILE) inst->shadow_compare = true; if (instr->op == nir_texop_tg4 && devinfo->gen == 6) emit_gen6_gather_wa(key_tex->gen6_gather_wa[texture], dst); fs_reg nir_dest[4]; for (unsigned i = 0; i < dest_size; i++) nir_dest[i] = offset(dst, bld, i); if (instr->op == nir_texop_query_levels) { /* # levels is in .w */ nir_dest[0] = offset(dst, bld, 3); } else if (instr->op == nir_texop_txs && dest_size >= 3 && devinfo->gen < 7) { /* Gen4-6 return 0 instead of 1 for single layer surfaces. */ fs_reg depth = offset(dst, bld, 2); nir_dest[2] = vgrf(glsl_type::int_type); bld.emit_minmax(nir_dest[2], depth, brw_imm_d(1), BRW_CONDITIONAL_GE); } bld.LOAD_PAYLOAD(get_nir_dest(instr->dest), nir_dest, dest_size, 0); } void fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr) { switch (instr->type) { case nir_jump_break: bld.emit(BRW_OPCODE_BREAK); break; case nir_jump_continue: bld.emit(BRW_OPCODE_CONTINUE); break; case nir_jump_return: default: unreachable("unknown jump"); } } /* * This helper takes a source register and un/shuffles it into the destination * register. * * If source type size is smaller than destination type size the operation * needed is a component shuffle. The opposite case would be an unshuffle. If * source/destination type size is equal a shuffle is done that would be * equivalent to a simple MOV. * * For example, if source is a 16-bit type and destination is 32-bit. A 3 * components .xyz 16-bit vector on SIMD8 would be. * * |x1|x2|x3|x4|x5|x6|x7|x8|y1|y2|y3|y4|y5|y6|y7|y8| * |z1|z2|z3|z4|z5|z6|z7|z8| | | | | | | | | * * This helper will return the following 2 32-bit components with the 16-bit * values shuffled: * * |x1 y1|x2 y2|x3 y3|x4 y4|x5 y5|x6 y6|x7 y7|x8 y8| * |z1 |z2 |z3 |z4 |z5 |z6 |z7 |z8 | * * For unshuffle, the example would be the opposite, a 64-bit type source * and a 32-bit destination. A 2 component .xy 64-bit vector on SIMD8 * would be: * * | x1l x1h | x2l x2h | x3l x3h | x4l x4h | * | x5l x5h | x6l x6h | x7l x7h | x8l x8h | * | y1l y1h | y2l y2h | y3l y3h | y4l y4h | * | y5l y5h | y6l y6h | y7l y7h | y8l y8h | * * The returned result would be the following 4 32-bit components unshuffled: * * | x1l | x2l | x3l | x4l | x5l | x6l | x7l | x8l | * | x1h | x2h | x3h | x4h | x5h | x6h | x7h | x8h | * | y1l | y2l | y3l | y4l | y5l | y6l | y7l | y8l | * | y1h | y2h | y3h | y4h | y5h | y6h | y7h | y8h | * * - Source and destination register must not be overlapped. * - components units are measured in terms of the smaller type between * source and destination because we are un/shuffling the smaller * components from/into the bigger ones. * - first_component parameter allows skipping source components. */ void shuffle_src_to_dst(const fs_builder &bld, const fs_reg &dst, const fs_reg &src, uint32_t first_component, uint32_t components) { if (type_sz(src.type) == type_sz(dst.type)) { assert(!regions_overlap(dst, type_sz(dst.type) * bld.dispatch_width() * components, offset(src, bld, first_component), type_sz(src.type) * bld.dispatch_width() * components)); for (unsigned i = 0; i < components; i++) { bld.MOV(retype(offset(dst, bld, i), src.type), offset(src, bld, i + first_component)); } } else if (type_sz(src.type) < type_sz(dst.type)) { /* Source is shuffled into destination */ unsigned size_ratio = type_sz(dst.type) / type_sz(src.type); assert(!regions_overlap(dst, type_sz(dst.type) * bld.dispatch_width() * DIV_ROUND_UP(components, size_ratio), offset(src, bld, first_component), type_sz(src.type) * bld.dispatch_width() * components)); brw_reg_type shuffle_type = brw_reg_type_from_bit_size(8 * type_sz(src.type), BRW_REGISTER_TYPE_D); for (unsigned i = 0; i < components; i++) { fs_reg shuffle_component_i = subscript(offset(dst, bld, i / size_ratio), shuffle_type, i % size_ratio); bld.MOV(shuffle_component_i, retype(offset(src, bld, i + first_component), shuffle_type)); } } else { /* Source is unshuffled into destination */ unsigned size_ratio = type_sz(src.type) / type_sz(dst.type); assert(!regions_overlap(dst, type_sz(dst.type) * bld.dispatch_width() * components, offset(src, bld, first_component / size_ratio), type_sz(src.type) * bld.dispatch_width() * DIV_ROUND_UP(components + (first_component % size_ratio), size_ratio))); brw_reg_type shuffle_type = brw_reg_type_from_bit_size(8 * type_sz(dst.type), BRW_REGISTER_TYPE_D); for (unsigned i = 0; i < components; i++) { fs_reg shuffle_component_i = subscript(offset(src, bld, (first_component + i) / size_ratio), shuffle_type, (first_component + i) % size_ratio); bld.MOV(retype(offset(dst, bld, i), shuffle_type), shuffle_component_i); } } } void shuffle_from_32bit_read(const fs_builder &bld, const fs_reg &dst, const fs_reg &src, uint32_t first_component, uint32_t components) { assert(type_sz(src.type) == 4); /* This function takes components in units of the destination type while * shuffle_src_to_dst takes components in units of the smallest type */ if (type_sz(dst.type) > 4) { assert(type_sz(dst.type) == 8); first_component *= 2; components *= 2; } shuffle_src_to_dst(bld, dst, src, first_component, components); } fs_reg shuffle_for_32bit_write(const fs_builder &bld, const fs_reg &src, uint32_t first_component, uint32_t components) { fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_D, DIV_ROUND_UP (components * type_sz(src.type), 4)); /* This function takes components in units of the source type while * shuffle_src_to_dst takes components in units of the smallest type */ if (type_sz(src.type) > 4) { assert(type_sz(src.type) == 8); first_component *= 2; components *= 2; } shuffle_src_to_dst(bld, dst, src, first_component, components); return dst; } fs_reg setup_imm_df(const fs_builder &bld, double v) { const struct gen_device_info *devinfo = bld.shader->devinfo; assert(devinfo->gen >= 7); if (devinfo->gen >= 8) return brw_imm_df(v); /* gen7.5 does not support DF immediates straighforward but the DIM * instruction allows to set the 64-bit immediate value. */ if (devinfo->is_haswell) { const fs_builder ubld = bld.exec_all().group(1, 0); fs_reg dst = ubld.vgrf(BRW_REGISTER_TYPE_DF, 1); ubld.DIM(dst, brw_imm_df(v)); return component(dst, 0); } /* gen7 does not support DF immediates, so we generate a 64-bit constant by * writing the low 32-bit of the constant to suboffset 0 of a VGRF and * the high 32-bit to suboffset 4 and then applying a stride of 0. * * Alternatively, we could also produce a normal VGRF (without stride 0) * by writing to all the channels in the VGRF, however, that would hit the * gen7 bug where we have to split writes that span more than 1 register * into instructions with a width of 4 (otherwise the write to the second * register written runs into an execmask hardware bug) which isn't very * nice. */ union { double d; struct { uint32_t i1; uint32_t i2; }; } di; di.d = v; const fs_builder ubld = bld.exec_all().group(1, 0); const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2); ubld.MOV(tmp, brw_imm_ud(di.i1)); ubld.MOV(horiz_offset(tmp, 1), brw_imm_ud(di.i2)); return component(retype(tmp, BRW_REGISTER_TYPE_DF), 0); } fs_reg setup_imm_b(const fs_builder &bld, int8_t v) { const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_B); bld.MOV(tmp, brw_imm_w(v)); return tmp; } fs_reg setup_imm_ub(const fs_builder &bld, uint8_t v) { const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UB); bld.MOV(tmp, brw_imm_uw(v)); return tmp; } @ 1.1.1.1 log @from maya: Import mesa 18.3.4. Mesa 18.3.4 implements the OpenGL 4.5 API. Some drivers don't support all the features required in OpenGL 4.5. @ text @@ 1.1.1.2 log @initial import of mesa-18.3.6 @ text @a513 9 /* If either opcode has source modifiers, bail. * * TODO: We can potentially handle source modifiers if both of the opcodes * we're combining are signed integers. */ if (instr->src[0].abs || instr->src[0].negate || src0->src[0].abs || src0->src[0].negate) return false; d1518 1 a1518 1 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8); a1524 7 } else if (byte->u32[0] & 1) { /* Extract the high byte from the word containing the desired byte * offset. */ bld.SHR(result, subscript(op[0], BRW_REGISTER_TYPE_UW, byte->u32[0] / 2), brw_imm_uw(8)); d1527 1 a1527 3 bld.AND(result, subscript(op[0], BRW_REGISTER_TYPE_UW, byte->u32[0] / 2), brw_imm_uw(0xff)); @ 1.1.1.3 log @Import mesa 19.1.7 New features in mesa 19.1.0: GL_ARB_parallel_shader_compile on all drivers. GL_EXT_gpu_shader4 on all GL 3.1 drivers. GL_EXT_shader_image_load_formatted on radeonsi. GL_EXT_texture_buffer_object on all GL 3.1 drivers. GL_EXT_texture_compression_s3tc_srgb on Gallium drivers and i965 (ES extension). GL_NV_compute_shader_derivatives on iris and i965. GL_KHR_parallel_shader_compile on all drivers. VK_EXT_buffer_device_address on Intel and RADV. VK_EXT_depth_clip_enable on Intel and RADV. VK_KHR_ycbcr_image_arrays on Intel. VK_EXT_inline_uniform_block on Intel and RADV. VK_EXT_external_memory_host on Intel. VK_EXT_host_query_reset on Intel and RADV. VK_KHR_surface_protected_capabilities on Intel and RADV. VK_EXT_pipeline_creation_feedback on Intel and RADV. VK_KHR_8bit_storage on RADV. VK_AMD_gpu_shader_int16 on RADV. VK_AMD_gpu_shader_half_float on RADV. VK_NV_compute_shader_derivatives on Intel. VK_KHR_shader_float16_int8 on Intel and RADV (RADV only supports int8). VK_KHR_shader_atomic_int64 on Intel. VK_EXT_descriptor_indexing on Intel. VK_KHR_shader_float16_int8 on Intel and RADV. GL_INTEL_conservative_rasterization on iris. VK_EXT_memory_budget on Intel. New features in mesa 19.0.0: GL_AMD_texture_texture4 on all GL 4.0 drivers. GL_EXT_shader_implicit_conversions on all drivers (ES extension). GL_EXT_texture_compression_bptc on all GL 4.0 drivers (ES extension). GL_EXT_texture_compression_rgtc on all GL 3.0 drivers (ES extension). GL_EXT_render_snorm on gallium drivers (ES extension). GL_EXT_texture_view on drivers supporting texture views (ES extension). GL_OES_texture_view on drivers supporting texture views (ES extension). GL_NV_shader_atomic_float on nvc0 (Fermi/Kepler only). Shader-based software implementations of GL_ARB_gpu_shader_fp64, GL_ARB_gpu_shader_int64, GL_ARB_vertex_attrib_64bit, and GL_ARB_shader_ballot on i965. VK_ANDROID_external_memory_android_hardware_buffer on Intel Fixed and re-exposed VK_EXT_pci_bus_info on Intel and RADV VK_EXT_scalar_block_layout on Intel and RADV VK_KHR_depth_stencil_resolve on Intel VK_KHR_draw_indirect_count on Intel VK_EXT_conditional_rendering on Intel VK_EXT_memory_budget on RADV Also, bug fixes. @ text @d26 1 a27 1 #include "nir_search_helpers.h" a28 1 #include "util/bitscan.h" d31 1 d62 1 a62 1 : type_size_vec4(var->type, true); d347 1 a347 1 const brw_reg_type reg_type = reg->bit_size == 8 ? BRW_REGISTER_TYPE_B : a384 18 bool invert; fs_reg cond_reg; /* If the condition has the form !other_condition, use other_condition as * the source, but invert the predicate on the if instruction. */ nir_alu_instr *cond = nir_src_as_alu_instr(if_stmt->condition); if (cond != NULL && cond->op == nir_op_inot) { assert(!cond->src[0].negate); assert(!cond->src[0].abs); invert = true; cond_reg = get_nir_src(cond->src[0].src); } else { invert = false; cond_reg = get_nir_src(if_stmt->condition); } d387 2 a388 1 retype(cond_reg, BRW_REGISTER_TYPE_D)); d391 1 a391 1 bld.IF(BRW_PREDICATE_NORMAL)->predicate_inverse = invert; d395 4 a398 4 if (!nir_cf_list_is_empty_block(&if_stmt->else_list)) { bld.emit(BRW_OPCODE_ELSE); nir_emit_cf_list(&if_stmt->else_list); } d440 1 a440 1 unreachable("All derefs should've been lowered"); d523 2 a524 1 unsigned element = nir_src_as_uint(src0->src[1].src); d538 1 a538 1 bld.MOV(result, subscript(op0, type, element))); d546 2 a547 2 nir_intrinsic_instr *src0 = nir_src_as_intrinsic(instr->src[0].src); if (src0 == NULL || src0->intrinsic != nir_intrinsic_load_front_face) d550 4 a553 2 if (!nir_src_is_const(instr->src[1].src) || !nir_src_is_const(instr->src[2].src)) d556 2 a557 3 const float value1 = nir_src_as_float(instr->src[1].src); const float value2 = nir_src_as_float(instr->src[2].src); if (fabsf(value1) != 1.0f || fabsf(value2) != 1.0f) d560 3 a562 2 /* nir_opt_algebraic should have gotten rid of bcsel(b, a, a) */ assert(value1 == -value2); d581 1 a581 1 if (value1 == -1.0f) { d602 1 a602 1 if (value1 == -1.0f) { d672 2 a673 5 fs_reg fs_visitor::prepare_alu_destination_and_sources(const fs_builder &bld, nir_alu_instr *instr, fs_reg *op, bool need_dest) d675 2 a676 2 fs_reg result = need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud(); d678 1 d683 1 d693 3 a695 4 /* Move and vecN instrutions may still be vectored. Return the raw, * vectored source and destination so that fs_visitor::nir_emit_alu can * handle it. Other callers should not have to handle these kinds of * instructions. a701 280 case nir_op_vec4: return result; default: break; } /* At this point, we have dealt with any instruction that operates on * more than a single channel. Therefore, we can just adjust the source * and destination registers for that channel and emit the instruction. */ unsigned channel = 0; if (nir_op_infos[instr->op].output_size == 0) { /* Since NIR is doing the scalarizing for us, we should only ever see * vectorized operations with a single channel. */ assert(util_bitcount(instr->dest.write_mask) == 1); channel = ffs(instr->dest.write_mask) - 1; result = offset(result, bld, channel); } for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { assert(nir_op_infos[instr->op].input_sizes[i] < 2); op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]); } return result; } void fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr, fs_reg *op) { for (unsigned i = 0; i < 2; i++) { nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[i].src); if (inot_instr != NULL && inot_instr->op == nir_op_inot && !inot_instr->src[0].abs && !inot_instr->src[0].negate) { /* The source of the inot is now the source of instr. */ prepare_alu_destination_and_sources(bld, inot_instr, &op[i], false); assert(!op[i].negate); op[i].negate = true; } else { op[i] = resolve_source_modifiers(op[i]); } } } bool fs_visitor::try_emit_b2fi_of_inot(const fs_builder &bld, fs_reg result, nir_alu_instr *instr) { if (devinfo->gen < 6 || devinfo->gen >= 12) return false; nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[0].src); if (inot_instr == NULL || inot_instr->op != nir_op_inot) return false; /* HF is also possible as a destination on BDW+. For nir_op_b2i, the set * of valid size-changing combinations is a bit more complex. * * The source restriction is just because I was lazy about generating the * constant below. */ if (nir_dest_bit_size(instr->dest.dest) != 32 || nir_src_bit_size(inot_instr->src[0].src) != 32) return false; /* b2[fi](inot(a)) maps a=0 => 1, a=-1 => 0. Since a can only be 0 or -1, * this is float(1 + a). */ fs_reg op; prepare_alu_destination_and_sources(bld, inot_instr, &op, false); /* Ignore the saturate modifier, if there is one. The result of the * arithmetic can only be 0 or 1, so the clamping will do nothing anyway. */ bld.ADD(result, op, brw_imm_d(1)); return true; } /** * Emit code for nir_op_fsign possibly fused with a nir_op_fmul * * If \c instr is not the \c nir_op_fsign, then \c fsign_src is the index of * the source of \c instr that is a \c nir_op_fsign. */ void fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr, fs_reg result, fs_reg *op, unsigned fsign_src) { fs_inst *inst; assert(instr->op == nir_op_fsign || instr->op == nir_op_fmul); assert(fsign_src < nir_op_infos[instr->op].num_inputs); if (instr->op != nir_op_fsign) { const nir_alu_instr *const fsign_instr = nir_src_as_alu_instr(instr->src[fsign_src].src); assert(!fsign_instr->dest.saturate); /* op[fsign_src] has the nominal result of the fsign, and op[1 - * fsign_src] has the other multiply source. This must be rearranged so * that op[0] is the source of the fsign op[1] is the other multiply * source. */ if (fsign_src != 0) op[1] = op[0]; op[0] = get_nir_src(fsign_instr->src[0].src); const nir_alu_type t = (nir_alu_type)(nir_op_infos[instr->op].input_types[0] | nir_src_bit_size(fsign_instr->src[0].src)); op[0].type = brw_type_for_nir_type(devinfo, t); op[0].abs = fsign_instr->src[0].abs; op[0].negate = fsign_instr->src[0].negate; unsigned channel = 0; if (nir_op_infos[instr->op].output_size == 0) { /* Since NIR is doing the scalarizing for us, we should only ever see * vectorized operations with a single channel. */ assert(util_bitcount(instr->dest.write_mask) == 1); channel = ffs(instr->dest.write_mask) - 1; } op[0] = offset(op[0], bld, fsign_instr->src[0].swizzle[channel]); } else { assert(!instr->dest.saturate); } if (op[0].abs) { /* Straightforward since the source can be assumed to be either strictly * >= 0 or strictly <= 0 depending on the setting of the negate flag. */ set_condmod(BRW_CONDITIONAL_NZ, bld.MOV(result, op[0])); if (instr->op == nir_op_fsign) { inst = (op[0].negate) ? bld.MOV(result, brw_imm_f(-1.0f)) : bld.MOV(result, brw_imm_f(1.0f)); } else { op[1].negate = (op[0].negate != op[1].negate); inst = bld.MOV(result, op[1]); } set_predicate(BRW_PREDICATE_NORMAL, inst); } else if (type_sz(op[0].type) == 2) { /* AND(val, 0x8000) gives the sign bit. * * Predicated OR ORs 1.0 (0x3c00) with the sign bit if val is not zero. */ fs_reg zero = retype(brw_imm_uw(0), BRW_REGISTER_TYPE_HF); bld.CMP(bld.null_reg_f(), op[0], zero, BRW_CONDITIONAL_NZ); op[0].type = BRW_REGISTER_TYPE_UW; result.type = BRW_REGISTER_TYPE_UW; bld.AND(result, op[0], brw_imm_uw(0x8000u)); if (instr->op == nir_op_fsign) inst = bld.OR(result, result, brw_imm_uw(0x3c00u)); else { /* Use XOR here to get the result sign correct. */ inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UW)); } inst->predicate = BRW_PREDICATE_NORMAL; } else if (type_sz(op[0].type) == 4) { /* AND(val, 0x80000000) gives the sign bit. * * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not * zero. */ bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ); op[0].type = BRW_REGISTER_TYPE_UD; result.type = BRW_REGISTER_TYPE_UD; bld.AND(result, op[0], brw_imm_ud(0x80000000u)); if (instr->op == nir_op_fsign) inst = bld.OR(result, result, brw_imm_ud(0x3f800000u)); else { /* Use XOR here to get the result sign correct. */ inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UD)); } inst->predicate = BRW_PREDICATE_NORMAL; } else { /* For doubles we do the same but we need to consider: * * - 2-src instructions can't operate with 64-bit immediates * - The sign is encoded in the high 32-bit of each DF * - We need to produce a DF result. */ fs_reg zero = vgrf(glsl_type::double_type); bld.MOV(zero, setup_imm_df(bld, 0.0)); bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ); bld.MOV(result, zero); fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1); bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1), brw_imm_ud(0x80000000u)); if (instr->op == nir_op_fsign) { set_predicate(BRW_PREDICATE_NORMAL, bld.OR(r, r, brw_imm_ud(0x3ff00000u))); } else { /* This could be done better in some cases. If the scale is an * immediate with the low 32-bits all 0, emitting a separate XOR and * OR would allow an algebraic optimization to remove the OR. There * are currently zero instances of fsign(double(x))*IMM in shader-db * or any test suite, so it is hard to care at this time. */ fs_reg result_int64 = retype(result, BRW_REGISTER_TYPE_UQ); inst = bld.XOR(result_int64, result_int64, retype(op[1], BRW_REGISTER_TYPE_UQ)); } } } /** * Deteremine whether sources of a nir_op_fmul can be fused with a nir_op_fsign * * Checks the operands of a \c nir_op_fmul to determine whether or not * \c emit_fsign could fuse the multiplication with the \c sign() calculation. * * \param instr The multiplication instruction * * \param fsign_src The source of \c instr that may or may not be a * \c nir_op_fsign */ static bool can_fuse_fmul_fsign(nir_alu_instr *instr, unsigned fsign_src) { assert(instr->op == nir_op_fmul); nir_alu_instr *const fsign_instr = nir_src_as_alu_instr(instr->src[fsign_src].src); /* Rules: * * 1. instr->src[fsign_src] must be a nir_op_fsign. * 2. The nir_op_fsign can only be used by this multiplication. * 3. The source that is the nir_op_fsign does not have source modifiers. * \c emit_fsign only examines the source modifiers of the source of the * \c nir_op_fsign. * * The nir_op_fsign must also not have the saturate modifier, but steps * have already been taken (in nir_opt_algebraic) to ensure that. */ return fsign_instr != NULL && fsign_instr->op == nir_op_fsign && is_used_once(fsign_instr) && !instr->src[fsign_src].abs && !instr->src[fsign_src].negate; } void fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr) { struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key; fs_inst *inst; fs_reg op[4]; fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, true); switch (instr->op) { case nir_op_imov: case nir_op_fmov: case nir_op_vec2: case nir_op_vec3: d742 3 d746 21 d780 1 a780 1 case nir_op_f2f16: d790 2 a791 1 assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */ d796 2 a797 9 case nir_op_b2i8: case nir_op_b2i16: case nir_op_b2i32: case nir_op_b2i64: case nir_op_b2f16: case nir_op_b2f32: case nir_op_b2f64: if (try_emit_b2fi_of_inot(bld, result, instr)) break; d801 3 d808 24 a831 5 case nir_op_f2f64: case nir_op_f2i64: case nir_op_f2u64: case nir_op_i2i32: case nir_op_u2u32: d835 6 a841 1 case nir_op_i2i16: a842 3 case nir_op_u2u16: case nir_op_f2i16: case nir_op_f2u16: a844 12 case nir_op_f2i8: case nir_op_f2u8: if (result.type == BRW_REGISTER_TYPE_B || result.type == BRW_REGISTER_TYPE_UB || result.type == BRW_REGISTER_TYPE_HF) assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */ if (op[0].type == BRW_REGISTER_TYPE_B || op[0].type == BRW_REGISTER_TYPE_UB || op[0].type == BRW_REGISTER_TYPE_HF) assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */ d849 50 a898 2 case nir_op_fsign: emit_fsign(bld, instr, result, op, 0); d900 20 a984 5 case nir_op_uadd_sat: inst = bld.ADD(result, op[0], op[1]); inst->saturate = true; break; a985 7 for (unsigned i = 0; i < 2; i++) { if (can_fuse_fmul_fsign(instr, i)) { emit_fsign(bld, instr, result, op, i); return; } } a989 5 case nir_op_imul_2x32_64: case nir_op_umul_2x32_64: bld.MUL(result, op[0], op[1]); break; d1054 4 a1057 4 case nir_op_flt32: case nir_op_fge32: case nir_op_feq32: case nir_op_fne32: { d1066 1 a1066 1 case nir_op_flt32: d1069 1 a1069 1 case nir_op_fge32: d1072 1 a1072 1 case nir_op_feq32: d1075 1 a1075 1 case nir_op_fne32: d1098 6 a1103 6 case nir_op_ilt32: case nir_op_ult32: case nir_op_ige32: case nir_op_uge32: case nir_op_ieq32: case nir_op_ine32: { a1105 7 /* On Gen11 we have an additional issue being that src1 cannot be a byte * type. So we convert both operands for the comparison. */ fs_reg temp_op[2]; temp_op[0] = bld.fix_byte_src(op[0]); temp_op[1] = bld.fix_byte_src(op[1]); d1108 1 a1108 1 dest = bld.vgrf(temp_op[0].type, 1); d1112 2 a1113 2 case nir_op_ilt32: case nir_op_ult32: d1116 2 a1117 2 case nir_op_ige32: case nir_op_uge32: d1120 1 a1120 1 case nir_op_ieq32: d1123 1 a1123 1 case nir_op_ine32: d1129 1 a1129 1 bld.CMP(dest, temp_op[0], temp_op[1], cond); a1146 58 nir_alu_instr *inot_src_instr = nir_src_as_alu_instr(instr->src[0].src); if (inot_src_instr != NULL && (inot_src_instr->op == nir_op_ior || inot_src_instr->op == nir_op_ixor || inot_src_instr->op == nir_op_iand) && !inot_src_instr->src[0].abs && !inot_src_instr->src[0].negate && !inot_src_instr->src[1].abs && !inot_src_instr->src[1].negate) { /* The sources of the source logical instruction are now the * sources of the instruction that will be generated. */ prepare_alu_destination_and_sources(bld, inot_src_instr, op, false); resolve_inot_sources(bld, inot_src_instr, op); /* Smash all of the sources and destination to be signed. This * doesn't matter for the operation of the instruction, but cmod * propagation fails on unsigned sources with negation (due to * fs_inst::can_do_cmod returning false). */ result.type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_type_int | nir_dest_bit_size(instr->dest.dest))); op[0].type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_type_int | nir_src_bit_size(inot_src_instr->src[0].src))); op[1].type = brw_type_for_nir_type(devinfo, (nir_alu_type)(nir_type_int | nir_src_bit_size(inot_src_instr->src[1].src))); /* For XOR, only invert one of the sources. Arbitrarily choose * the first source. */ op[0].negate = !op[0].negate; if (inot_src_instr->op != nir_op_ixor) op[1].negate = !op[1].negate; switch (inot_src_instr->op) { case nir_op_ior: bld.AND(result, op[0], op[1]); return; case nir_op_iand: bld.OR(result, op[0], op[1]); return; case nir_op_ixor: bld.XOR(result, op[0], op[1]); return; default: unreachable("impossible opcode"); } } d1153 2 a1154 1 resolve_inot_sources(bld, instr, op); d1160 2 a1161 1 resolve_inot_sources(bld, instr, op); d1167 2 a1168 1 resolve_inot_sources(bld, instr, op); d1176 12 a1187 12 case nir_op_b32all_fequal2: case nir_op_b32all_iequal2: case nir_op_b32all_fequal3: case nir_op_b32all_iequal3: case nir_op_b32all_fequal4: case nir_op_b32all_iequal4: case nir_op_b32any_fnequal2: case nir_op_b32any_inequal2: case nir_op_b32any_fnequal3: case nir_op_b32any_inequal3: case nir_op_b32any_fnequal4: case nir_op_b32any_inequal4: d1221 2 a1222 2 case nir_op_i2b32: case nir_op_f2b32: { d1229 1 a1229 1 if (instr->op == nir_op_f2b32) { d1248 1 a1248 1 zero = instr->op == nir_op_f2b32 ? brw_imm_f(0.0f) : brw_imm_d(0); d1251 1 a1251 1 zero = instr->op == nir_op_f2b32 ? d1341 1 a1341 2 inst = bld.emit(BRW_OPCODE_F16TO32, result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0)); d1345 1 a1345 2 inst = bld.emit(BRW_OPCODE_F16TO32, result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1)); a1460 2 bld.SHL(result, op[0], op[1]); break; d1462 27 a1488 4 bld.ASR(result, op[0], op[1]); break; case nir_op_ushr: bld.SHR(result, op[0], op[1]); d1490 1 d1506 1 a1506 1 case nir_op_b32csel: d1517 2 a1518 1 unsigned byte = nir_src_as_uint(instr->src[1].src); d1532 1 a1532 1 bld.MOV(w_temp, subscript(op[0], type, byte)); d1534 1 a1534 1 } else if (byte & 1) { d1539 1 a1539 1 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2), d1544 1 a1544 1 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2), d1549 1 a1549 1 bld.MOV(result, subscript(op[0], type, byte)); d1557 3 a1559 2 unsigned word = nir_src_as_uint(instr->src[1].src); bld.MOV(result, subscript(op[0], type, word)); d1590 1 a1590 1 bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8)); d1595 1 a1595 1 bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16)); d1600 1 a1600 1 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32)); d1609 1 a1609 1 setup_imm_df(bld, instr->value[i].f64)); d1613 1 a1613 1 bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64)); d1670 1 d1672 1 a1672 2 return nir_src_is_const(src) ? fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(src); d1893 1 a1893 1 unsigned mlen = 2; d1901 1 a1901 1 unsigned i = 0; d2070 3 d2079 4 a2082 4 nir_src_is_const(offset_src) && nir_src_is_const(vertex_src) && 4 * (base_offset + nir_src_as_uint(offset_src)) < push_reg_count) { int imm_offset = (base_offset + nir_src_as_uint(offset_src)) * 4 + nir_src_as_uint(vertex_src) * push_reg_count; d2097 1 a2097 1 if (nir_src_is_const(vertex_src)) { d2100 1 a2100 1 retype(brw_vec8_grf(first_icp_handle + nir_src_as_uint(vertex_src), 0), d2141 2 a2142 3 if (nir_src_is_const(vertex_src)) { unsigned vertex = nir_src_as_uint(vertex_src); assert(devinfo->gen >= 9 || vertex <= 5); d2144 3 a2146 1 retype(brw_vec1_grf(first_icp_handle + vertex / 8, vertex % 8), d2190 1 a2190 1 if (nir_src_is_const(offset_src)) { d2208 1 a2208 1 inst->offset = base_offset + nir_src_as_uint(offset_src); d2246 1 a2246 1 if(nir_src_is_const(offset_src)) { d2261 1 d2263 1 a2263 1 if (nir_src_is_const(*offset_src)) { d2268 1 a2268 1 assert(nir_src_as_uint(*offset_src) == 0); d2275 101 d2396 3 a2398 1 src = offset(src, bld, nir_src_as_uint(instr->src[0])); d2461 3 a2463 4 if (devinfo->gen < 11) { /* Copy "Barrier ID" from r0.2, bits 16:13 */ chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(INTEL_MASK(16, 13))); d2465 2 a2466 6 /* Shift it up to bits 27:24. */ chanbld.SHL(m0_2, m0_2, brw_imm_ud(11)); } else { chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(INTEL_MASK(30, 24))); } d2469 2 a2470 7 if (devinfo->gen < 11) { chanbld.OR(m0_2, m0_2, brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15))); } else { chanbld.OR(m0_2, m0_2, brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15))); } d2485 1 d2491 1 a2491 1 if (nir_src_is_const(vertex_src)) { a2493 1 unsigned vertex = nir_src_as_uint(vertex_src); d2495 2 a2496 1 retype(brw_vec1_grf(1 + (vertex >> 3), vertex & 7), d2499 3 a2501 2 nir_src_as_intrinsic(vertex_src) != NULL && nir_src_as_intrinsic(vertex_src)->intrinsic == nir_intrinsic_load_invocation_id) { d3057 4 d3078 1 a3078 1 emit_mcs_fetch(coords, 3, brw_imm_ud(surface), fs_reg()) : fs_reg(); d3090 5 a3094 9 fs_reg srcs[TEX_LOGICAL_NUM_SRCS]; srcs[TEX_LOGICAL_SRC_COORDINATE] = coords; srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_ud(0); srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = sample; srcs[TEX_LOGICAL_SRC_MCS] = mcs; srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface); srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(0); srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_ud(3); srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_ud(0); d3210 2 a3211 1 const unsigned store_offset = nir_src_as_uint(instr->src[1]); d3213 1 a3213 1 SET_FIELD(store_offset, BRW_NIR_FRAG_OUTPUT_LOCATION); d3228 3 a3230 2 const unsigned load_offset = nir_src_as_uint(instr->src[0]); const unsigned target = l - FRAG_RESULT_DATA0 + load_offset; d3319 4 a3322 2 if (nir_src_is_const(instr->src[0])) { unsigned msg_data = nir_src_as_uint(instr->src[0]) << 4; d3391 2 a3392 3 assert(nir_src_bit_size(instr->src[0]) == 32); unsigned off_x = MIN2((int)(const_offset[0].f32 * 16), 7) & 0xf; unsigned off_y = MIN2((int)(const_offset[1].f32 * 16), 7) & 0xf; d3495 4 a3498 3 if (nir_src_is_const(instr->src[src])) { int64_t add_val = nir_src_as_int(instr->src[src]); if (add_val == 1) d3500 1 a3500 1 else if (add_val == -1) d3545 2 a3546 4 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); /* num components */ d3550 8 a3557 3 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(i << 2); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, offset(dest, bld, i), srcs, SURFACE_LOGICAL_NUM_SRCS); a3603 1 assert(stage == MESA_SHADER_COMPUTE); d3605 1 a3605 5 const unsigned bit_size = nir_dest_bit_size(instr->dest); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[0]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); d3607 11 a3617 2 /* Make dest unsigned because that's what the temporary will be */ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); d3620 2 a3621 17 if (nir_intrinsic_align(instr) >= 4) { assert(nir_dest_bit_size(instr->dest) == 32); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); fs_inst *inst = bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); inst->size_written = instr->num_components * dispatch_width * 4; } else { assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_dest_num_components(instr->dest) == 1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS); bld.MOV(dest, read_result); } a3626 1 assert(stage == MESA_SHADER_COMPUTE); d3628 51 a3678 22 const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); fs_reg data = get_nir_src(instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(nir_intrinsic_write_mask(instr) == (1u << instr->num_components) - 1); if (nir_intrinsic_align(instr) >= 4) { assert(nir_src_bit_size(instr->src[0]) == 32); assert(nir_src_num_components(instr->src[0]) <= 4); srcs[SURFACE_LOGICAL_SRC_DATA] = data; srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); } else { assert(nir_src_bit_size(instr->src[0]) <= 32); assert(nir_src_num_components(instr->src[0]) == 1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); d3680 4 a3683 2 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data); d3685 4 a3688 2 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); d3690 1 d3708 1 a3708 1 return retype(brw_imm_uw(value.u16), type); d3710 1 a3710 1 return retype(brw_imm_ud(value.u32), type); d3713 1 a3713 1 return setup_imm_df(bld, value.f64); d3715 1 a3715 1 return retype(brw_imm_u64(value.u64), type); a3782 21 fs_reg fs_visitor::get_nir_ssbo_intrinsic_index(const brw::fs_builder &bld, nir_intrinsic_instr *instr) { /* SSBO stores are weird in that their index is in src[1] */ const unsigned src = instr->intrinsic == nir_intrinsic_store_ssbo ? 1 : 0; fs_reg surf_index; if (nir_src_is_const(instr->src[src])) { unsigned index = stage_prog_data->binding_table.ssbo_start + nir_src_as_uint(instr->src[src]); surf_index = brw_imm_ud(index); } else { surf_index = vgrf(glsl_type::uint_type); bld.ADD(surf_index, get_nir_src(instr->src[src]), brw_imm_ud(stage_prog_data->binding_table.ssbo_start)); } return bld.emit_uniformize(surf_index); } d3821 1 a3821 11 case nir_intrinsic_image_atomic_comp_swap: case nir_intrinsic_bindless_image_load: case nir_intrinsic_bindless_image_store: case nir_intrinsic_bindless_image_atomic_add: case nir_intrinsic_bindless_image_atomic_min: case nir_intrinsic_bindless_image_atomic_max: case nir_intrinsic_bindless_image_atomic_and: case nir_intrinsic_bindless_image_atomic_or: case nir_intrinsic_bindless_image_atomic_xor: case nir_intrinsic_bindless_image_atomic_exchange: case nir_intrinsic_bindless_image_atomic_comp_swap: { d3828 1 d3830 1 d3832 5 a3836 27 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; switch (instr->intrinsic) { case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_atomic_add: case nir_intrinsic_image_atomic_min: case nir_intrinsic_image_atomic_max: case nir_intrinsic_image_atomic_and: case nir_intrinsic_image_atomic_or: case nir_intrinsic_image_atomic_xor: case nir_intrinsic_image_atomic_exchange: case nir_intrinsic_image_atomic_comp_swap: srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_image_intrinsic_image(bld, instr); break; default: /* Bindless */ srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(get_nir_src(instr->src[0])); break; } srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(image_intrinsic_coord_components(instr)); d3839 7 a3845 13 if (instr->intrinsic == nir_intrinsic_image_load || instr->intrinsic == nir_intrinsic_bindless_image_load) { srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); fs_inst *inst = bld.emit(SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); inst->size_written = instr->num_components * dispatch_width * 4; } else if (instr->intrinsic == nir_intrinsic_image_store || instr->intrinsic == nir_intrinsic_bindless_image_store) { srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[3]); bld.emit(SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); a3851 1 case nir_intrinsic_bindless_image_atomic_add: a3859 1 case nir_intrinsic_bindless_image_atomic_min: a3863 1 case nir_intrinsic_bindless_image_atomic_max: a3867 1 case nir_intrinsic_bindless_image_atomic_and: a3870 1 case nir_intrinsic_bindless_image_atomic_or: a3873 1 case nir_intrinsic_bindless_image_atomic_xor: a3876 1 case nir_intrinsic_bindless_image_atomic_exchange: a3879 1 case nir_intrinsic_bindless_image_atomic_comp_swap: d3886 4 a3889 1 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op); d3891 2 a3892 10 fs_reg data; if (num_srcs >= 4) data = get_nir_src(instr->src[3]); if (num_srcs >= 5) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[4]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } srcs[SURFACE_LOGICAL_SRC_DATA] = data; d3894 4 a3897 2 bld.emit(SHADER_OPCODE_TYPED_ATOMIC_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); d3902 1 a3902 2 case nir_intrinsic_image_size: case nir_intrinsic_bindless_image_size: { a3904 2 * Incidentally, this means that we can handle bindless with exactly the * same code. a3909 9 fs_reg srcs[TEX_LOGICAL_NUM_SRCS]; if (instr->intrinsic == nir_intrinsic_image_size) srcs[TEX_LOGICAL_SRC_SURFACE] = image; else srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = image; srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_d(0); srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(0); srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(0); d3915 4 d3920 2 a3921 2 fs_inst *inst = ubld.emit(SHADER_OPCODE_IMAGE_SIZE_LOGICAL, tmp, srcs, ARRAY_SIZE(srcs)); d3938 11 a3948 11 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_image_intrinsic_image(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); fs_inst *inst = bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); inst->size_written = instr->num_components * dispatch_width * 4; d3953 5 a3957 2 if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; d3959 1 a3959 7 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_image_intrinsic_image(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[2]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); d3961 2 a3962 2 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); d3974 1 a3974 2 ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp, brw_vec8_grf(0, 0), brw_imm_ud(0)) d4002 3 a4004 3 if (nir_src_is_const(instr->src[0])) { unsigned load_offset = nir_src_as_uint(instr->src[0]); assert(load_offset % type_sz(dest.type) == 0); d4008 1 a4008 1 src.offset = load_offset + instr->const_index[0] % 4; d4058 1 d4060 2 a4061 1 if (nir_src_is_const(instr->src[0])) { d4063 1 a4063 1 nir_src_as_uint(instr->src[0]); d4065 1 d4075 7 d4084 2 a4085 1 if (!nir_src_is_const(instr->src[1])) { a4101 1 const unsigned load_offset = nir_src_as_uint(instr->src[1]); d4104 3 a4106 3 if (nir_src_is_const(instr->src[0])) { const unsigned ubo_block = nir_src_as_uint(instr->src[0]); const unsigned offset_256b = load_offset / 32; d4116 1 a4116 1 push_reg.offset = load_offset - 32 * range->start; d4135 1 a4135 1 const unsigned base = load_offset + c * type_size; d4157 5 a4161 2 case nir_intrinsic_load_global: { assert(devinfo->gen >= 8); d4163 23 a4185 9 if (nir_intrinsic_align(instr) >= 4) { assert(nir_dest_bit_size(instr->dest) == 32); fs_inst *inst = bld.emit(SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL, dest, get_nir_src(instr->src[0]), /* Address */ fs_reg(), /* No source data */ brw_imm_ud(instr->num_components)); inst->size_written = instr->num_components * inst->dst.component_size(inst->exec_size); d4187 1 a4187 12 const unsigned bit_size = nir_dest_bit_size(instr->dest); assert(bit_size <= 32); assert(nir_dest_num_components(instr->dest) == 1); brw_reg_type data_type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL, tmp, get_nir_src(instr->src[0]), /* Address */ fs_reg(), /* No source data */ brw_imm_ud(bit_size)); bld.MOV(retype(dest, data_type), tmp); d4189 5 d4197 2 a4198 2 case nir_intrinsic_store_global: assert(devinfo->gen >= 8); d4203 9 a4211 7 if (nir_intrinsic_align(instr) >= 4) { assert(nir_src_bit_size(instr->src[0]) == 32); bld.emit(SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL, fs_reg(), get_nir_src(instr->src[1]), /* Address */ get_nir_src(instr->src[0]), /* Data */ brw_imm_ud(instr->num_components)); d4213 7 a4219 12 const unsigned bit_size = nir_src_bit_size(instr->src[0]); assert(bit_size <= 32); assert(nir_src_num_components(instr->src[0]) == 1); brw_reg_type data_type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(tmp, retype(get_nir_src(instr->src[0]), data_type)); bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(), get_nir_src(instr->src[1]), /* Address */ tmp, /* Data */ brw_imm_ud(nir_src_bit_size(instr->src[0]))); a4220 1 break; d4222 5 a4226 39 case nir_intrinsic_global_atomic_add: nir_emit_global_atomic(bld, get_op_for_atomic_add(instr, 1), instr); break; case nir_intrinsic_global_atomic_imin: nir_emit_global_atomic(bld, BRW_AOP_IMIN, instr); break; case nir_intrinsic_global_atomic_umin: nir_emit_global_atomic(bld, BRW_AOP_UMIN, instr); break; case nir_intrinsic_global_atomic_imax: nir_emit_global_atomic(bld, BRW_AOP_IMAX, instr); break; case nir_intrinsic_global_atomic_umax: nir_emit_global_atomic(bld, BRW_AOP_UMAX, instr); break; case nir_intrinsic_global_atomic_and: nir_emit_global_atomic(bld, BRW_AOP_AND, instr); break; case nir_intrinsic_global_atomic_or: nir_emit_global_atomic(bld, BRW_AOP_OR, instr); break; case nir_intrinsic_global_atomic_xor: nir_emit_global_atomic(bld, BRW_AOP_XOR, instr); break; case nir_intrinsic_global_atomic_exchange: nir_emit_global_atomic(bld, BRW_AOP_MOV, instr); break; case nir_intrinsic_global_atomic_comp_swap: nir_emit_global_atomic(bld, BRW_AOP_CMPWR, instr); break; case nir_intrinsic_global_atomic_fmin: nir_emit_global_atomic_float(bld, BRW_AOP_FMIN, instr); break; case nir_intrinsic_global_atomic_fmax: nir_emit_global_atomic_float(bld, BRW_AOP_FMAX, instr); break; case nir_intrinsic_global_atomic_fcomp_swap: nir_emit_global_atomic_float(bld, BRW_AOP_FCMPWR, instr); break; d4228 8 a4235 2 case nir_intrinsic_load_ssbo: { assert(devinfo->gen >= 7); d4237 9 a4245 6 const unsigned bit_size = nir_dest_bit_size(instr->dest); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); d4247 1 a4247 2 /* Make dest unsigned because that's what the temporary will be */ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); d4249 24 a4272 20 /* Read the vector */ if (nir_intrinsic_align(instr) >= 4) { assert(nir_dest_bit_size(instr->dest) == 32); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); fs_inst *inst = bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); inst->size_written = instr->num_components * dispatch_width * 4; } else { assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_dest_num_components(instr->dest) == 1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS); bld.MOV(dest, read_result); } break; } d4274 25 a4298 2 case nir_intrinsic_store_ssbo: { assert(devinfo->gen >= 7); d4300 1 a4300 2 if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; d4302 9 a4310 23 const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[2]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); fs_reg data = get_nir_src(instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(nir_intrinsic_write_mask(instr) == (1u << instr->num_components) - 1); if (nir_intrinsic_align(instr) >= 4) { assert(nir_src_bit_size(instr->src[0]) == 32); assert(nir_src_num_components(instr->src[0]) <= 4); srcs[SURFACE_LOGICAL_SRC_DATA] = data; srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); } else { assert(nir_src_bit_size(instr->src[0]) <= 32); assert(nir_src_num_components(instr->src[0]) == 1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); d4312 23 a4334 2 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data); d4336 4 a4339 2 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); d4347 3 a4349 1 unsigned store_offset = nir_src_as_uint(instr->src[1]); d4358 1 a4358 1 4 * store_offset), src.type); d4407 2 a4408 3 assert(nir_src_num_components(instr->src[0]) == 1); unsigned ssbo_index = nir_src_is_const(instr->src[0]) ? nir_src_as_uint(instr->src[0]) : 0; d4461 2 d4547 1 a4547 2 value.type = bit_size == 8 ? BRW_REGISTER_TYPE_B : brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_F); d4644 2 a4645 1 const unsigned index = nir_src_as_uint(instr->src[1]); d4648 1 a4648 1 value, brw_imm_ud(index), brw_imm_ud(4)); d4655 1 a4655 13 if (devinfo->gen <= 7) { /* The hardware doesn't seem to support these crazy regions with * compressed instructions on gen7 and earlier so we fall back to * using quad swizzles. Fortunately, we don't support 64-bit * anything in Vulkan on gen7. */ assert(nir_src_bit_size(instr->src[0]) == 32); const fs_builder ubld = bld.exec_all(); ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value, brw_imm_ud(BRW_SWIZZLE4(1,0,3,2))); bld.MOV(retype(dest, value.type), tmp); } else { const fs_builder ubld = bld.exec_all().group(dispatch_width / 2, 0); d4657 4 a4660 4 const fs_reg src_left = horiz_stride(value, 2); const fs_reg src_right = horiz_stride(horiz_offset(value, 1), 2); const fs_reg tmp_left = horiz_stride(tmp, 2); const fs_reg tmp_right = horiz_stride(horiz_offset(tmp, 1), 2); d4662 25 a4688 1 d4829 3 a4831 2 ubld.emit(SHADER_OPCODE_INTERLOCK, tmp, brw_vec8_grf(0, 0)) ->size_written = 2 * REG_SIZE; d4836 1 a4836 14 /* For endInvocationInterlock(), we need to insert a memory fence which * stalls in the shader until the memory transactions prior to that * fence are complete. This ensures that the shader does not end before * any writes from its critical section have landed. Otherwise, you can * end up with a case where the next invocation on that pixel properly * stalls for previous FS invocation on its pixel to complete but * doesn't actually wait for the dataport memory transactions from that * thread to land before submitting its own. */ const fs_builder ubld = bld.group(8, 0); const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2); ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp, brw_vec8_grf(0, 0), brw_imm_ud(1)) ->size_written = 2 * REG_SIZE; a4851 7 /* The BTI untyped atomic messages only support 32-bit atomics. If you * just look at the big table of messages in the Vol 7 of the SKL PRM, they * appear to exist. However, if you look at Vol 2a, there are no message * descriptors provided for Qword atomic ops except for A64 messages. */ assert(nir_dest_bit_size(instr->dest) == 32); d4856 19 a4874 5 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op); d4876 2 a4877 1 fs_reg data; d4879 4 a4882 9 data = get_nir_src(instr->src[2]); if (op == BRW_AOP_CMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[3]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } srcs[SURFACE_LOGICAL_SRC_DATA] = data; d4886 7 a4892 2 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); d4906 18 a4923 12 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op); fs_reg data = get_nir_src(instr->src[2]); if (op == BRW_AOP_FCMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[3]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; d4925 6 a4930 1 srcs[SURFACE_LOGICAL_SRC_DATA] = data; d4934 7 a4940 2 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); d4951 3 a4953 6 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op); fs_reg data; d4955 4 a4958 8 data = get_nir_src(instr->src[1]); if (op == BRW_AOP_CMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[2]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } srcs[SURFACE_LOGICAL_SRC_DATA] = data; d4961 3 a4963 3 if (nir_src_is_const(instr->src[0])) { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0])); d4965 2 a4966 2 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type); bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS], d4973 7 a4979 2 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); d4990 6 a4995 13 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM); srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op); fs_reg data = get_nir_src(instr->src[1]); if (op == BRW_AOP_FCMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[2]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } srcs[SURFACE_LOGICAL_SRC_DATA] = data; d4998 3 a5000 3 if (nir_src_is_const(instr->src[0])) { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0])); d5002 4 a5005 4 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type); bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS], retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD), brw_imm_ud(instr->const_index[0])); d5010 7 a5016 62 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } void fs_visitor::nir_emit_global_atomic(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); fs_reg addr = get_nir_src(instr->src[0]); fs_reg data; if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC) data = get_nir_src(instr->src[1]); if (op == BRW_AOP_CMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[2]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } if (nir_dest_bit_size(instr->dest) == 64) { bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL, dest, addr, data, brw_imm_ud(op)); } else { assert(nir_dest_bit_size(instr->dest) == 32); bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL, dest, addr, data, brw_imm_ud(op)); } } void fs_visitor::nir_emit_global_atomic_float(const fs_builder &bld, int op, nir_intrinsic_instr *instr) { if (stage == MESA_SHADER_FRAGMENT) brw_wm_prog_data(prog_data)->has_side_effects = true; assert(nir_intrinsic_infos[instr->intrinsic].has_dest); fs_reg dest = get_nir_dest(instr->dest); fs_reg addr = get_nir_src(instr->src[0]); assert(op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC); fs_reg data = get_nir_src(instr->src[1]); if (op == BRW_AOP_FCMPWR) { fs_reg tmp = bld.vgrf(data.type, 2); fs_reg sources[2] = { data, get_nir_src(instr->src[2]) }; bld.LOAD_PAYLOAD(tmp, sources, 2, 0); data = tmp; } bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL, dest, addr, data, brw_imm_ud(op)); a5082 4 case nir_tex_src_min_lod: srcs[TEX_LOGICAL_SRC_MIN_LOD] = retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F); break; d5088 7 a5094 2 uint32_t offset_bits = 0; if (brw_texture_offset(instr, i, &offset_bits)) { d5107 9 a5130 12 case nir_tex_src_texture_handle: assert(nir_tex_instr_src_index(instr, nir_tex_src_texture_offset) == -1); srcs[TEX_LOGICAL_SRC_SURFACE] = fs_reg(); srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(src); break; case nir_tex_src_sampler_handle: assert(nir_tex_instr_src_index(instr, nir_tex_src_sampler_offset) == -1); srcs[TEX_LOGICAL_SRC_SAMPLER] = fs_reg(); srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = bld.emit_uniformize(src); break; d5137 3 a5139 1 const uint32_t plane = nir_src_as_uint(instr->src[i].src); d5162 1 a5162 2 srcs[TEX_LOGICAL_SRC_SURFACE], srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE]); d5174 2 a5175 1 opcode = SHADER_OPCODE_TEX_LOGICAL; @ 1.1.1.4 log @initial import of mesa 21.3.7 main changes since 19.1.7 include: - more support for Vulkan functions - better supported for newer radeonsi (both amdgpu and radeon backends) - various bug fixes in many drivers - many fixes and enhancements for intel drivers - some fixes for nvidia - OpenGL 4.6 for some drivers (intel, radeonsi) - intel Tigerlake and Rocketlake support - Vulkan 1.2 for some drivers - OpenGL 4.5, GLES 3.2, and more on llvmpipe - working Panfrost and Midgard drivers - fix warnings in radeonsi vs newer llvm @ text @a26 2 #include "brw_rt.h" #include "brw_eu.h" a35 2 emit_shader_float_controls_execution_mode(); a41 1 last_scratch = ALIGN(nir->scratch_size, 4) * dispatch_width; a43 2 bld.emit(SHADER_OPCODE_HALT_TARGET); d58 1 a58 1 nir_foreach_shader_out_variable(var, nir) { d77 1 a77 2 for (unsigned i = 1; i < reg_size; i++) { assert(i + loc < ARRAY_SIZE(vec4s)); a78 1 } d81 1 a81 2 for (unsigned i = 0; i < reg_size; i++) { assert(loc + i < ARRAY_SIZE(outputs)); a82 1 } d99 4 a102 3 if ((stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL) && devinfo->verx10 < 125) { /* Add uniforms for builtins after regular NIR uniforms. */ d104 1 a104 16 uint32_t *param; if (nir->info.workgroup_size_variable && compiler->lower_variable_group_size) { param = brw_stage_prog_data_add_params(prog_data, 3); for (unsigned i = 0; i < 3; i++) { param[i] = (BRW_PARAM_BUILTIN_WORK_GROUP_SIZE_X + i); group_size[i] = fs_reg(UNIFORM, uniforms++, BRW_REGISTER_TYPE_UD); } } /* Subgroup ID must be the last uniform on the list. This will make * easier later to split between cross thread and per thread * uniforms. */ param = brw_stage_prog_data_add_params(prog_data, 1); d163 1 a163 1 assert(v->devinfo->ver >= 7); d169 3 a171 4 case nir_intrinsic_load_workgroup_id: assert(v->stage == MESA_SHADER_COMPUTE || v->stage == MESA_SHADER_KERNEL); reg = &v->nir_system_values[SYSTEM_VALUE_WORKGROUP_ID]; d183 1 a183 1 /* On Gfx6+ (gl_HelperInvocation is only exposed on Gfx7+) the d209 1 a209 1 * The negate source-modifier bit of logical instructions on Gfx8+ d214 1 a214 1 if (v->devinfo->ver < 8) { a230 6 case nir_intrinsic_load_frag_shading_rate: reg = &v->nir_system_values[SYSTEM_VALUE_FRAG_SHADING_RATE]; if (reg->file == BAD_FILE) *reg = *v->emit_shading_rate_setup(); break; d270 65 d393 3 a397 1 cond_reg = offset(cond_reg, bld, cond->src[0].swizzle[0]); d419 1 a419 1 if (devinfo->ver < 7) d433 1 a433 1 if (devinfo->ver < 7) d453 1 a453 1 nir_emit_alu(abld, nir_instr_as_alu(instr), true); a477 1 case MESA_SHADER_KERNEL: a479 8 case MESA_SHADER_RAYGEN: case MESA_SHADER_ANY_HIT: case MESA_SHADER_CLOSEST_HIT: case MESA_SHADER_MISS: case MESA_SHADER_INTERSECTION: case MESA_SHADER_CALLABLE: nir_emit_bs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; d531 9 d553 2 a554 1 bld.MOV(result, subscript(op0, type, element)); d580 1 a580 17 if (devinfo->ver >= 12) { /* Bit 15 of g1.1 is 0 if the polygon is front facing. */ fs_reg g1 = fs_reg(retype(brw_vec1_grf(1, 1), BRW_REGISTER_TYPE_W)); /* For (gl_FrontFacing ? 1.0 : -1.0), emit: * * or(8) tmp.1<2>W g1.1<0,1,0>W 0x00003f80W * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D * * and negate g1.1<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0). */ if (value1 == -1.0f) g1.negate = true; bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1), g1, brw_imm_uw(0x3f80)); } else if (devinfo->ver >= 6) { a685 10 static brw_rnd_mode brw_rnd_mode_from_execution_mode(unsigned execution_mode) { if (nir_has_any_rounding_mode_rtne(execution_mode)) return BRW_RND_MODE_RTNE; if (nir_has_any_rounding_mode_rtz(execution_mode)) return BRW_RND_MODE_RTZ; return BRW_RND_MODE_UNSPECIFIED; } a698 2 assert(!instr->dest.saturate); a699 4 /* We don't lower to source modifiers so they should not exist. */ assert(!instr->src[i].abs); assert(!instr->src[i].negate); d704 2 d714 2 a715 1 case nir_op_mov: a718 2 case nir_op_vec8: case nir_op_vec16: d754 2 a755 1 if (inot_instr != NULL && inot_instr->op == nir_op_inot) { d772 1 a772 1 if (devinfo->ver < 6 || devinfo->ver >= 12) d824 2 d841 2 d854 2 d858 17 a874 1 if (type_sz(op[0].type) == 2) { d980 2 a981 1 is_used_once(fsign_instr); d985 1 a985 2 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr, bool need_dest) a988 31 unsigned execution_mode = bld.shader->nir->info.float_controls_execution_mode; fs_reg op[NIR_MAX_VEC_COMPONENTS]; fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest); #ifndef NDEBUG /* Everything except raw moves, some type conversions, iabs, and ineg * should have 8-bit sources lowered by nir_lower_bit_size in * brw_preprocess_nir or by brw_nir_lower_conversions in * brw_postprocess_nir. */ switch (instr->op) { case nir_op_mov: case nir_op_vec2: case nir_op_vec3: case nir_op_vec4: case nir_op_vec8: case nir_op_vec16: case nir_op_i2f16: case nir_op_i2f32: case nir_op_i2i16: case nir_op_i2i32: case nir_op_u2f16: case nir_op_u2f32: case nir_op_u2u16: case nir_op_u2u32: case nir_op_iabs: case nir_op_ineg: case nir_op_pack_32_4x8_split: break; d990 2 a991 6 default: for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) { assert(type_sz(op[i].type) > 1); } } #endif d994 2 a995 1 case nir_op_mov: d998 1 a998 3 case nir_op_vec4: case nir_op_vec8: case nir_op_vec16: { d1014 2 a1015 2 if (instr->op == nir_op_mov) { bld.MOV(offset(temp, bld, i), d1018 1 a1018 1 bld.MOV(offset(temp, bld, i), d1021 1 d1044 1 d1049 4 a1052 11 case nir_op_f2f16: { brw_rnd_mode rnd = BRW_RND_MODE_UNSPECIFIED; if (nir_op_f2f16 == instr->op) rnd = brw_rnd_mode_from_execution_mode(execution_mode); else rnd = brw_rnd_mode_from_nir_op(instr->op); if (BRW_RND_MODE_UNSPECIFIED != rnd) bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); d1059 2 a1060 2 * only for gfx8+, it will be better to use directly the MOV, and use * BRW_OPCODE_F32TO16 when/if we work for HF support on gfx7. d1064 1 a1065 1 } d1078 1 a1078 1 FALLTHROUGH; d1088 1 d1092 1 d1094 1 d1097 2 d1112 1 a1112 70 break; case nir_op_i2i8: case nir_op_u2u8: assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */ FALLTHROUGH; case nir_op_i2i16: case nir_op_u2u16: { /* Emit better code for u2u8(extract_u8(a, b)) and similar patterns. * Emitting the instructions one by one results in two MOV instructions * that won't be propagated. By handling both instructions here, a * single MOV is emitted. */ nir_alu_instr *extract_instr = nir_src_as_alu_instr(instr->src[0].src); if (extract_instr != NULL) { if (extract_instr->op == nir_op_extract_u8 || extract_instr->op == nir_op_extract_i8) { prepare_alu_destination_and_sources(bld, extract_instr, op, false); const unsigned byte = nir_src_as_uint(extract_instr->src[1].src); const brw_reg_type type = brw_int_type(1, extract_instr->op == nir_op_extract_i8); op[0] = subscript(op[0], type, byte); } else if (extract_instr->op == nir_op_extract_u16 || extract_instr->op == nir_op_extract_i16) { prepare_alu_destination_and_sources(bld, extract_instr, op, false); const unsigned word = nir_src_as_uint(extract_instr->src[1].src); const brw_reg_type type = brw_int_type(2, extract_instr->op == nir_op_extract_i16); op[0] = subscript(op[0], type, word); } } inst = bld.MOV(result, op[0]); break; } case nir_op_fsat: inst = bld.MOV(result, op[0]); inst->saturate = true; break; case nir_op_fneg: case nir_op_ineg: op[0].negate = true; inst = bld.MOV(result, op[0]); break; case nir_op_fabs: case nir_op_iabs: op[0].negate = false; op[0].abs = true; inst = bld.MOV(result, op[0]); break; case nir_op_f2f32: if (nir_has_any_rounding_mode_enabled(execution_mode)) { brw_rnd_mode rnd = brw_rnd_mode_from_execution_mode(execution_mode); bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); } if (op[0].type == BRW_REGISTER_TYPE_HF) assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */ inst = bld.MOV(result, op[0]); d1121 1 d1126 1 d1131 1 d1136 1 d1141 1 d1150 1 d1154 1 d1158 1 d1166 1 d1170 1 d1174 1 d1177 1 a1178 8 if (nir_has_any_rounding_mode_enabled(execution_mode)) { brw_rnd_mode rnd = brw_rnd_mode_from_execution_mode(execution_mode); bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); } FALLTHROUGH; case nir_op_iadd: d1180 1 a1182 5 case nir_op_iadd3: inst = bld.ADD3(result, op[0], op[1], op[2]); break; case nir_op_iadd_sat: a1187 33 case nir_op_isub_sat: bld.emit(SHADER_OPCODE_ISUB_SAT, result, op[0], op[1]); break; case nir_op_usub_sat: bld.emit(SHADER_OPCODE_USUB_SAT, result, op[0], op[1]); break; case nir_op_irhadd: case nir_op_urhadd: assert(nir_dest_bit_size(instr->dest.dest) < 64); inst = bld.AVG(result, op[0], op[1]); break; case nir_op_ihadd: case nir_op_uhadd: { assert(nir_dest_bit_size(instr->dest.dest) < 64); fs_reg tmp = bld.vgrf(result.type); if (devinfo->ver >= 8) { op[0] = resolve_source_modifiers(op[0]); op[1] = resolve_source_modifiers(op[1]); } /* AVG(x, y) - ((x ^ y) & 1) */ bld.XOR(tmp, op[0], op[1]); bld.AND(tmp, tmp, retype(brw_imm_ud(1), result.type)); bld.AVG(result, op[0], op[1]); inst = bld.ADD(result, result, tmp); inst->src[1].negate = true; break; } a1195 11 /* We emit the rounding mode after the previous fsign optimization since * it won't result in a MUL, but will try to negate the value by other * means. */ if (nir_has_any_rounding_mode_enabled(execution_mode)) { brw_rnd_mode rnd = brw_rnd_mode_from_execution_mode(execution_mode); bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); } d1197 1 a1204 28 case nir_op_imul_32x16: case nir_op_umul_32x16: { const bool ud = instr->op == nir_op_umul_32x16; assert(nir_dest_bit_size(instr->dest.dest) == 32); /* Before Gfx7, the order of the 32-bit source and the 16-bit source was * swapped. The extension isn't enabled on those platforms, so don't * pretend to support the differences. */ assert(devinfo->ver >= 7); if (op[1].file == IMM) op[1] = ud ? brw_imm_uw(op[1].ud) : brw_imm_w(op[1].d); else { const enum brw_reg_type word_type = ud ? BRW_REGISTER_TYPE_UW : BRW_REGISTER_TYPE_W; op[1] = subscript(op[1], word_type, 0); } const enum brw_reg_type dword_type = ud ? BRW_REGISTER_TYPE_UD : BRW_REGISTER_TYPE_D; bld.MUL(result, retype(op[0], dword_type), op[1]); break; } d1272 1 a1272 1 case nir_op_fneu32: { d1279 19 a1297 1 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op)); d1321 8 a1328 1 const uint32_t bit_size = type_sz(op[0].type) * 8; d1330 1 a1330 1 dest = bld.vgrf(op[0].type, 1); d1332 20 a1351 2 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op)); d1368 1 a1368 1 if (devinfo->ver >= 8) { d1374 5 a1378 1 inot_src_instr->op == nir_op_iand)) { d1432 1 a1432 1 if (devinfo->ver >= 8) { d1438 1 a1438 1 if (devinfo->ver >= 8) { d1444 1 a1444 1 if (devinfo->ver >= 8) { d1467 18 d1490 1 d1495 1 d1538 1 a1538 6 if (devinfo->ver < 6) { set_condmod(BRW_CONDITIONAL_R, inst); set_predicate(BRW_PREDICATE_NORMAL, bld.ADD(result, result, brw_imm_f(1.0f))); inst = bld.MOV(result, result); /* for potential saturation */ } d1547 1 d1552 1 d1556 1 d1560 1 a1560 6 if (devinfo->ver < 6) { set_condmod(BRW_CONDITIONAL_R, inst); set_predicate(BRW_PREDICATE_NORMAL, bld.ADD(result, result, brw_imm_f(1.0f))); inst = bld.MOV(result, result); /* for potential saturation */ } d1587 1 d1595 1 d1602 1 a1616 3 case nir_op_unpack_half_2x16_split_x_flush_to_zero: assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode); FALLTHROUGH; d1620 1 a1621 4 case nir_op_unpack_half_2x16_split_y_flush_to_zero: assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode); FALLTHROUGH; d1625 1 a1632 4 case nir_op_pack_32_4x8_split: bld.emit(FS_OPCODE_PACK, result, op, 4); break; d1653 1 a1671 5 case nir_op_uclz: assert(nir_dest_bit_size(instr->dest.dest) == 32); bld.LZD(retype(result, BRW_REGISTER_TYPE_UD), op[0]); break; d1675 1 a1675 1 if (devinfo->ver < 7) { d1697 1 a1697 1 if (devinfo->ver < 7) { a1738 29 /* For all shift operations: * * Gen4 - Gen7: After application of source modifiers, the low 5-bits of * src1 are used an unsigned value for the shift count. * * Gen8: As with earlier platforms, but for Q and UQ types on src0, the low * 6-bit of src1 are used. * * Gen9+: The low bits of src1 matching the size of src0 (e.g., 4-bits for * W or UW src0). * * The implication is that the following instruction will produce a * different result on Gen9+ than on previous platforms: * * shr(8) g4<1>UW g12<8,8,1>UW 0x0010UW * * where Gen9+ will shift by zero, and earlier platforms will shift by 16. * * This does not seem to be the case. Experimentally, it has been * determined that shifts of 16-bit values on Gen8 behave properly. Shifts * of 8-bit values on both Gen8 and Gen9 do not. Gen11+ lowers 8-bit * values, so those platforms were not tested. No features expose access * to 8- or 16-bit types on Gen7 or earlier, so those platforms were not * tested either. See * https://gitlab.freedesktop.org/mesa/crucible/-/merge_requests/76. * * This is part of the reason 8-bit values are lowered to 16-bit on all * platforms. */ a1748 7 case nir_op_urol: bld.ROL(result, op[0], op[1]); break; case nir_op_uror: bld.ROR(result, op[0], op[1]); break; a1752 33 case nir_op_sdot_4x8_iadd: case nir_op_sdot_4x8_iadd_sat: inst = bld.DP4A(result, retype(op[2], BRW_REGISTER_TYPE_D), retype(op[0], BRW_REGISTER_TYPE_D), retype(op[1], BRW_REGISTER_TYPE_D)); if (instr->op == nir_op_sdot_4x8_iadd_sat) inst->saturate = true; break; case nir_op_udot_4x8_uadd: case nir_op_udot_4x8_uadd_sat: inst = bld.DP4A(result, retype(op[2], BRW_REGISTER_TYPE_UD), retype(op[0], BRW_REGISTER_TYPE_UD), retype(op[1], BRW_REGISTER_TYPE_UD)); if (instr->op == nir_op_udot_4x8_uadd_sat) inst->saturate = true; break; case nir_op_sudot_4x8_iadd: case nir_op_sudot_4x8_iadd_sat: inst = bld.DP4A(result, retype(op[2], BRW_REGISTER_TYPE_D), retype(op[0], BRW_REGISTER_TYPE_D), retype(op[1], BRW_REGISTER_TYPE_UD)); if (instr->op == nir_op_sudot_4x8_iadd_sat) inst->saturate = true; break; a1753 7 if (nir_has_any_rounding_mode_enabled(execution_mode)) { brw_rnd_mode rnd = brw_rnd_mode_from_execution_mode(execution_mode); bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); } d1755 1 a1758 7 if (nir_has_any_rounding_mode_enabled(execution_mode)) { brw_rnd_mode rnd = brw_rnd_mode_from_execution_mode(execution_mode); bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd)); } d1760 1 d1825 1 a1825 2 if (devinfo->ver <= 5 && !result.is_null() && d1859 3 a1861 3 assert(devinfo->ver >= 7); if (devinfo->ver == 7) { /* We don't get 64-bit integer types until gfx8 */ d1884 1 a1884 1 if (nir_src_is_undef(src)) { d1898 2 a1899 2 if (nir_src_bit_size(src) == 64 && devinfo->ver == 7) { /* The only 64-bit type available on gfx7 is DF, so use that. */ d1917 2 a1918 2 * We could theoretically support 64-bit on gfx8+ but we choose not to * because it wouldn't work in general (no gfx7 support) and there are a1940 1 bld.UNDEF(nir_ssa_values[dest.ssa.index]); d2020 1 a2020 1 GFX7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) { d2311 1 a2311 1 GFX7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) { a2323 1 assert(type_sz(dst.type) == 4); d2328 1 d2330 1 d2395 1 a2395 1 assert(devinfo->ver >= 9 || vertex <= 5); d2424 2 d2427 12 d2440 24 a2463 3 if (nir_src_is_const(offset_src)) { /* Constant indexing - use global offset. */ if (first_component != 0) { d2466 16 a2481 6 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle); inst->size_written = read_components * tmp.component_size(inst->exec_size); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); d2483 10 a2492 4 } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle); inst->size_written = num_components * dst.component_size(inst->exec_size); d2494 9 a2502 17 inst->offset = base_offset + nir_src_as_uint(offset_src); inst->mlen = 1; } else { /* Indirect indexing - use per-slot offsets as well. */ const fs_reg srcs[] = { icp_handle, indirect_offset }; unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); inst->size_written = read_components * tmp.component_size(inst->exec_size); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); a2503 4 } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload); inst->size_written = num_components * dst.component_size(inst->exec_size); a2504 2 inst->offset = base_offset; inst->mlen = 2; a2540 1 assert(nir_dest_bit_size(instr->dest) == 32); d2542 3 a2544 1 src = offset(src, bld, nir_intrinsic_component(instr)); d2547 8 a2554 2 for (unsigned i = 0; i < instr->num_components; i++) bld.MOV(offset(dest, bld, i), offset(src, bld, i)); d2572 3 a2574 3 fs_reg fs_visitor::get_tcs_single_patch_icp_handle(const fs_builder &bld, nir_intrinsic_instr *instr) d2576 2 a2578 22 const nir_src &vertex_src = instr->src[0]; nir_intrinsic_instr *vertex_intrin = nir_src_as_intrinsic(vertex_src); fs_reg icp_handle; if (nir_src_is_const(vertex_src)) { /* Emit a MOV to resolve <0,1,0> regioning. */ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); unsigned vertex = nir_src_as_uint(vertex_src); bld.MOV(icp_handle, retype(brw_vec1_grf(1 + (vertex >> 3), vertex & 7), BRW_REGISTER_TYPE_UD)); } else if (tcs_prog_data->instances == 1 && vertex_intrin && vertex_intrin->intrinsic == nir_intrinsic_load_invocation_id) { /* For the common case of only 1 instance, an array index of * gl_InvocationID means reading g1. Skip all the indirect work. */ icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD); } else { /* The vertex index is non-constant. We need to use indirect * addressing to fetch the proper URB handle. */ icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); d2580 3 a2582 5 /* Each ICP handle is a single DWord (4 bytes) */ fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); bld.SHL(vertex_offset_bytes, retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(2u)); d2584 5 a2588 98 /* Start at g1. We might read up to 4 registers. */ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle, retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes, brw_imm_ud(4 * REG_SIZE)); } return icp_handle; } fs_reg fs_visitor::get_tcs_eight_patch_icp_handle(const fs_builder &bld, nir_intrinsic_instr *instr) { struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key; struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data); const nir_src &vertex_src = instr->src[0]; unsigned first_icp_handle = tcs_prog_data->include_primitive_id ? 3 : 2; if (nir_src_is_const(vertex_src)) { return fs_reg(retype(brw_vec8_grf(first_icp_handle + nir_src_as_uint(vertex_src), 0), BRW_REGISTER_TYPE_UD)); } /* The vertex index is non-constant. We need to use indirect * addressing to fetch the proper URB handle. * * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0> * indicating that channel should read the handle from * DWord . We convert that to bytes by multiplying by 4. * * Next, we convert the vertex index to bytes by multiplying * by 32 (shifting by 5), and add the two together. This is * the final indirect byte offset. */ fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1); fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1); /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */ bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210))); /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */ bld.SHL(channel_offsets, sequence, brw_imm_ud(2u)); /* Convert vertex_index to bytes (multiply by 32) */ bld.SHL(vertex_offset_bytes, retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD), brw_imm_ud(5u)); bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets); /* Use first_icp_handle as the base offset. There is one register * of URB handles per vertex, so inform the register allocator that * we might read up to nir->info.gs.vertices_in registers. */ bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle, retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type), icp_offset_bytes, brw_imm_ud(tcs_key->input_vertices * REG_SIZE)); return icp_handle; } struct brw_reg fs_visitor::get_tcs_output_urb_handle() { struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data); if (vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_SINGLE_PATCH) { return retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD); } else { assert(vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH); return retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD); } } void fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(stage == MESA_SHADER_TESS_CTRL); struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key; struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data); struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base; bool eight_patch = vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH; fs_reg dst; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dst = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_primitive_id: bld.MOV(dst, fs_reg(eight_patch ? brw_vec8_grf(2, 0) : brw_vec1_grf(0, 1))); break; case nir_intrinsic_load_invocation_id: d2596 1 a2596 1 case nir_intrinsic_control_barrier: { d2608 1 a2608 15 if (devinfo->verx10 >= 125) { /* From BSpec: 54006, mov r0.2[31:24] into m0.2[31:24] and m0.2[23:16] */ fs_reg m0_10ub = component(retype(m0, BRW_REGISTER_TYPE_UB), 10); fs_reg r0_11ub = stride(suboffset(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UB), 11), 0, 1, 0); bld.exec_all().group(2, 0).MOV(m0_10ub, r0_11ub); } else if (devinfo->ver >= 11) { chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(INTEL_MASK(30, 24))); /* Set the Barrier Count and the enable bit */ chanbld.OR(m0_2, m0_2, brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15))); } else { d2615 4 d2620 2 a2621 1 /* Set the Barrier Count and the enable bit */ d2624 3 a2637 1 assert(nir_dest_bit_size(instr->dest) == 32); d2640 3 d2645 33 a2677 3 fs_reg icp_handle = eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr) : get_tcs_single_patch_icp_handle(bld, instr); d2683 1 d2686 11 d2698 13 a2710 9 if (indirect_offset.file == BAD_FILE) { /* Constant indexing - use global offset. */ if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); d2712 2 d2715 19 a2733 1 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle); d2735 24 a2758 19 inst->offset = imm_offset; inst->mlen = 1; } else { /* Indirect indexing - use per-slot offsets as well. */ const fs_reg srcs[] = { icp_handle, indirect_offset }; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dst.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dst, bld, i), offset(tmp, bld, i + first_component)); } } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload); a2759 5 inst->offset = imm_offset; inst->mlen = 2; } inst->size_written = (num_components + first_component) * inst->dst.component_size(inst->exec_size); d2761 7 a2767 9 /* Copy the temporary to the destination to deal with writemasking. * * Also attempt to deal with gl_PointSize being in the .w component. */ if (inst->offset == 0 && indirect_offset.file == BAD_FILE) { assert(type_sz(dst.type) == 4); inst->dst = bld.vgrf(dst.type, 4); inst->size_written = 4 * REG_SIZE; bld.MOV(dst, offset(inst->dst, bld, 3)); a2773 1 assert(nir_dest_bit_size(instr->dest) == 32); a2777 2 struct brw_reg output_handles = get_tcs_output_urb_handle(); d2780 1 a2780 3 /* This MOV replicates the output handle to all enabled channels * is SINGLE_PATCH mode. */ d2782 2 a2783 1 bld.MOV(patch_handle, output_handles); d2807 4 a2810 1 const fs_reg srcs[] = { output_handles, indirect_offset }; a2836 1 assert(nir_src_bit_size(instr->src[0]) == 32); d2838 2 a2843 2 struct brw_reg output_handles = get_tcs_output_urb_handle(); d2845 1 a2845 1 srcs[header_regs++] = output_handles; d2860 2 d2863 8 d2873 49 a2921 10 if (mask != WRITEMASK_XYZW) { srcs[header_regs++] = brw_imm_ud(mask << 16); opcode = indirect_offset.file != BAD_FILE ? SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT : SHADER_OPCODE_URB_WRITE_SIMD8_MASKED; } else { opcode = indirect_offset.file != BAD_FILE ? SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT : SHADER_OPCODE_URB_WRITE_SIMD8; } d2923 5 a2927 3 for (unsigned i = 0; i < num_components; i++) { if (!(mask & (1 << (i + first_component)))) continue; d2929 6 a2934 2 srcs[header_regs + i + first_component] = offset(value, bld, i); } d2936 3 a2938 4 unsigned mlen = header_regs + num_components + first_component; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen); bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs); d2940 8 a2947 3 fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload); inst->offset = imm_offset; inst->mlen = mlen; a2980 1 assert(nir_dest_bit_size(instr->dest) == 32); d2985 4 d2994 4 d2999 1 a2999 1 if (imm_offset < max_push_slots) { d3002 2 a3003 1 unsigned comp = 4 * (imm_offset % 2) + i + first_component; d3009 1 a3009 1 (imm_offset / 2) + 1); d3044 1 d3046 48 a3093 6 const fs_reg srcs[] = { retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD), indirect_offset }; fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2); bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0); d3095 6 a3100 9 if (first_component != 0) { unsigned read_components = num_components + first_component; fs_reg tmp = bld.vgrf(dest.type, read_components); inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp, payload); for (unsigned i = 0; i < num_components; i++) { bld.MOV(offset(dest, bld, i), offset(tmp, bld, i + first_component)); a3101 3 } else { inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest, payload); a3102 4 inst->mlen = 2; inst->offset = imm_offset; inst->size_written = (num_components + first_component) * inst->dst.component_size(inst->exec_size); d3148 1 a3148 1 case nir_intrinsic_set_vertex_and_primitive_count: d3172 1 a3172 9 if (bld.shader->devinfo->ver >= 12) { /* The render target array index is provided in the thread payload as * bits 26:16 of r1.1. */ const fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.AND(idx, brw_uw1_reg(BRW_GENERAL_REGISTER_FILE, 1, 3), brw_imm_uw(0x7ff)); return idx; } else if (bld.shader->devinfo->ver >= 6) { d3196 1 a3196 1 const struct intel_device_info *devinfo = bld.shader->devinfo; d3237 1 a3237 1 devinfo->ver >= 9 ? SHADER_OPCODE_TXF_CMS_W_LOGICAL : d3264 1 a3264 1 assert(bld.shader->devinfo->ver >= 9); a3351 17 case nir_intrinsic_is_helper_invocation: { /* Unlike the regular gl_HelperInvocation, that is defined at dispatch, * the helperInvocationEXT() (aka SpvOpIsHelperInvocationEXT) takes into * consideration demoted invocations. That information is stored in * f0.1. */ dest.type = BRW_REGISTER_TYPE_UD; bld.MOV(dest, brw_imm_ud(0)); fs_inst *mov = bld.MOV(dest, brw_imm_ud(~0)); mov->predicate = BRW_PREDICATE_NORMAL; mov->predicate_inverse = true; mov->flag_subreg = sample_mask_flag_subreg(this); break; } d3354 1 a3354 2 case nir_intrinsic_load_sample_id: case nir_intrinsic_load_frag_shading_rate: { a3398 1 case nir_intrinsic_demote: d3400 4 a3403 7 case nir_intrinsic_terminate: case nir_intrinsic_demote_if: case nir_intrinsic_discard_if: case nir_intrinsic_terminate_if: { /* We track our discarded pixels in f0.1/f1.0. By predicating on it, we * can update just the flag bits that aren't yet discarded. If there's * no condition, we emit a CMP of g0 != g0, so all currently executing d3406 4 a3409 50 fs_inst *cmp = NULL; if (instr->intrinsic == nir_intrinsic_demote_if || instr->intrinsic == nir_intrinsic_discard_if || instr->intrinsic == nir_intrinsic_terminate_if) { nir_alu_instr *alu = nir_src_as_alu_instr(instr->src[0]); if (alu != NULL && alu->op != nir_op_bcsel && (devinfo->ver > 5 || (alu->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) != BRW_NIR_BOOLEAN_NEEDS_RESOLVE || alu->op == nir_op_fneu32 || alu->op == nir_op_feq32 || alu->op == nir_op_flt32 || alu->op == nir_op_fge32 || alu->op == nir_op_ine32 || alu->op == nir_op_ieq32 || alu->op == nir_op_ilt32 || alu->op == nir_op_ige32 || alu->op == nir_op_ult32 || alu->op == nir_op_uge32)) { /* Re-emit the instruction that generated the Boolean value, but * do not store it. Since this instruction will be conditional, * other instructions that want to use the real Boolean value may * get garbage. This was a problem for piglit's fs-discard-exit-2 * test. * * Ideally we'd detect that the instruction cannot have a * conditional modifier before emitting the instructions. Alas, * that is nigh impossible. Instead, we're going to assume the * instruction (or last instruction) generated can have a * conditional modifier. If it cannot, fallback to the old-style * compare, and hope dead code elimination will clean up the * extra instructions generated. */ nir_emit_alu(bld, alu, false); cmp = (fs_inst *) instructions.get_tail(); if (cmp->conditional_mod == BRW_CONDITIONAL_NONE) { if (cmp->can_do_cmod()) cmp->conditional_mod = BRW_CONDITIONAL_Z; else cmp = NULL; } else { /* The old sequence that would have been generated is, * basically, bool_result == false. This is equivalent to * !bool_result, so negate the old modifier. */ cmp->conditional_mod = brw_negate_cmod(cmp->conditional_mod); } } if (cmp == NULL) { cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_Z); } a3414 1 d3416 1 a3416 1 cmp->flag_subreg = sample_mask_flag_subreg(this); d3418 2 a3419 12 fs_inst *jump = bld.emit(BRW_OPCODE_HALT); jump->flag_subreg = sample_mask_flag_subreg(this); jump->predicate_inverse = true; if (instr->intrinsic == nir_intrinsic_terminate || instr->intrinsic == nir_intrinsic_terminate_if) { jump->predicate = BRW_PREDICATE_NORMAL; } else { /* Only jump when the whole quad is demoted. For historical * reasons this is also used for discard. */ jump->predicate = BRW_PREDICATE_ALIGN1_ANY4H; d3422 1 a3422 3 if (devinfo->ver < 7) limit_dispatch_width( 16, "Fragment discard/demote not implemented in SIMD32 mode.\n"); a3427 1 assert(nir_dest_bit_size(instr->dest) == 32); d3431 2 d3440 10 d3451 2 a3452 2 bld.MOV(offset(dest, bld, i), retype(component(interp_reg(base, comp + i), 3), dest.type)); a3453 2 break; } d3455 4 a3458 9 case nir_intrinsic_load_fs_input_interp_deltas: { assert(stage == MESA_SHADER_FRAGMENT); assert(nir_src_as_uint(instr->src[0]) == 0); fs_reg interp = interp_reg(nir_intrinsic_base(instr), nir_intrinsic_component(instr)); dest.type = BRW_REGISTER_TYPE_F; bld.MOV(offset(dest, bld, 0), component(interp, 3)); bld.MOV(offset(dest, bld, 1), component(interp, 1)); bld.MOV(offset(dest, bld, 2), component(interp, 0)); d3464 2 a3465 9 case nir_intrinsic_load_barycentric_sample: { /* Use the delta_xy values computed from the payload */ const glsl_interp_mode interp_mode = (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr); enum brw_barycentric_mode bary = brw_barycentric_mode(interp_mode, instr->intrinsic); const fs_reg srcs[] = { offset(this->delta_xy[bary], bld, 0), offset(this->delta_xy[bary], bld, 1) }; bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0); a3466 1 } d3494 1 a3494 1 component(msg_data, 0), d3543 2 a3544 2 unsigned off_x = const_offset[0].u32 & 0xf; unsigned off_y = const_offset[1].u32 & 0xf; d3553 29 a3581 1 fs_reg src = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_D); d3593 5 a3597 3 case nir_intrinsic_load_frag_coord: emit_fragcoord_interpolation(dest); break; a3598 1 case nir_intrinsic_load_interpolated_input: { d3610 1 a3610 1 /* Use the result of the PI message. */ d3616 1 d3627 1 a3627 1 if (devinfo->ver < 6 && interp_mode == INTERP_MODE_SMOOTH) { d3644 14 d3662 1 a3662 1 assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL); d3670 1 a3670 11 case nir_intrinsic_control_barrier: /* The whole workgroup fits in a single HW thread, so all the * invocations are already executed lock-step. Instead of an actual * barrier just emit a scheduling fence, that will generate no code. */ if (!nir->info.workgroup_size_variable && workgroup_size() <= dispatch_width) { bld.exec_all().group(1, 0).emit(FS_OPCODE_SCHEDULING_FENCE); break; } d3676 1 a3676 6 if (devinfo->verx10 >= 125) bld.AND(retype(dest, BRW_REGISTER_TYPE_UD), retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD), brw_imm_ud(INTEL_MASK(7, 0))); else bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), subgroup_id); d3680 1 a3680 1 case nir_intrinsic_load_workgroup_id: { d3690 1 a3690 2 case nir_intrinsic_load_num_workgroups: { assert(nir_dest_bit_size(instr->dest) == 32); d3699 5 a3703 4 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(3); /* num components */ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(0); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); fs_inst *inst = d3705 2 a3706 2 dest, srcs, SURFACE_LOGICAL_NUM_SRCS); inst->size_written = 3 * dispatch_width * 4; d3711 2 d3714 2 d3717 2 d3720 2 d3723 2 d3726 2 d3729 2 d3732 2 d3735 2 d3738 1 a3738 1 nir_emit_shared_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr); d3741 2 d3744 2 d3747 1 a3747 1 nir_emit_shared_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr); d3751 2 a3752 2 assert(devinfo->ver >= 7); assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL); d3756 1 a3756 1 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); a3758 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); d3764 2 a3765 5 assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_intrinsic_align(instr) > 0); if (nir_dest_bit_size(instr->dest) == 32 && nir_intrinsic_align(instr) >= 4) { assert(nir_dest_num_components(instr->dest) <= 4); d3772 1 d3779 1 a3779 1 bld.MOV(dest, subscript(read_result, dest.type, 0)); d3785 2 a3786 2 assert(devinfo->ver >= 7); assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL); d3790 1 a3790 1 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); a3792 4 /* No point in masking with sample mask, here we're handling compute * intrinsics. */ srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); a3796 1 assert(nir_src_bit_size(instr->src[0]) <= 32); d3799 2 a3800 3 assert(nir_intrinsic_align(instr) > 0); if (nir_src_bit_size(instr->src[0]) == 32 && nir_intrinsic_align(instr) >= 4) { d3807 1 a3819 56 case nir_intrinsic_load_workgroup_size: { assert(compiler->lower_variable_group_size); assert(nir->info.workgroup_size_variable); for (unsigned i = 0; i < 3; i++) { bld.MOV(retype(offset(dest, bld, i), BRW_REGISTER_TYPE_UD), group_size[i]); } break; } default: nir_emit_intrinsic(bld, instr); break; } } void fs_visitor::nir_emit_bs_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr) { assert(brw_shader_stage_is_bindless(stage)); fs_reg dest; if (nir_intrinsic_infos[instr->intrinsic].has_dest) dest = get_nir_dest(instr->dest); switch (instr->intrinsic) { case nir_intrinsic_load_btd_global_arg_addr_intel: bld.MOV(dest, retype(brw_vec1_grf(2, 0), dest.type)); break; case nir_intrinsic_load_btd_local_arg_addr_intel: bld.MOV(dest, retype(brw_vec1_grf(2, 2), dest.type)); break; case nir_intrinsic_trace_ray_initial_intel: bld.emit(RT_OPCODE_TRACE_RAY_LOGICAL, bld.null_reg_ud(), brw_imm_ud(BRW_RT_BVH_LEVEL_WORLD), brw_imm_ud(GEN_RT_TRACE_RAY_INITAL)); break; case nir_intrinsic_trace_ray_commit_intel: bld.emit(RT_OPCODE_TRACE_RAY_LOGICAL, bld.null_reg_ud(), brw_imm_ud(BRW_RT_BVH_LEVEL_OBJECT), brw_imm_ud(GEN_RT_TRACE_RAY_COMMIT)); break; case nir_intrinsic_trace_ray_continue_intel: bld.emit(RT_OPCODE_TRACE_RAY_LOGICAL, bld.null_reg_ud(), brw_imm_ud(BRW_RT_BVH_LEVEL_OBJECT), brw_imm_ud(GEN_RT_TRACE_RAY_CONTINUE)); break; a3831 7 case 1: if (type == BRW_REGISTER_TYPE_UB) { return brw_imm_uw(value.u8); } else { assert(type == BRW_REGISTER_TYPE_B); return brw_imm_w(value.i8); } d3833 1 a3895 1 fs_reg surf_index = image; d3899 1 a3899 2 surf_index = brw_imm_ud(image.d + stage_prog_data->binding_table.image_start); d3901 1 a3901 2 surf_index = vgrf(glsl_type::uint_type); bld.ADD(surf_index, image, d3906 1 a3906 1 return bld.emit_uniformize(surf_index); d3914 1 a3914 4 const bool is_store = instr->intrinsic == nir_intrinsic_store_ssbo || instr->intrinsic == nir_intrinsic_store_ssbo_block_intel; const unsigned src = is_store ? 1 : 0; d3916 1 d3920 1 a3920 1 return brw_imm_ud(index); d3922 1 a3922 1 fs_reg surf_index = vgrf(glsl_type::uint_type); a3924 1 return bld.emit_uniformize(surf_index); a3925 1 } d3927 1 a3927 53 /** * The offsets we get from NIR act as if each SIMD channel has it's own blob * of contiguous space. However, if we actually place each SIMD channel in * it's own space, we end up with terrible cache performance because each SIMD * channel accesses a different cache line even when they're all accessing the * same byte offset. To deal with this problem, we swizzle the address using * a simple algorithm which ensures that any time a SIMD message reads or * writes the same address, it's all in the same cache line. We have to keep * the bottom two bits fixed so that we can read/write up to a dword at a time * and the individual element is contiguous. We do this by splitting the * address as follows: * * 31 4-6 2 0 * +-------------------------------+------------+----------+ * | Hi address bits | chan index | addr low | * +-------------------------------+------------+----------+ * * In other words, the bottom two address bits stay, and the top 30 get * shifted up so that we can stick the SIMD channel index in the middle. This * way, we can access 8, 16, or 32-bit elements and, when accessing a 32-bit * at the same logical offset, the scratch read/write instruction acts on * continuous elements and we get good cache locality. */ fs_reg fs_visitor::swizzle_nir_scratch_addr(const brw::fs_builder &bld, const fs_reg &nir_addr, bool in_dwords) { const fs_reg &chan_index = nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]; const unsigned chan_index_bits = ffs(dispatch_width) - 1; fs_reg addr = bld.vgrf(BRW_REGISTER_TYPE_UD); if (in_dwords) { /* In this case, we know the address is aligned to a DWORD and we want * the final address in DWORDs. */ bld.SHL(addr, nir_addr, brw_imm_ud(chan_index_bits - 2)); bld.OR(addr, addr, chan_index); } else { /* This case substantially more annoying because we have to pay * attention to those pesky two bottom bits. */ fs_reg addr_hi = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.AND(addr_hi, nir_addr, brw_imm_ud(~0x3u)); bld.SHL(addr_hi, addr_hi, brw_imm_ud(chan_index_bits)); fs_reg chan_addr = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.SHL(chan_addr, chan_index, brw_imm_ud(2)); bld.AND(addr, nir_addr, brw_imm_ud(0x3u)); bld.OR(addr, addr, addr_hi); bld.OR(addr, addr, chan_addr); } return addr; d3931 1 a3931 1 choose_oword_block_size_dwords(unsigned dwords) d3933 15 a3947 7 unsigned block; if (dwords >= 32) { block = 32; } else if (dwords >= 16) { block = 16; } else { block = 8; a3948 32 assert(block <= dwords); return block; } static void increment_a64_address(const fs_builder &bld, fs_reg address, uint32_t v) { if (bld.shader->devinfo->has_64bit_int) { bld.ADD(address, address, brw_imm_ud(v)); } else { fs_reg low = retype(address, BRW_REGISTER_TYPE_UD); fs_reg high = offset(low, bld, 1); /* Add low and if that overflows, add carry to high. */ bld.ADD(low, low, brw_imm_ud(v))->conditional_mod = BRW_CONDITIONAL_O; bld.ADD(high, high, brw_imm_ud(0x1))->predicate = BRW_PREDICATE_NORMAL; } } static fs_reg emit_fence(const fs_builder &bld, enum opcode opcode, uint8_t sfid, bool commit_enable, uint8_t bti) { assert(opcode == SHADER_OPCODE_INTERLOCK || opcode == SHADER_OPCODE_MEMORY_FENCE); fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD); fs_inst *fence = bld.emit(opcode, dst, brw_vec8_grf(0, 0), brw_imm_ud(commit_enable), brw_imm_ud(bti)); fence->sfid = sfid; return dst; d3962 2 a3963 4 case nir_intrinsic_image_atomic_imin: case nir_intrinsic_image_atomic_umin: case nir_intrinsic_image_atomic_imax: case nir_intrinsic_image_atomic_umax: d3972 2 a3973 4 case nir_intrinsic_bindless_image_atomic_imin: case nir_intrinsic_bindless_image_atomic_umin: case nir_intrinsic_bindless_image_atomic_imax: case nir_intrinsic_bindless_image_atomic_umax: d3979 4 d3985 1 d3993 2 a3994 4 case nir_intrinsic_image_atomic_imin: case nir_intrinsic_image_atomic_umin: case nir_intrinsic_image_atomic_imax: case nir_intrinsic_image_atomic_umax: d4013 1 a4013 1 brw_imm_ud(nir_image_intrinsic_coord_components(instr)); a4018 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); a4026 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); d4030 1 d4032 4 a4035 2 int op = brw_aop_for_nir_intrinsic(instr); if (op == BRW_AOP_INC || op == BRW_AOP_DEC) { d4037 38 a4074 1 num_srcs = 3; a4088 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); a4097 3 /* Cube image sizes should have previously been lowered to a 2D array */ assert(nir_intrinsic_image_dim(instr) != GLSL_SAMPLER_DIM_CUBE); a4106 2 assert(nir_src_as_uint(instr->src[1]) == 0); d4127 8 a4134 2 bld.MOV(offset(retype(dest, tmp.type), bld, c), component(offset(tmp, ubld, c), 0)); a4145 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); d4155 3 a4164 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); a4170 3 case nir_intrinsic_scoped_barrier: assert(nir_intrinsic_execution_scope(instr) == NIR_SCOPE_NONE); FALLTHROUGH; d4173 1 d4176 1 a4176 76 case nir_intrinsic_memory_barrier: case nir_intrinsic_begin_invocation_interlock: case nir_intrinsic_end_invocation_interlock: { bool ugm_fence, slm_fence, tgm_fence, urb_fence; const enum opcode opcode = instr->intrinsic == nir_intrinsic_begin_invocation_interlock ? SHADER_OPCODE_INTERLOCK : SHADER_OPCODE_MEMORY_FENCE; switch (instr->intrinsic) { case nir_intrinsic_scoped_barrier: { nir_variable_mode modes = nir_intrinsic_memory_modes(instr); ugm_fence = modes & (nir_var_mem_ssbo | nir_var_mem_global); slm_fence = modes & nir_var_mem_shared; tgm_fence = modes & nir_var_mem_ssbo; urb_fence = modes & nir_var_shader_out; break; } case nir_intrinsic_begin_invocation_interlock: case nir_intrinsic_end_invocation_interlock: /* For beginInvocationInterlockARB(), we will generate a memory fence * but with a different opcode so that generator can pick SENDC * instead of SEND. * * For endInvocationInterlockARB(), we need to insert a memory fence which * stalls in the shader until the memory transactions prior to that * fence are complete. This ensures that the shader does not end before * any writes from its critical section have landed. Otherwise, you can * end up with a case where the next invocation on that pixel properly * stalls for previous FS invocation on its pixel to complete but * doesn't actually wait for the dataport memory transactions from that * thread to land before submitting its own. * * Handling them here will allow the logic for IVB render cache (see * below) to be reused. */ assert(stage == MESA_SHADER_FRAGMENT); ugm_fence = tgm_fence = true; slm_fence = urb_fence = false; break; default: ugm_fence = instr->intrinsic != nir_intrinsic_memory_barrier_shared && instr->intrinsic != nir_intrinsic_memory_barrier_image; slm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier || instr->intrinsic == nir_intrinsic_memory_barrier || instr->intrinsic == nir_intrinsic_memory_barrier_shared; tgm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier || instr->intrinsic == nir_intrinsic_memory_barrier || instr->intrinsic == nir_intrinsic_memory_barrier_image; urb_fence = instr->intrinsic == nir_intrinsic_memory_barrier; break; } if (nir->info.shared_size > 0) { assert(gl_shader_stage_uses_workgroup(stage)); } else { slm_fence = false; } /* If the workgroup fits in a single HW thread, the messages for SLM are * processed in-order and the shader itself is already synchronized so * the memory fence is not necessary. * * TODO: Check if applies for many HW threads sharing same Data Port. */ if (!nir->info.workgroup_size_variable && slm_fence && workgroup_size() <= dispatch_width) slm_fence = false; if (stage != MESA_SHADER_TESS_CTRL) urb_fence = false; unsigned fence_regs_count = 0; fs_reg fence_regs[3] = {}; d4178 4 a4181 95 if (devinfo->has_lsc) { assert(devinfo->verx10 >= 125); if (ugm_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX12_SFID_UGM, true /* commit_enable */, 0 /* bti; ignored for LSC */); } if (tgm_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX12_SFID_TGM, true /* commit_enable */, 0 /* bti; ignored for LSC */); } if (slm_fence) { assert(opcode == SHADER_OPCODE_MEMORY_FENCE); fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX12_SFID_SLM, true /* commit_enable */, 0 /* BTI; ignored for LSC */); } if (urb_fence) { assert(opcode == SHADER_OPCODE_MEMORY_FENCE); fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, BRW_SFID_URB, true /* commit_enable */, 0 /* BTI; ignored for LSC */); } } else if (devinfo->ver >= 11) { if (tgm_fence || ugm_fence || urb_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX7_SFID_DATAPORT_DATA_CACHE, true /* commit_enable HSD ES # 1404612949 */, 0 /* BTI = 0 means data cache */); } if (slm_fence) { assert(opcode == SHADER_OPCODE_MEMORY_FENCE); fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX7_SFID_DATAPORT_DATA_CACHE, true /* commit_enable HSD ES # 1404612949 */, GFX7_BTI_SLM); } } else { /* Prior to Icelake, they're all lumped into a single cache except on * Ivy Bridge and Bay Trail where typed messages actually go through * the render cache. There, we need both fences because we may * access storage images as either typed or untyped. */ const bool render_fence = tgm_fence && devinfo->verx10 == 70; const bool commit_enable = render_fence || instr->intrinsic == nir_intrinsic_end_invocation_interlock; if (tgm_fence || ugm_fence || slm_fence || urb_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX7_SFID_DATAPORT_DATA_CACHE, commit_enable, 0 /* BTI */); } if (render_fence) { fence_regs[fence_regs_count++] = emit_fence(ubld, opcode, GFX6_SFID_DATAPORT_RENDER_CACHE, commit_enable, /* bti */ 0); } } assert(fence_regs_count <= ARRAY_SIZE(fence_regs)); /* There are three cases where we want to insert a stall: * * 1. If we're a nir_intrinsic_end_invocation_interlock. This is * required to ensure that the shader EOT doesn't happen until * after the fence returns. Otherwise, we might end up with the * next shader invocation for that pixel not respecting our fence * because it may happen on a different HW thread. * * 2. If we have multiple fences. This is required to ensure that * they all complete and nothing gets weirdly out-of-order. * * 3. If we have no fences. In this case, we need at least a * scheduling barrier to keep the compiler from moving things * around in an invalid way. */ if (instr->intrinsic == nir_intrinsic_end_invocation_interlock || fence_regs_count != 1) { ubld.exec_all().group(1, 0).emit( FS_OPCODE_SCHEDULING_FENCE, ubld.null_reg_ud(), fence_regs, fence_regs_count); } a4184 3 case nir_intrinsic_memory_barrier_tcs_patch: break; a4198 7 case nir_intrinsic_load_reloc_const_intel: { uint32_t id = nir_intrinsic_param_idx(instr); bld.emit(SHADER_OPCODE_MOV_RELOC_IMM, dest, brw_imm_ud(id)); break; } d4234 1 a4234 1 !devinfo->is_cherryview && !intel_device_info_is_9lp(devinfo); d4286 1 a4286 4 base_offset, i * type_sz(dest.type), nir_dest_bit_size(instr->dest) / 8); prog_data->has_ubo_pull = true; a4325 2 prog_data->has_ubo_pull = true; d4353 5 a4357 9 case nir_intrinsic_load_global: case nir_intrinsic_load_global_constant: { assert(devinfo->ver >= 8); assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_intrinsic_align(instr) > 0); if (nir_dest_bit_size(instr->dest) == 32 && nir_intrinsic_align(instr) >= 4) { assert(nir_dest_num_components(instr->dest) <= 4); d4367 1 d4369 2 d4377 1 a4377 1 bld.MOV(dest, subscript(tmp, dest.type, 0)); d4383 1 a4383 1 assert(devinfo->ver >= 8); d4385 5 a4389 7 assert(nir_src_bit_size(instr->src[0]) <= 32); assert(nir_intrinsic_write_mask(instr) == (1u << instr->num_components) - 1); assert(nir_intrinsic_align(instr) > 0); if (nir_src_bit_size(instr->src[0]) == 32 && nir_intrinsic_align(instr) >= 4) { assert(nir_src_num_components(instr->src[0]) <= 4); d4396 2 a4398 1 const unsigned bit_size = nir_src_bit_size(instr->src[0]); d4412 2 d4415 2 d4418 2 d4421 2 d4424 2 d4427 2 d4430 2 d4433 2 d4436 2 d4439 1 a4439 1 nir_emit_global_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr); a4440 1 case nir_intrinsic_global_atomic_fadd: d4442 2 d4445 2 d4448 1 a4448 57 nir_emit_global_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr); break; case nir_intrinsic_load_global_const_block_intel: { assert(nir_dest_bit_size(instr->dest) == 32); assert(instr->num_components == 8 || instr->num_components == 16); const fs_builder ubld = bld.exec_all().group(instr->num_components, 0); fs_reg load_val; bool is_pred_const = nir_src_is_const(instr->src[1]); if (is_pred_const && nir_src_as_uint(instr->src[1]) == 0) { /* In this case, we don't want the UBO load at all. We really * shouldn't get here but it's possible. */ load_val = brw_imm_ud(0); } else { /* The uniform process may stomp the flag so do this first */ fs_reg addr = bld.emit_uniformize(get_nir_src(instr->src[0])); load_val = ubld.vgrf(BRW_REGISTER_TYPE_UD); /* If the predicate is constant and we got here, then it's non-zero * and we don't need the predicate at all. */ if (!is_pred_const) { /* Load the predicate */ fs_reg pred = bld.emit_uniformize(get_nir_src(instr->src[1])); fs_inst *mov = ubld.MOV(bld.null_reg_d(), pred); mov->conditional_mod = BRW_CONDITIONAL_NZ; /* Stomp the destination with 0 if we're OOB */ mov = ubld.MOV(load_val, brw_imm_ud(0)); mov->predicate = BRW_PREDICATE_NORMAL; mov->predicate_inverse = true; } fs_inst *load = ubld.emit(SHADER_OPCODE_A64_OWORD_BLOCK_READ_LOGICAL, load_val, addr, fs_reg(), /* No source data */ brw_imm_ud(instr->num_components)); if (!is_pred_const) load->predicate = BRW_PREDICATE_NORMAL; } /* From the HW perspective, we just did a single SIMD16 instruction * which loaded a dword in each SIMD channel. From NIR's perspective, * this instruction returns a vec16. Any users of this data in the * back-end will expect a vec16 per SIMD channel so we have to emit a * pile of MOVs to resolve this discrepancy. Fortunately, copy-prop * will generally clean them up for us. */ for (unsigned i = 0; i < instr->num_components; i++) { bld.MOV(retype(offset(dest, bld, i), BRW_REGISTER_TYPE_UD), component(load_val, i)); } a4449 1 } d4452 1 a4452 1 assert(devinfo->ver >= 7); a4459 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); d4465 2 a4466 5 assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_intrinsic_align(instr) > 0); if (nir_dest_bit_size(instr->dest) == 32 && nir_intrinsic_align(instr) >= 4) { assert(nir_dest_num_components(instr->dest) <= 4); d4473 1 d4480 1 a4480 1 bld.MOV(dest, subscript(read_result, dest.type, 0)); d4486 4 a4489 1 assert(devinfo->ver >= 7); a4496 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); a4500 1 assert(nir_src_bit_size(instr->src[0]) <= 32); d4503 2 a4504 3 assert(nir_intrinsic_align(instr) > 0); if (nir_src_bit_size(instr->src[0]) == 32 && nir_intrinsic_align(instr) >= 4) { d4511 1 a4524 1 assert(nir_src_bit_size(instr->src[0]) == 32); d4530 4 d4545 2 d4548 2 d4551 2 d4554 2 d4557 2 d4560 2 d4563 2 d4566 2 d4569 2 d4572 1 a4572 1 nir_emit_ssbo_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr); a4573 1 case nir_intrinsic_ssbo_atomic_fadd: d4575 2 d4578 2 d4581 1 a4581 1 nir_emit_ssbo_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr); d4584 1 a4584 1 case nir_intrinsic_get_ssbo_size: { a4642 143 case nir_intrinsic_load_scratch: { assert(devinfo->ver >= 7); assert(nir_dest_num_components(instr->dest) == 1); const unsigned bit_size = nir_dest_bit_size(instr->dest); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; if (devinfo->verx10 >= 125) { const fs_builder ubld = bld.exec_all().group(1, 0); fs_reg handle = component(ubld.vgrf(BRW_REGISTER_TYPE_UD), 0); ubld.AND(handle, retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD), brw_imm_ud(~0x3ffu)); srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] = handle; } else if (devinfo->ver >= 8) { srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX8_BTI_STATELESS_NON_COHERENT); } else { srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(BRW_BTI_STATELESS); } srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); const fs_reg nir_addr = get_nir_src(instr->src[0]); /* Make dest unsigned because that's what the temporary will be */ dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); /* Read the vector */ assert(nir_dest_num_components(instr->dest) == 1); assert(nir_dest_bit_size(instr->dest) <= 32); assert(nir_intrinsic_align(instr) > 0); if (devinfo->verx10 >= 125) { assert(nir_dest_bit_size(instr->dest) == 32 && nir_intrinsic_align(instr) >= 4); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, false); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } else if (nir_dest_bit_size(instr->dest) >= 4 && nir_intrinsic_align(instr) >= 4) { /* The offset for a DWORD scattered message is in dwords. */ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, true); bld.emit(SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } else { srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, false); fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL, read_result, srcs, SURFACE_LOGICAL_NUM_SRCS); bld.MOV(dest, read_result); } break; } case nir_intrinsic_store_scratch: { assert(devinfo->ver >= 7); assert(nir_src_num_components(instr->src[0]) == 1); const unsigned bit_size = nir_src_bit_size(instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; if (devinfo->verx10 >= 125) { const fs_builder ubld = bld.exec_all().group(1, 0); fs_reg handle = component(ubld.vgrf(BRW_REGISTER_TYPE_UD), 0); ubld.AND(handle, retype(brw_vec1_grf(0, 5), BRW_REGISTER_TYPE_UD), brw_imm_ud(~0x3ffu)); srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] = handle; } else if (devinfo->ver >= 8) { srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX8_BTI_STATELESS_NON_COHERENT); } else { srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(BRW_BTI_STATELESS); } srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size); /** * While this instruction has side-effects, it should not be predicated * on sample mask, because otherwise fs helper invocations would * load undefined values from scratch memory. And scratch memory * load-stores are produced from operations without side-effects, thus * they should not have different behaviour in the helper invocations. */ srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(0); const fs_reg nir_addr = get_nir_src(instr->src[1]); fs_reg data = get_nir_src(instr->src[0]); data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD); assert(nir_src_num_components(instr->src[0]) == 1); assert(nir_src_bit_size(instr->src[0]) <= 32); assert(nir_intrinsic_write_mask(instr) == 1); assert(nir_intrinsic_align(instr) > 0); if (devinfo->verx10 >= 125) { assert(nir_src_bit_size(instr->src[0]) == 32 && nir_intrinsic_align(instr) >= 4); srcs[SURFACE_LOGICAL_SRC_DATA] = data; srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, false); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL, dest, srcs, SURFACE_LOGICAL_NUM_SRCS); } else if (nir_src_bit_size(instr->src[0]) == 32 && nir_intrinsic_align(instr) >= 4) { srcs[SURFACE_LOGICAL_SRC_DATA] = data; /* The offset for a DWORD scattered message is in dwords. */ srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, true); bld.emit(SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); } else { srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = swizzle_nir_scratch_addr(bld, nir_addr, false); bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); } break; } case nir_intrinsic_load_subgroup_size: /* This should only happen for fragment shaders because every other case * is lowered in NIR so we can optimize on it. */ assert(stage == MESA_SHADER_FRAGMENT); bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(dispatch_width)); break; d4832 1 a4832 1 if (devinfo->ver <= 7) { d4834 1 a4834 1 * compressed instructions on gfx7 and earlier so we fall back to d4836 1 a4836 1 * anything in Vulkan on gfx7. d4990 3 a4992 4 case nir_intrinsic_load_global_block_intel: { assert(nir_dest_bit_size(instr->dest) == 32); fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[0])); d4994 2 a4995 24 const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); const fs_builder ubld16 = bld.exec_all().group(16, 0); const unsigned total = instr->num_components * dispatch_width; unsigned loaded = 0; while (loaded < total) { const unsigned block = choose_oword_block_size_dwords(total - loaded); const unsigned block_bytes = block * 4; const fs_builder &ubld = block == 8 ? ubld8 : ubld16; ubld.emit(SHADER_OPCODE_A64_UNALIGNED_OWORD_BLOCK_READ_LOGICAL, retype(byte_offset(dest, loaded * 4), BRW_REGISTER_TYPE_UD), address, fs_reg(), /* No source data */ brw_imm_ud(block))->size_written = block_bytes; increment_a64_address(ubld1, address, block_bytes); loaded += block; } assert(loaded == total); d4999 15 a5013 113 case nir_intrinsic_store_global_block_intel: { assert(nir_src_bit_size(instr->src[0]) == 32); fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[1])); fs_reg src = get_nir_src(instr->src[0]); const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); const fs_builder ubld16 = bld.exec_all().group(16, 0); const unsigned total = instr->num_components * dispatch_width; unsigned written = 0; while (written < total) { const unsigned block = choose_oword_block_size_dwords(total - written); const fs_builder &ubld = block == 8 ? ubld8 : ubld16; ubld.emit(SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL, fs_reg(), address, retype(byte_offset(src, written * 4), BRW_REGISTER_TYPE_UD), brw_imm_ud(block)); const unsigned block_bytes = block * 4; increment_a64_address(ubld1, address, block_bytes); written += block; } assert(written == total); break; } case nir_intrinsic_load_shared_block_intel: case nir_intrinsic_load_ssbo_block_intel: { assert(nir_dest_bit_size(instr->dest) == 32); const bool is_ssbo = instr->intrinsic == nir_intrinsic_load_ssbo_block_intel; fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 1 : 0])); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = is_ssbo ? get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM)); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = address; const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); const fs_builder ubld16 = bld.exec_all().group(16, 0); const unsigned total = instr->num_components * dispatch_width; unsigned loaded = 0; while (loaded < total) { const unsigned block = choose_oword_block_size_dwords(total - loaded); const unsigned block_bytes = block * 4; srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(block); const fs_builder &ubld = block == 8 ? ubld8 : ubld16; ubld.emit(SHADER_OPCODE_UNALIGNED_OWORD_BLOCK_READ_LOGICAL, retype(byte_offset(dest, loaded * 4), BRW_REGISTER_TYPE_UD), srcs, SURFACE_LOGICAL_NUM_SRCS)->size_written = block_bytes; ubld1.ADD(address, address, brw_imm_ud(block_bytes)); loaded += block; } assert(loaded == total); break; } case nir_intrinsic_store_shared_block_intel: case nir_intrinsic_store_ssbo_block_intel: { assert(nir_src_bit_size(instr->src[0]) == 32); const bool is_ssbo = instr->intrinsic == nir_intrinsic_store_ssbo_block_intel; fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 2 : 1])); fs_reg src = get_nir_src(instr->src[0]); fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS]; srcs[SURFACE_LOGICAL_SRC_SURFACE] = is_ssbo ? get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM)); srcs[SURFACE_LOGICAL_SRC_ADDRESS] = address; const fs_builder ubld1 = bld.exec_all().group(1, 0); const fs_builder ubld8 = bld.exec_all().group(8, 0); const fs_builder ubld16 = bld.exec_all().group(16, 0); const unsigned total = instr->num_components * dispatch_width; unsigned written = 0; while (written < total) { const unsigned block = choose_oword_block_size_dwords(total - written); srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(block); srcs[SURFACE_LOGICAL_SRC_DATA] = retype(byte_offset(src, written * 4), BRW_REGISTER_TYPE_UD); const fs_builder &ubld = block == 8 ? ubld8 : ubld16; ubld.emit(SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL, fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS); const unsigned block_bytes = block * 4; ubld1.ADD(address, address, brw_imm_ud(block_bytes)); written += block; } assert(written == total); a5016 38 case nir_intrinsic_load_btd_dss_id_intel: bld.emit(SHADER_OPCODE_GET_DSS_ID, retype(dest, BRW_REGISTER_TYPE_UD)); break; case nir_intrinsic_load_btd_stack_id_intel: if (stage == MESA_SHADER_COMPUTE) { assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); } else { assert(brw_shader_stage_is_bindless(stage)); } /* Stack IDs are always in R1 regardless of whether we're coming from a * bindless shader or a regular compute shader. */ bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UW)); break; case nir_intrinsic_btd_spawn_intel: if (stage == MESA_SHADER_COMPUTE) { assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); } else { assert(brw_shader_stage_is_bindless(stage)); } bld.emit(SHADER_OPCODE_BTD_SPAWN_LOGICAL, bld.null_reg_ud(), bld.emit_uniformize(get_nir_src(instr->src[0])), get_nir_src(instr->src[1])); break; case nir_intrinsic_btd_retire_intel: if (stage == MESA_SHADER_COMPUTE) { assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); } else { assert(brw_shader_stage_is_bindless(stage)); } bld.emit(SHADER_OPCODE_BTD_RETIRE_LOGICAL); break; d5026 3 d5034 1 a5034 2 assert(nir_dest_bit_size(instr->dest) == 32 || (nir_dest_bit_size(instr->dest) == 64 && devinfo->has_lsc)); a5044 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); d5068 3 a5079 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); d5105 1 a5105 1 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); a5107 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); d5146 1 a5146 1 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GFX7_BTI_SLM); a5148 1 srcs[SURFACE_LOGICAL_SRC_ALLOW_SAMPLE_MASK] = brw_imm_ud(1); a5175 12 static fs_reg expand_to_32bit(const fs_builder &bld, const fs_reg &src) { if (type_sz(src.type) == 2) { fs_reg src32 = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.MOV(src32, retype(src, BRW_REGISTER_TYPE_UW)); return src32; } else { return src; } } d5180 3 d5191 1 a5191 1 data = expand_to_32bit(bld, get_nir_src(instr->src[1])); d5195 1 a5195 4 fs_reg sources[2] = { data, expand_to_32bit(bld, get_nir_src(instr->src[2])) }; d5200 5 a5204 9 switch (nir_dest_bit_size(instr->dest)) { case 16: { fs_reg dest32 = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT16_LOGICAL, dest32, addr, data, brw_imm_ud(op)); bld.MOV(retype(dest, BRW_REGISTER_TYPE_UW), dest32); break; } case 32: a5206 7 break; case 64: bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL, dest, addr, data, brw_imm_ud(op)); break; default: unreachable("Unsupported bit size"); d5214 3 d5223 1 a5223 1 fs_reg data = expand_to_32bit(bld, get_nir_src(instr->src[1])); d5227 1 a5227 4 fs_reg sources[2] = { data, expand_to_32bit(bld, get_nir_src(instr->src[2])) }; d5232 2 a5233 19 switch (nir_dest_bit_size(instr->dest)) { case 16: { fs_reg dest32 = bld.vgrf(BRW_REGISTER_TYPE_UD); bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT16_LOGICAL, dest32, addr, data, brw_imm_ud(op)); bld.MOV(retype(dest, BRW_REGISTER_TYPE_UW), dest32); break; } case 32: bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT32_LOGICAL, dest, addr, data, brw_imm_ud(op)); break; case 64: bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT64_LOGICAL, dest, addr, data, brw_imm_ud(op)); break; default: unreachable("Unsupported bit size"); } d5268 1 a5268 1 case nir_texop_txf_ms_mcs_intel: a5275 7 /* Wa_14013363432: * * Compiler should send U,V,R parameters even if V,R are 0. */ if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && devinfo->verx10 == 125) assert(instr->coord_components >= 3u); d5350 1 a5350 1 case nir_tex_src_ms_mcs_intel: d5374 1 a5374 1 if (devinfo->ver >= 7 && d5412 1 a5412 1 case nir_texop_txf_ms_mcs_intel: d5471 1 a5471 1 if (devinfo->ver >= 9 && d5486 2 a5487 2 if (instr->op == nir_texop_tg4 && devinfo->ver == 6) emit_gfx6_gather_wa(key_tex->gfx6_gather_wa[texture], dst); d5489 1 a5489 1 fs_reg nir_dest[5]; d5495 1 a5495 15 if (devinfo->ver <= 9) { /** * Wa_1940217: * * When a surface of type SURFTYPE_NULL is accessed by resinfo, the * MIPCount returned is undefined instead of 0. */ fs_inst *mov = bld.MOV(bld.null_reg_d(), dst); mov->conditional_mod = BRW_CONDITIONAL_NZ; nir_dest[0] = bld.vgrf(BRW_REGISTER_TYPE_D); fs_inst *sel = bld.SEL(nir_dest[0], offset(dst, bld, 3), brw_imm_d(0)); sel->predicate = BRW_PREDICATE_NORMAL; } else { nir_dest[0] = offset(dst, bld, 3); } d5497 2 a5498 2 dest_size >= 3 && devinfo->ver < 7) { /* Gfx4-6 return 0 instead of 1 for single layer surfaces. */ a5516 3 case nir_jump_halt: bld.emit(BRW_OPCODE_HALT); break; d5646 22 d5670 2 a5671 2 const struct intel_device_info *devinfo = bld.shader->devinfo; assert(devinfo->ver >= 7); d5673 1 a5673 1 if (devinfo->ver >= 8) d5676 1 a5676 1 /* gfx7.5 does not support DF immediates straighforward but the DIM d5686 1 a5686 1 /* gfx7 does not support DF immediates, so we generate a 64-bit constant by d5692 1 a5692 1 * gfx7 bug where we have to split writes that span more than 1 register @