/* * Copyright © 2014 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /* * This lowering pass converts references to input/output variables with * loads/stores to actual input/output intrinsics. */ #include "nir.h" #include "nir_builder.h" #include "nir_deref.h" #include "util/u_math.h" struct lower_io_state { void *dead_ctx; nir_builder builder; int (*type_size)(const struct glsl_type *type, bool); nir_variable_mode modes; nir_lower_io_options options; struct set variable_names; }; static const char * add_variable_name(struct lower_io_state *state, const char *name) { if (!name) return NULL; bool found = false; struct set_entry *entry = _mesa_set_search_or_add(&state->variable_names, name, &found); if (!found) entry->key = (void *)ralloc_strdup(state->builder.shader, name); return entry->key; } /** * Some inputs and outputs are arrayed, meaning that there is an extra level * of array indexing to handle mismatches between the shader interface and the * dispatch pattern of the shader. For instance, geometry shaders are * executed per-primitive while their inputs and outputs are specified * per-vertex so all inputs and outputs have to be additionally indexed with * the vertex index within the primitive. */ bool nir_is_arrayed_io(const nir_variable *var, gl_shader_stage stage) { if (var->data.patch || !glsl_type_is_array(var->type)) return false; if (var->data.per_view) { /* Nested arrayed outputs (both per-view and per-{vertex,primitive}) are * unsupported. */ assert(stage == MESA_SHADER_VERTEX); assert(var->data.mode == nir_var_shader_out); return true; } if (stage == MESA_SHADER_MESH) { /* NV_mesh_shader: this is flat array for the whole workgroup. */ if (var->data.location == VARYING_SLOT_PRIMITIVE_INDICES) return var->data.per_primitive; } if (var->data.mode == nir_var_shader_in) { if (var->data.per_vertex) { assert(stage == MESA_SHADER_FRAGMENT); return true; } return stage == MESA_SHADER_GEOMETRY || stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL; } if (var->data.mode == nir_var_shader_out) return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_MESH; return false; } static bool uses_high_dvec2_semantic(struct lower_io_state *state, const nir_variable *var) { return state->builder.shader->info.stage == MESA_SHADER_VERTEX && state->options & nir_lower_io_lower_64bit_to_32_new && var->data.mode == nir_var_shader_in && glsl_type_is_dual_slot(glsl_without_array(var->type)); } static unsigned get_number_of_slots(struct lower_io_state *state, const nir_variable *var) { const struct glsl_type *type = var->type; if (nir_is_arrayed_io(var, state->builder.shader->info.stage)) { assert(glsl_type_is_array(type)); type = glsl_get_array_element(type); } /* NV_mesh_shader: * PRIMITIVE_INDICES is a flat array, not a proper arrayed output, * as opposed to D3D-style mesh shaders where it's addressed by * the primitive index. * Prevent assigning several slots to primitive indices, * to avoid some issues. */ if (state->builder.shader->info.stage == MESA_SHADER_MESH && var->data.location == VARYING_SLOT_PRIMITIVE_INDICES && !nir_is_arrayed_io(var, state->builder.shader->info.stage)) return 1; return state->type_size(type, var->data.bindless) / (uses_high_dvec2_semantic(state, var) ? 2 : 1); } static nir_def * get_io_offset(nir_builder *b, nir_deref_instr *deref, nir_def **array_index, int (*type_size)(const struct glsl_type *, bool), unsigned *component, bool bts) { nir_deref_path path; nir_deref_path_init(&path, deref, NULL); assert(path.path[0]->deref_type == nir_deref_type_var); nir_deref_instr **p = &path.path[1]; /* For arrayed I/O (e.g., per-vertex input arrays in geometry shader * inputs), skip the outermost array index. Process the rest normally. */ if (array_index != NULL) { assert((*p)->deref_type == nir_deref_type_array); *array_index = (*p)->arr.index.ssa; p++; } if (path.path[0]->var->data.compact && nir_src_is_const((*p)->arr.index)) { assert((*p)->deref_type == nir_deref_type_array); assert(glsl_type_is_scalar((*p)->type)); /* We always lower indirect dereferences for "compact" array vars. */ const unsigned index = nir_src_as_uint((*p)->arr.index); const unsigned total_offset = *component + index; const unsigned slot_offset = total_offset / 4; *component = total_offset % 4; return nir_imm_int(b, type_size(glsl_vec4_type(), bts) * slot_offset); } /* Just emit code and let constant-folding go to town */ nir_def *offset = nir_imm_int(b, 0); for (; *p; p++) { if ((*p)->deref_type == nir_deref_type_array) { unsigned size = type_size((*p)->type, bts); nir_def *mul = nir_amul_imm(b, (*p)->arr.index.ssa, size); offset = nir_iadd(b, offset, mul); } else if ((*p)->deref_type == nir_deref_type_struct) { /* p starts at path[1], so this is safe */ nir_deref_instr *parent = *(p - 1); unsigned field_offset = 0; for (unsigned i = 0; i < (*p)->strct.index; i++) { field_offset += type_size(glsl_get_struct_field(parent->type, i), bts); } offset = nir_iadd_imm(b, offset, field_offset); } else { UNREACHABLE("Unsupported deref type"); } } nir_deref_path_finish(&path); return offset; } static bool is_medium_precision(const nir_shader *shader, const nir_variable *var) { if (shader->options->io_options & nir_io_mediump_is_32bit) return false; return var->data.precision == GLSL_PRECISION_MEDIUM || var->data.precision == GLSL_PRECISION_LOW; } static enum glsl_interp_mode get_interp_mode(const nir_variable *var) { unsigned interp_mode = var->data.interpolation; /* INTERP_MODE_NONE is an artifact of OpenGL. Change it to SMOOTH * to enable CSE between load_barycentric_pixel(NONE->SMOOTH) and * load_barycentric_pixel(SMOOTH), which also enables IO vectorization when * one component originally had NONE and an adjacent component had SMOOTH. * * Color varyings must preserve NONE. NONE for colors means that * glShadeModel determines the interpolation mode. */ if (var->data.location != VARYING_SLOT_COL0 && var->data.location != VARYING_SLOT_COL1 && var->data.location != VARYING_SLOT_BFC0 && var->data.location != VARYING_SLOT_BFC1 && interp_mode == INTERP_MODE_NONE) return INTERP_MODE_SMOOTH; return interp_mode; } static nir_def * emit_load(struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, unsigned num_components, unsigned bit_size, nir_alu_type dest_type, bool high_dvec2) { nir_builder *b = &state->builder; const nir_shader *nir = b->shader; nir_variable_mode mode = var->data.mode; nir_def *barycentric = NULL; nir_intrinsic_op op; switch (mode) { case nir_var_shader_in: if (nir->info.stage == MESA_SHADER_FRAGMENT && state->options & nir_lower_io_use_interpolated_input_intrinsics && var->data.interpolation != INTERP_MODE_FLAT && !var->data.per_primitive) { if (var->data.interpolation == INTERP_MODE_EXPLICIT || var->data.per_vertex) { assert(array_index != NULL); op = nir_intrinsic_load_input_vertex; } else { assert(array_index == NULL); nir_intrinsic_op bary_op; if (var->data.sample) bary_op = nir_intrinsic_load_barycentric_sample; else if (var->data.centroid) bary_op = nir_intrinsic_load_barycentric_centroid; else bary_op = nir_intrinsic_load_barycentric_pixel; barycentric = nir_load_barycentric(&state->builder, bary_op, get_interp_mode(var)); op = nir_intrinsic_load_interpolated_input; } } else { if (var->data.per_primitive) op = nir_intrinsic_load_per_primitive_input; else if (array_index) op = nir_intrinsic_load_per_vertex_input; else op = nir_intrinsic_load_input; } break; case nir_var_shader_out: if (!array_index) op = nir_intrinsic_load_output; else if (var->data.per_primitive) op = nir_intrinsic_load_per_primitive_output; else if (var->data.per_view) op = nir_intrinsic_load_per_view_output; else op = nir_intrinsic_load_per_vertex_output; break; case nir_var_uniform: op = nir_intrinsic_load_uniform; break; default: UNREACHABLE("Unknown variable mode"); } nir_intrinsic_instr *load = nir_intrinsic_instr_create(state->builder.shader, op); load->num_components = num_components; load->name = add_variable_name(state, var->name); nir_intrinsic_set_base(load, var->data.driver_location); if (nir_intrinsic_has_range(load)) { const struct glsl_type *type = var->type; if (array_index) type = glsl_get_array_element(type); unsigned var_size = state->type_size(type, var->data.bindless); if (var_size) nir_intrinsic_set_range(load, var_size); else nir_intrinsic_set_range(load, ~0); } if (mode == nir_var_shader_in || mode == nir_var_shader_out) nir_intrinsic_set_component(load, component); if (nir_intrinsic_has_access(load)) nir_intrinsic_set_access(load, var->data.access); nir_intrinsic_set_dest_type(load, dest_type); if (load->intrinsic != nir_intrinsic_load_uniform) { int location = var->data.location; unsigned num_slots = get_number_of_slots(state, var); /* Maximum values in nir_io_semantics. */ assert(num_slots <= 63); assert(location >= 0 && location + num_slots <= NUM_TOTAL_VARYING_SLOTS); nir_io_semantics semantics = { 0 }; semantics.location = location; semantics.num_slots = num_slots; semantics.fb_fetch_output = var->data.fb_fetch_output; if (semantics.fb_fetch_output) { semantics.fb_fetch_output_coherent = !!(var->data.access & ACCESS_COHERENT); } semantics.medium_precision = is_medium_precision(b->shader, var); semantics.high_dvec2 = high_dvec2; /* "per_vertex" is misnamed. It means "explicit interpolation with * the original vertex order", which is a stricter version of * INTERP_MODE_EXPLICIT. */ semantics.interp_explicit_strict = var->data.per_vertex; nir_intrinsic_set_io_semantics(load, semantics); } if (array_index) { load->src[0] = nir_src_for_ssa(array_index); load->src[1] = nir_src_for_ssa(offset); } else if (barycentric) { load->src[0] = nir_src_for_ssa(barycentric); load->src[1] = nir_src_for_ssa(offset); } else { load->src[0] = nir_src_for_ssa(offset); } nir_def_init(&load->instr, &load->def, num_components, bit_size); nir_builder_instr_insert(b, &load->instr); return &load->def; } static nir_def * lower_load(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { const bool lower_double = !glsl_type_is_integer(type) && state->options & nir_lower_io_lower_64bit_float_to_32; if (intrin->def.bit_size == 64 && (lower_double || (state->options & (nir_lower_io_lower_64bit_to_32_new | nir_lower_io_lower_64bit_to_32)))) { nir_builder *b = &state->builder; bool use_high_dvec2_semantic = uses_high_dvec2_semantic(state, var); /* Each slot is a dual slot, so divide the offset within the variable * by 2. */ if (use_high_dvec2_semantic) offset = nir_ushr_imm(b, offset, 1); const unsigned slot_size = state->type_size(glsl_dvec_type(2), false); nir_def *comp64[4]; assert(component == 0 || component == 2); unsigned dest_comp = 0; bool high_dvec2 = false; while (dest_comp < intrin->def.num_components) { const unsigned num_comps = MIN2(intrin->def.num_components - dest_comp, (4 - component) / 2); nir_def *data32 = emit_load(state, array_index, var, offset, component, num_comps * 2, 32, nir_type_uint32, high_dvec2); for (unsigned i = 0; i < num_comps; i++) { comp64[dest_comp + i] = nir_pack_64_2x32(b, nir_channels(b, data32, 3 << (i * 2))); } /* Only the first store has a component offset */ component = 0; dest_comp += num_comps; if (use_high_dvec2_semantic) { /* Increment the offset when we wrap around the dual slot. */ if (high_dvec2) offset = nir_iadd_imm(b, offset, slot_size); high_dvec2 = !high_dvec2; } else { offset = nir_iadd_imm(b, offset, slot_size); } } return nir_vec(b, comp64, intrin->def.num_components); } else if (intrin->def.bit_size == 1) { /* Booleans are 32-bit */ assert(glsl_type_is_boolean(type)); return nir_b2b1(&state->builder, emit_load(state, array_index, var, offset, component, intrin->def.num_components, 32, nir_type_bool32, false)); } else { return emit_load(state, array_index, var, offset, component, intrin->def.num_components, intrin->def.bit_size, nir_get_nir_type_for_glsl_type(type), false); } } static void emit_store(struct lower_io_state *state, nir_def *data, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, unsigned num_components, nir_component_mask_t write_mask, nir_alu_type src_type) { nir_builder *b = &state->builder; assert(var->data.mode == nir_var_shader_out); nir_intrinsic_op op; if (!array_index) op = nir_intrinsic_store_output; else if (var->data.per_view) op = nir_intrinsic_store_per_view_output; else if (var->data.per_primitive) op = nir_intrinsic_store_per_primitive_output; else op = nir_intrinsic_store_per_vertex_output; nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->builder.shader, op); store->num_components = num_components; store->name = add_variable_name(state, var->name); store->src[0] = nir_src_for_ssa(data); const struct glsl_type *type = var->type; if (array_index) type = glsl_get_array_element(type); unsigned var_size = state->type_size(type, var->data.bindless); nir_intrinsic_set_base(store, var->data.driver_location); nir_intrinsic_set_range(store, var_size); nir_intrinsic_set_component(store, component); nir_intrinsic_set_src_type(store, src_type); nir_intrinsic_set_write_mask(store, write_mask); if (nir_intrinsic_has_access(store)) nir_intrinsic_set_access(store, var->data.access); if (array_index) store->src[1] = nir_src_for_ssa(array_index); store->src[array_index ? 2 : 1] = nir_src_for_ssa(offset); unsigned gs_streams = 0; if (state->builder.shader->info.stage == MESA_SHADER_GEOMETRY) { if (var->data.stream & NIR_STREAM_PACKED) { gs_streams = var->data.stream & ~NIR_STREAM_PACKED; } else { assert(var->data.stream < 4); gs_streams = 0; for (unsigned i = 0; i < num_components; ++i) gs_streams |= var->data.stream << (2 * i); } } int location = var->data.location; unsigned num_slots = get_number_of_slots(state, var); /* Maximum values in nir_io_semantics. */ assert(num_slots <= 63); assert(location >= 0 && location + num_slots <= NUM_TOTAL_VARYING_SLOTS); nir_io_semantics semantics = { 0 }; semantics.location = location; semantics.num_slots = num_slots; semantics.dual_source_blend_index = var->data.index; semantics.gs_streams = gs_streams; semantics.medium_precision = is_medium_precision(b->shader, var); semantics.per_view = var->data.per_view; nir_intrinsic_set_io_semantics(store, semantics); nir_builder_instr_insert(b, &store->instr); } static void lower_store(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_def *array_index, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { const bool lower_double = !glsl_type_is_integer(type) && state->options & nir_lower_io_lower_64bit_float_to_32; if (intrin->src[1].ssa->bit_size == 64 && (lower_double || (state->options & (nir_lower_io_lower_64bit_to_32 | nir_lower_io_lower_64bit_to_32_new)))) { nir_builder *b = &state->builder; const unsigned slot_size = state->type_size(glsl_dvec_type(2), false); assert(component == 0 || component == 2); unsigned src_comp = 0; nir_component_mask_t write_mask = nir_intrinsic_write_mask(intrin); while (src_comp < intrin->num_components) { const unsigned num_comps = MIN2(intrin->num_components - src_comp, (4 - component) / 2); if (write_mask & BITFIELD_MASK(num_comps)) { nir_def *data = nir_channels(b, intrin->src[1].ssa, BITFIELD_RANGE(src_comp, num_comps)); nir_def *data32 = nir_bitcast_vector(b, data, 32); uint32_t write_mask32 = 0; for (unsigned i = 0; i < num_comps; i++) { if (write_mask & BITFIELD_MASK(num_comps) & (1 << i)) write_mask32 |= 3 << (i * 2); } emit_store(state, data32, array_index, var, offset, component, data32->num_components, write_mask32, nir_type_uint32); } /* Only the first store has a component offset */ component = 0; src_comp += num_comps; write_mask >>= num_comps; offset = nir_iadd_imm(b, offset, slot_size); } } else if (intrin->def.bit_size == 1) { /* Booleans are 32-bit */ assert(glsl_type_is_boolean(type)); nir_def *b32_val = nir_b2b32(&state->builder, intrin->src[1].ssa); emit_store(state, b32_val, array_index, var, offset, component, intrin->num_components, nir_intrinsic_write_mask(intrin), nir_type_bool32); } else { emit_store(state, intrin->src[1].ssa, array_index, var, offset, component, intrin->num_components, nir_intrinsic_write_mask(intrin), nir_get_nir_type_for_glsl_type(type)); } } static nir_def * lower_interpolate_at(nir_intrinsic_instr *intrin, struct lower_io_state *state, nir_variable *var, nir_def *offset, unsigned component, const struct glsl_type *type) { nir_builder *b = &state->builder; assert(var->data.mode == nir_var_shader_in); /* Ignore interpolateAt() for flat variables - flat is flat. Lower * interpolateAtVertex() for explicit variables. */ if (var->data.interpolation == INTERP_MODE_FLAT || var->data.interpolation == INTERP_MODE_EXPLICIT) { nir_def *vertex_index = NULL; if (var->data.interpolation == INTERP_MODE_EXPLICIT) { assert(intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex); vertex_index = intrin->src[1].ssa; } return lower_load(intrin, state, vertex_index, var, offset, component, type); } /* None of the supported APIs allow interpolation on 64-bit things */ assert(intrin->def.bit_size <= 32); nir_intrinsic_op bary_op; switch (intrin->intrinsic) { case nir_intrinsic_interp_deref_at_centroid: bary_op = nir_intrinsic_load_barycentric_centroid; break; case nir_intrinsic_interp_deref_at_sample: bary_op = nir_intrinsic_load_barycentric_at_sample; break; case nir_intrinsic_interp_deref_at_offset: bary_op = nir_intrinsic_load_barycentric_at_offset; break; default: UNREACHABLE("Bogus interpolateAt() intrinsic."); } nir_intrinsic_instr *bary_setup = nir_intrinsic_instr_create(state->builder.shader, bary_op); nir_def_init(&bary_setup->instr, &bary_setup->def, 2, 32); nir_intrinsic_set_interp_mode(bary_setup, get_interp_mode(var)); if (intrin->intrinsic == nir_intrinsic_interp_deref_at_sample || intrin->intrinsic == nir_intrinsic_interp_deref_at_offset || intrin->intrinsic == nir_intrinsic_interp_deref_at_vertex) bary_setup->src[0] = nir_src_for_ssa(intrin->src[1].ssa); nir_builder_instr_insert(b, &bary_setup->instr); nir_io_semantics semantics = { 0 }; semantics.location = var->data.location; semantics.num_slots = get_number_of_slots(state, var); semantics.medium_precision = is_medium_precision(b->shader, var); nir_def *load = nir_load_interpolated_input(&state->builder, intrin->def.num_components, intrin->def.bit_size, &bary_setup->def, offset, .base = var->data.driver_location, .component = component, .io_semantics = semantics); return load; } /** * Convert a compact view index emitted by nir_lower_multiview to an absolute * view index. */ static nir_def * uncompact_view_index(nir_builder *b, nir_src compact_index_src) { /* We require nir_lower_io_vars_to_temporaries when using absolute view indices, * which ensures index is constant */ assert(nir_src_is_const(compact_index_src)); unsigned compact_index = nir_src_as_uint(compact_index_src); unsigned view_index; uint32_t view_mask = b->shader->info.view_mask; for (unsigned i = 0; i <= compact_index; i++) { view_index = u_bit_scan(&view_mask); } return nir_imm_int(b, view_index); } static bool nir_lower_io_block(nir_block *block, struct lower_io_state *state) { nir_builder *b = &state->builder; const nir_shader_compiler_options *options = b->shader->options; bool progress = false; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: case nir_intrinsic_store_deref: /* We can lower the io for this nir instrinsic */ break; case nir_intrinsic_interp_deref_at_centroid: case nir_intrinsic_interp_deref_at_sample: case nir_intrinsic_interp_deref_at_offset: case nir_intrinsic_interp_deref_at_vertex: /* We can optionally lower these to load_interpolated_input */ if (state->options & nir_lower_io_use_interpolated_input_intrinsics || options->lower_interpolate_at) break; FALLTHROUGH; default: /* We can't lower the io for this nir instrinsic, so skip it */ continue; } nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); if (!nir_deref_mode_is_one_of(deref, state->modes)) continue; nir_variable *var = nir_deref_instr_get_variable(deref); b->cursor = nir_before_instr(instr); const bool is_arrayed = nir_is_arrayed_io(var, b->shader->info.stage); nir_def *offset; nir_def *array_index = NULL; unsigned component_offset = var->data.location_frac; bool bindless_type_size = var->data.mode == nir_var_shader_in || var->data.mode == nir_var_shader_out || var->data.bindless; if (nir_deref_instr_is_known_out_of_bounds(deref)) { /* Section 5.11 (Out-of-Bounds Accesses) of the GLSL 4.60 spec says: * * In the subsections described above for array, vector, matrix and * structure accesses, any out-of-bounds access produced undefined * behavior.... * Out-of-bounds reads return undefined values, which * include values from other variables of the active program or zero. * Out-of-bounds writes may be discarded or overwrite * other variables of the active program. * * GL_KHR_robustness and GL_ARB_robustness encourage us to return zero * for reads. * * Otherwise get_io_offset would return out-of-bound offset which may * result in out-of-bound loading/storing of inputs/outputs, * that could cause issues in drivers down the line. */ if (intrin->intrinsic != nir_intrinsic_store_deref) { nir_def *zero = nir_imm_zero(b, intrin->def.num_components, intrin->def.bit_size); nir_def_rewrite_uses(&intrin->def, zero); } nir_instr_remove(&intrin->instr); progress = true; continue; } offset = get_io_offset(b, deref, is_arrayed ? &array_index : NULL, state->type_size, &component_offset, bindless_type_size); if (!options->compact_view_index && array_index && var->data.per_view) array_index = uncompact_view_index(b, nir_src_for_ssa(array_index)); nir_def *replacement = NULL; switch (intrin->intrinsic) { case nir_intrinsic_load_deref: replacement = lower_load(intrin, state, array_index, var, offset, component_offset, deref->type); break; case nir_intrinsic_store_deref: lower_store(intrin, state, array_index, var, offset, component_offset, deref->type); break; case nir_intrinsic_interp_deref_at_centroid: case nir_intrinsic_interp_deref_at_sample: case nir_intrinsic_interp_deref_at_offset: case nir_intrinsic_interp_deref_at_vertex: assert(array_index == NULL); replacement = lower_interpolate_at(intrin, state, var, offset, component_offset, deref->type); break; default: continue; } if (replacement) { nir_def_rewrite_uses(&intrin->def, replacement); } nir_instr_remove(&intrin->instr); progress = true; } return progress; } static bool nir_lower_io_impl(nir_function_impl *impl, nir_variable_mode modes, int (*type_size)(const struct glsl_type *, bool), nir_lower_io_options options) { struct lower_io_state state; bool progress = false; state.builder = nir_builder_create(impl); state.dead_ctx = ralloc_context(NULL); state.modes = modes; state.type_size = type_size; state.options = options; _mesa_set_init(&state.variable_names, state.dead_ctx, _mesa_hash_string, _mesa_key_string_equal); ASSERTED nir_variable_mode supported_modes = nir_var_shader_in | nir_var_shader_out | nir_var_uniform; assert(!(modes & ~supported_modes)); nir_foreach_block(block, impl) { progress |= nir_lower_io_block(block, &state); } ralloc_free(state.dead_ctx); nir_progress(true, impl, nir_metadata_none); return progress; } /** Lower load/store_deref intrinsics on I/O variables to offset-based intrinsics * * This pass is intended to be used for cross-stage shader I/O and driver- * managed uniforms to turn deref-based access into a simpler model using * locations or offsets. For fragment shader inputs, it can optionally turn * load_deref into an explicit interpolation using barycentrics coming from * one of the load_barycentric_* intrinsics. This pass requires that all * deref chains are complete and contain no casts. */ bool nir_lower_io(nir_shader *shader, nir_variable_mode modes, int (*type_size)(const struct glsl_type *, bool), nir_lower_io_options options) { bool progress = false; nir_foreach_function_impl(impl, shader) { progress |= nir_lower_io_impl(impl, modes, type_size, options); } return progress; } /** * Return the offset source number for a load/store intrinsic or -1 if there's no offset. */ int nir_get_io_offset_src_number(const nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_input: case nir_intrinsic_load_per_primitive_input: case nir_intrinsic_load_output: case nir_intrinsic_load_shared: case nir_intrinsic_load_task_payload: case nir_intrinsic_load_uniform: case nir_intrinsic_load_constant: case nir_intrinsic_load_push_constant: case nir_intrinsic_load_kernel_input: case nir_intrinsic_load_global: case nir_intrinsic_load_global_2x32: case nir_intrinsic_load_global_constant: case nir_intrinsic_load_global_etna: case nir_intrinsic_load_scratch: case nir_intrinsic_load_fs_input_interp_deltas: case nir_intrinsic_shared_atomic: case nir_intrinsic_shared_atomic_swap: case nir_intrinsic_task_payload_atomic: case nir_intrinsic_task_payload_atomic_swap: case nir_intrinsic_global_atomic: case nir_intrinsic_global_atomic_2x32: case nir_intrinsic_global_atomic_swap: case nir_intrinsic_global_atomic_swap_2x32: case nir_intrinsic_load_coefficients_agx: case nir_intrinsic_load_shared_block_intel: case nir_intrinsic_load_global_block_intel: case nir_intrinsic_load_shared_uniform_block_intel: case nir_intrinsic_load_global_constant_uniform_block_intel: case nir_intrinsic_load_shared2_amd: case nir_intrinsic_load_const_ir3: case nir_intrinsic_load_shared_ir3: return 0; case nir_intrinsic_load_ubo: case nir_intrinsic_load_ubo_vec4: case nir_intrinsic_load_ssbo: case nir_intrinsic_load_input_vertex: case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_load_per_view_output: case nir_intrinsic_load_per_primitive_output: case nir_intrinsic_load_interpolated_input: case nir_intrinsic_load_smem_amd: case nir_intrinsic_store_output: case nir_intrinsic_store_shared: case nir_intrinsic_store_task_payload: case nir_intrinsic_store_global: case nir_intrinsic_store_global_2x32: case nir_intrinsic_store_global_etna: case nir_intrinsic_store_scratch: case nir_intrinsic_ssbo_atomic: case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_ldc_nv: case nir_intrinsic_ldcx_nv: case nir_intrinsic_load_ssbo_block_intel: case nir_intrinsic_store_global_block_intel: case nir_intrinsic_store_shared_block_intel: case nir_intrinsic_load_ubo_uniform_block_intel: case nir_intrinsic_load_ssbo_uniform_block_intel: case nir_intrinsic_load_buffer_amd: case nir_intrinsic_store_shared2_amd: case nir_intrinsic_store_shared_ir3: case nir_intrinsic_load_ssbo_intel: return 1; case nir_intrinsic_store_ssbo: case nir_intrinsic_store_per_vertex_output: case nir_intrinsic_store_per_view_output: case nir_intrinsic_store_per_primitive_output: case nir_intrinsic_load_attribute_pan: case nir_intrinsic_store_ssbo_block_intel: case nir_intrinsic_store_buffer_amd: case nir_intrinsic_store_ssbo_intel: return 2; case nir_intrinsic_load_ssbo_ir3: /* This intrinsic has 2 offsets (src1 bytes, src2 dwords), we return the * dwords one for opt_offsets. */ return 2; case nir_intrinsic_store_ssbo_ir3: /* This intrinsic has 2 offsets (src2 bytes, src3 dwords), we return the * dwords one for opt_offsets. */ return 3; default: return -1; } } /** * Return the offset source for a load/store intrinsic. */ nir_src * nir_get_io_offset_src(nir_intrinsic_instr *instr) { const int idx = nir_get_io_offset_src_number(instr); return idx >= 0 ? &instr->src[idx] : NULL; } /** * Return the index or handle source number for a load/store intrinsic or -1 * if there's no index or handle. */ int nir_get_io_index_src_number(const nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_ubo: case nir_intrinsic_load_ssbo: case nir_intrinsic_load_input_vertex: case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_load_per_view_output: case nir_intrinsic_load_per_primitive_output: case nir_intrinsic_load_interpolated_input: case nir_intrinsic_load_smem_amd: case nir_intrinsic_ldc_nv: case nir_intrinsic_ldcx_nv: case nir_intrinsic_load_ssbo_intel: case nir_intrinsic_load_ssbo_block_intel: case nir_intrinsic_store_global_block_intel: case nir_intrinsic_store_shared_block_intel: case nir_intrinsic_load_ubo_uniform_block_intel: case nir_intrinsic_load_ssbo_uniform_block_intel: return 0; case nir_intrinsic_store_ssbo: case nir_intrinsic_store_per_vertex_output: case nir_intrinsic_store_per_view_output: case nir_intrinsic_store_per_primitive_output: case nir_intrinsic_store_ssbo_block_intel: case nir_intrinsic_store_ssbo_intel: return 1; default: return -1; } } /** * Return the offset or handle source for a load/store intrinsic. */ nir_src * nir_get_io_index_src(nir_intrinsic_instr *instr) { const int idx = nir_get_io_index_src_number(instr); return idx >= 0 ? &instr->src[idx] : NULL; } /** * Return the array index source number for an arrayed load/store intrinsic or -1 if there's no offset. */ int nir_get_io_arrayed_index_src_number(const nir_intrinsic_instr *instr) { switch (instr->intrinsic) { case nir_intrinsic_load_per_vertex_input: case nir_intrinsic_load_per_vertex_output: case nir_intrinsic_load_per_view_output: case nir_intrinsic_load_per_primitive_output: return 0; case nir_intrinsic_store_per_vertex_output: case nir_intrinsic_store_per_view_output: case nir_intrinsic_store_per_primitive_output: return 1; default: return -1; } } bool nir_is_output_load(nir_intrinsic_instr *intr) { return intr->intrinsic == nir_intrinsic_load_output || intr->intrinsic == nir_intrinsic_load_per_vertex_output || intr->intrinsic == nir_intrinsic_load_per_primitive_output || intr->intrinsic == nir_intrinsic_load_per_view_output; } /** * Return the array index source for an arrayed load/store intrinsic. */ nir_src * nir_get_io_arrayed_index_src(nir_intrinsic_instr *instr) { const int idx = nir_get_io_arrayed_index_src_number(instr); return idx >= 0 ? &instr->src[idx] : NULL; } static int type_size_vec4(const struct glsl_type *type, bool bindless) { return glsl_count_attribute_slots(type, false); } /** * This runs all compiler passes needed to lower IO, lower indirect IO access, * set transform feedback info in IO intrinsics, and clean up the IR. * * \param renumber_vs_inputs * Set to true if holes between VS inputs should be removed, which is safe * to do in any shader linker that can handle that. Set to false if you want * to keep holes between VS inputs, which is recommended to do in gallium * drivers so as not to break the mapping of vertex elements to VS inputs * expected by gallium frontends. */ void nir_lower_io_passes(nir_shader *nir, bool renumber_vs_inputs) { if (gl_shader_stage_is_compute(nir->info.stage)) return; bool lower_indirect_inputs = !(nir->options->support_indirect_inputs & BITFIELD_BIT(nir->info.stage)); /* Transform feedback requires that indirect outputs are lowered. */ bool lower_indirect_outputs = !(nir->options->support_indirect_outputs & BITFIELD_BIT(nir->info.stage)) || nir->xfb_info; /* TODO: This is a hack until a better solution is available. * For all shaders except TCS, lower all outputs to temps because: * - there can be output loads (nobody expects those outside of TCS) * - drivers don't expect when an output is only written in control flow * * "lower_indirect_outputs = true" causes all outputs to be lowered to temps, * which lowers indirect stores, eliminates output loads, and moves all * output stores to the end or GS emits. */ if (nir->info.stage != MESA_SHADER_TESS_CTRL) lower_indirect_outputs = true; /* TODO: Sorting variables by location is required due to some bug * in nir_lower_io_vars_to_temporaries. If variables are not sorted, * dEQP-GLES31.functional.separate_shader.random.0 fails. * * This isn't needed if nir_assign_io_var_locations is called because it * also sorts variables. However, if IO is lowered sooner than that, we * must sort explicitly here to get what nir_assign_io_var_locations does. */ unsigned varying_var_mask = (nir->info.stage != MESA_SHADER_VERTEX ? nir_var_shader_in : 0) | (nir->info.stage != MESA_SHADER_FRAGMENT ? nir_var_shader_out : 0); nir_sort_variables_by_location(nir, varying_var_mask); if (lower_indirect_outputs) { NIR_PASS(_, nir, nir_lower_io_vars_to_temporaries, nir_shader_get_entrypoint(nir), true, false); /* We need to lower all the copy_deref's introduced by lower_io_to- * _temporaries before calling nir_lower_io. */ NIR_PASS(_, nir, nir_split_var_copies); NIR_PASS(_, nir, nir_lower_var_copies); NIR_PASS(_, nir, nir_lower_global_vars_to_local); /* This is partially redundant with nir_lower_io_vars_to_temporaries. * The problem is that nir_lower_io_vars_to_temporaries doesn't handle TCS. */ if (nir->info.stage == MESA_SHADER_TESS_CTRL) { NIR_PASS(_, nir, nir_lower_indirect_derefs, nir_var_shader_out, UINT32_MAX); } } /* The correct lower_64bit_to_32 flag is required by st/mesa depending * on whether the GLSL linker lowers IO or not. Setting the wrong flag * would break 64-bit vertex attribs for GLSL. */ NIR_PASS(_, nir, nir_lower_io, nir_var_shader_out | nir_var_shader_in, type_size_vec4, (renumber_vs_inputs ? nir_lower_io_lower_64bit_to_32_new : nir_lower_io_lower_64bit_to_32) | nir_lower_io_use_interpolated_input_intrinsics); /* nir_io_add_const_offset_to_base needs actual constants. */ NIR_PASS(_, nir, nir_opt_constant_folding); NIR_PASS(_, nir, nir_io_add_const_offset_to_base, nir_var_shader_in | nir_var_shader_out); /* This must be called after nir_io_add_const_offset_to_base. */ if (lower_indirect_inputs) NIR_PASS(_, nir, nir_lower_io_indirect_loads, nir_var_shader_in); /* Lower and remove dead derefs and variables to clean up the IR. */ NIR_PASS(_, nir, nir_lower_vars_to_ssa); NIR_PASS(_, nir, nir_opt_dce); NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_function_temp, NULL); /* If IO is lowered before var->data.driver_location is assigned, driver * locations are all 0, which means IO bases are all 0. It's not necessary * to set driver_location before lowering IO because the only thing that * identifies outputs is their semantic, and IO bases can always be * computed from the semantics. * * This assigns IO bases from scratch, using IO semantics to tell which * intrinsics refer to the same IO. If the bases already exist, they * will be reassigned, sorted by the semantic, and all holes removed. * This kind of canonicalizes all bases. * * This must be done after DCE to remove dead load_input intrinsics. */ NIR_PASS(_, nir, nir_recompute_io_bases, (nir->info.stage != MESA_SHADER_VERTEX || renumber_vs_inputs ? nir_var_shader_in : 0) | nir_var_shader_out); if (nir->xfb_info) NIR_PASS(_, nir, nir_io_add_intrinsic_xfb_info); if (nir->options->lower_mediump_io) nir->options->lower_mediump_io(nir); nir->info.io_lowered = true; }