/* * Copyright © 2022 Imagination Technologies Ltd. * * based in part on v3dv driver which is: * Copyright © 2019 Raspberry Pi * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include "compiler/shader_enums.h" #include "hwdef/rogue_hw_utils.h" #include "nir/nir.h" #include "pco/pco.h" #include "pco/pco_data.h" #include "pvr_bo.h" #include "pvr_csb.h" #include "pvr_csb_enum_helpers.h" #include "pvr_hardcode.h" #include "pvr_pds.h" #include "pvr_private.h" #include "pvr_robustness.h" #include "pvr_shader.h" #include "pvr_types.h" #include "rogue/rogue.h" #include "util/log.h" #include "util/macros.h" #include "util/ralloc.h" #include "util/u_dynarray.h" #include "util/u_math.h" #include "vk_alloc.h" #include "vk_format.h" #include "vk_graphics_state.h" #include "vk_log.h" #include "vk_object.h" #include "vk_pipeline_cache.h" #include "vk_pipeline_layout.h" #include "vk_render_pass.h" #include "vk_util.h" #include "vulkan/runtime/vk_pipeline.h" /***************************************************************************** PDS functions *****************************************************************************/ /* If allocator == NULL, the internal one will be used. */ static VkResult pvr_pds_coeff_program_create_and_upload( struct pvr_device *device, const VkAllocationCallbacks *allocator, struct pvr_pds_coeff_loading_program *program, struct pvr_fragment_shader_state *fragment_state) { uint32_t staging_buffer_size; uint32_t *staging_buffer; VkResult result; assert(program->num_fpu_iterators < PVR_MAXIMUM_ITERATIONS); /* Get the size of the program and then allocate that much memory. */ pvr_pds_coefficient_loading(program, NULL, PDS_GENERATE_SIZES); if (!program->code_size) { fragment_state->pds_coeff_program.pvr_bo = NULL; fragment_state->pds_coeff_program.code_size = 0; fragment_state->pds_coeff_program.data_size = 0; fragment_state->stage_state.pds_temps_count = 0; return VK_SUCCESS; } staging_buffer_size = PVR_DW_TO_BYTES(program->code_size + program->data_size); staging_buffer = vk_alloc2(&device->vk.alloc, allocator, staging_buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (!staging_buffer) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); /* Generate the program into is the staging_buffer. */ pvr_pds_coefficient_loading(program, staging_buffer, PDS_GENERATE_CODEDATA_SEGMENTS); /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, &staging_buffer[0], program->data_size, 16, &staging_buffer[program->data_size], program->code_size, 16, 16, &fragment_state->pds_coeff_program); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, staging_buffer); return result; } vk_free2(&device->vk.alloc, allocator, staging_buffer); fragment_state->stage_state.pds_temps_count = program->temps_used; return VK_SUCCESS; } /* FIXME: move this elsewhere since it's also called in pvr_pass.c? */ /* If allocator == NULL, the internal one will be used. */ VkResult pvr_pds_fragment_program_create_and_upload( struct pvr_device *device, const VkAllocationCallbacks *allocator, pco_shader *fs, struct pvr_fragment_shader_state *fragment_state) { /* TODO: remove the below + revert the pvr_pds_setup_doutu * args and make sure fs isn't NULL instead; * temporarily in place for hardcoded load ops in * pvr_pass.c:pvr_generate_load_op_shader() */ unsigned temps = 0; bool has_phase_rate_change = false; unsigned entry_offset = 0; if (fs) { pco_data *fs_data = pco_shader_data(fs); temps = fs_data->common.temps; has_phase_rate_change = fs_data->fs.uses.phase_change; entry_offset = fs_data->common.entry_offset; } struct pvr_pds_kickusc_program program = { 0 }; uint32_t staging_buffer_size; uint32_t *staging_buffer; VkResult result; const pvr_dev_addr_t exec_addr = PVR_DEV_ADDR_OFFSET(fragment_state->bo->dev_addr, /* fs_data->common.entry_offset */ entry_offset); /* Note this is not strictly required to be done before calculating the * staging_buffer_size in this particular case. It can also be done after * allocating the buffer. The size from pvr_pds_kick_usc() is constant. */ pvr_pds_setup_doutu( &program.usc_task_control, exec_addr.addr, /* fs_data->common.temps */ temps, fragment_state->sample_rate, /* fs_data->fs.uses.phase_change */ has_phase_rate_change); pvr_pds_kick_usc(&program, NULL, 0, false, PDS_GENERATE_SIZES); staging_buffer_size = PVR_DW_TO_BYTES(program.code_size + program.data_size); staging_buffer = vk_alloc2(&device->vk.alloc, allocator, staging_buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (!staging_buffer) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); pvr_pds_kick_usc(&program, staging_buffer, 0, false, PDS_GENERATE_CODEDATA_SEGMENTS); /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, &staging_buffer[0], program.data_size, 16, &staging_buffer[program.data_size], program.code_size, 16, 16, &fragment_state->pds_fragment_program); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, staging_buffer); return result; } vk_free2(&device->vk.alloc, allocator, staging_buffer); return VK_SUCCESS; } static inline size_t pvr_pds_get_max_vertex_program_const_map_size_in_bytes( const struct pvr_device_info *dev_info, bool robust_buffer_access) { /* FIXME: Use more local variable to improve formatting. */ /* Maximum memory allocation needed for const map entries in * pvr_pds_generate_vertex_primary_program(). * When robustBufferAccess is disabled, it must be >= 410. * When robustBufferAccess is enabled, it must be >= 570. * * 1. Size of entry for base instance * (pvr_const_map_entry_base_instance) * * 2. Max. number of vertex inputs (PVR_MAX_VERTEX_INPUT_BINDINGS) * ( * if (!robustBufferAccess) * size of vertex attribute entry * (pvr_const_map_entry_vertex_attribute_address) + * else * size of robust vertex attribute entry * (pvr_const_map_entry_robust_vertex_attribute_address) + * size of entry for max attribute index * (pvr_const_map_entry_vertex_attribute_max_index) + * fi * size of Unified Store burst entry * (pvr_const_map_entry_literal32) + * size of entry for vertex stride * (pvr_const_map_entry_literal32) + * size of entries for DDMAD control word * (num_ddmad_literals * pvr_const_map_entry_literal32)) * * 3. Size of entry for DOUTW vertex/instance control word * (pvr_const_map_entry_literal32) * * 4. Size of DOUTU entry (pvr_const_map_entry_doutu_address) */ const size_t attribute_size = (!robust_buffer_access) ? sizeof(struct pvr_const_map_entry_vertex_attribute_address) : sizeof(struct pvr_const_map_entry_robust_vertex_attribute_address) + sizeof(struct pvr_const_map_entry_vertex_attribute_max_index); /* If has_pds_ddmadt the DDMAD control word is now a DDMADT control word * and is increased by one DWORD to contain the data for the DDMADT's * out-of-bounds check. */ const size_t pvr_pds_const_map_vertex_entry_num_ddmad_literals = 1U + (size_t)PVR_HAS_FEATURE(dev_info, pds_ddmadt); return (sizeof(struct pvr_const_map_entry_base_instance) + PVR_MAX_VERTEX_INPUT_BINDINGS * (attribute_size + (2 + pvr_pds_const_map_vertex_entry_num_ddmad_literals) * sizeof(struct pvr_const_map_entry_literal32)) + sizeof(struct pvr_const_map_entry_literal32) + sizeof(struct pvr_const_map_entry_doutu_address)); } static VkResult pvr_pds_vertex_attrib_program_create_and_upload( struct pvr_device *const device, const VkAllocationCallbacks *const allocator, struct pvr_pds_vertex_primary_program_input *const input, struct pvr_pds_attrib_program *const program_out) { const size_t const_entries_size_in_bytes = pvr_pds_get_max_vertex_program_const_map_size_in_bytes( &device->pdevice->dev_info, device->vk.enabled_features.robustBufferAccess); struct pvr_pds_upload *const program = &program_out->program; struct pvr_pds_info *const info = &program_out->info; struct pvr_const_map_entry *new_entries; ASSERTED uint32_t code_size_in_dwords; size_t staging_buffer_size; uint32_t *staging_buffer; VkResult result; memset(info, 0, sizeof(*info)); info->entries = vk_alloc2(&device->vk.alloc, allocator, const_entries_size_in_bytes, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!info->entries) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_out; } info->entries_size_in_bytes = const_entries_size_in_bytes; pvr_pds_generate_vertex_primary_program( input, NULL, info, device->vk.enabled_features.robustBufferAccess, &device->pdevice->dev_info); code_size_in_dwords = info->code_size_in_dwords; staging_buffer_size = PVR_DW_TO_BYTES(info->code_size_in_dwords); staging_buffer = vk_alloc2(&device->vk.alloc, allocator, staging_buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (!staging_buffer) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_free_entries; } /* This also fills in info->entries. */ pvr_pds_generate_vertex_primary_program( input, staging_buffer, info, device->vk.enabled_features.robustBufferAccess, &device->pdevice->dev_info); assert(info->code_size_in_dwords <= code_size_in_dwords); /* FIXME: Add a vk_realloc2() ? */ new_entries = vk_realloc((!allocator) ? &device->vk.alloc : allocator, info->entries, info->entries_written_size_in_bytes, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!new_entries) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_free_staging_buffer; } info->entries = new_entries; info->entries_size_in_bytes = info->entries_written_size_in_bytes; /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, NULL, 0, 0, staging_buffer, info->code_size_in_dwords, 16, 16, program); if (result != VK_SUCCESS) goto err_free_staging_buffer; vk_free2(&device->vk.alloc, allocator, staging_buffer); return VK_SUCCESS; err_free_staging_buffer: vk_free2(&device->vk.alloc, allocator, staging_buffer); err_free_entries: vk_free2(&device->vk.alloc, allocator, info->entries); err_out: return result; } static inline void pvr_pds_vertex_attrib_program_destroy( struct pvr_device *const device, const struct VkAllocationCallbacks *const allocator, struct pvr_pds_attrib_program *const program) { pvr_bo_suballoc_free(program->program.pvr_bo); vk_free2(&device->vk.alloc, allocator, program->info.entries); } /* This is a const pointer to an array of pvr_pds_attrib_program structs. * The array being pointed to is of PVR_PDS_VERTEX_ATTRIB_PROGRAM_COUNT size. */ typedef struct pvr_pds_attrib_program (*const pvr_pds_attrib_programs_array_ptr) [PVR_PDS_VERTEX_ATTRIB_PROGRAM_COUNT]; /* Generate and uploads a PDS program for DMAing vertex attribs into USC vertex * inputs. This will bake the code segment and create a template of the data * segment for the command buffer to fill in. */ /* If allocator == NULL, the internal one will be used. * * programs_out_ptr is a pointer to the array where the outputs will be placed. */ static VkResult pvr_pds_vertex_attrib_programs_create_and_upload( struct pvr_device *device, const VkAllocationCallbacks *const allocator, pco_data *shader_data, const struct pvr_pds_vertex_dma dma_descriptions[static const PVR_MAX_VERTEX_ATTRIB_DMAS], uint32_t dma_count, pvr_pds_attrib_programs_array_ptr programs_out_ptr) { struct pvr_pds_vertex_primary_program_input input = { .dma_list = dma_descriptions, .dma_count = dma_count, }; uint32_t usc_temp_count = shader_data->common.temps; struct pvr_pds_attrib_program *const programs_out = *programs_out_ptr; VkResult result; pco_range *sys_vals = shader_data->common.sys_vals; if (sys_vals[SYSTEM_VALUE_VERTEX_ID].count > 0) { input.flags |= PVR_PDS_VERTEX_FLAGS_VERTEX_ID_REQUIRED; input.vertex_id_register = sys_vals[SYSTEM_VALUE_VERTEX_ID].start; } if (sys_vals[SYSTEM_VALUE_INSTANCE_ID].count > 0) { input.flags |= PVR_PDS_VERTEX_FLAGS_INSTANCE_ID_REQUIRED; input.instance_id_register = sys_vals[SYSTEM_VALUE_INSTANCE_ID].start; } if (sys_vals[SYSTEM_VALUE_BASE_INSTANCE].count > 0) { input.flags |= PVR_PDS_VERTEX_FLAGS_BASE_INSTANCE_REQUIRED; input.base_instance_register = sys_vals[SYSTEM_VALUE_BASE_INSTANCE].start; } if (sys_vals[SYSTEM_VALUE_BASE_VERTEX].count > 0) { input.flags |= PVR_PDS_VERTEX_FLAGS_BASE_VERTEX_REQUIRED; input.base_vertex_register = sys_vals[SYSTEM_VALUE_BASE_VERTEX].start; } if (sys_vals[SYSTEM_VALUE_DRAW_ID].count > 0) { input.flags |= PVR_PDS_VERTEX_FLAGS_DRAW_INDEX_REQUIRED; input.draw_index_register = sys_vals[SYSTEM_VALUE_DRAW_ID].start; } pvr_pds_setup_doutu(&input.usc_task_control, 0, usc_temp_count, ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE, false); /* Note: programs_out_ptr is a pointer to an array so this is fine. See the * typedef. */ for (uint32_t i = 0; i < ARRAY_SIZE(*programs_out_ptr); i++) { uint32_t extra_flags; switch (i) { case PVR_PDS_VERTEX_ATTRIB_PROGRAM_BASIC: extra_flags = 0; break; case PVR_PDS_VERTEX_ATTRIB_PROGRAM_BASE_INSTANCE: extra_flags = PVR_PDS_VERTEX_FLAGS_BASE_INSTANCE_VARIANT; break; case PVR_PDS_VERTEX_ATTRIB_PROGRAM_DRAW_INDIRECT: extra_flags = PVR_PDS_VERTEX_FLAGS_DRAW_INDIRECT_VARIANT; break; default: UNREACHABLE("Invalid vertex attrib program type."); } input.flags |= extra_flags; result = pvr_pds_vertex_attrib_program_create_and_upload(device, allocator, &input, &programs_out[i]); if (result != VK_SUCCESS) { for (uint32_t j = 0; j < i; j++) { pvr_pds_vertex_attrib_program_destroy(device, allocator, &programs_out[j]); } return result; } input.flags &= ~extra_flags; } return VK_SUCCESS; } size_t pvr_pds_get_max_descriptor_upload_const_map_size_in_bytes(void) { /* Maximum memory allocation needed for const map entries in * pvr_pds_generate_descriptor_upload_program(). * It must be >= 688 bytes. This size is calculated as the sum of: * * 1. Max. number of descriptor sets (8) * ( * size of descriptor entry * (pvr_const_map_entry_descriptor_set) + * size of Common Store burst entry * (pvr_const_map_entry_literal32)) * * 2. Max. number of PDS program buffers (24) * ( * size of the largest buffer structure * (pvr_const_map_entry_constant_buffer) + * size of Common Store burst entry * (pvr_const_map_entry_literal32) * * 3. Size of DOUTU entry (pvr_const_map_entry_doutu_address) */ /* FIXME: PVR_MAX_DESCRIPTOR_SETS is 4 and not 8. The comment above seems to * say that it should be 8. * Figure our a define for this or is the comment wrong? */ return (8 * (sizeof(struct pvr_const_map_entry_descriptor_set) + sizeof(struct pvr_const_map_entry_literal32)) + PVR_PDS_MAX_BUFFERS * (sizeof(struct pvr_const_map_entry_constant_buffer) + sizeof(struct pvr_const_map_entry_literal32)) + sizeof(struct pvr_const_map_entry_doutu_address)); } static VkResult pvr_pds_descriptor_program_create_and_upload( struct pvr_device *const device, const VkAllocationCallbacks *const allocator, const struct vk_pipeline_layout *const layout, gl_shader_stage stage, pco_data *data, struct pvr_stage_allocation_descriptor_state *const descriptor_state) { const size_t const_entries_size_in_bytes = pvr_pds_get_max_descriptor_upload_const_map_size_in_bytes(); struct pvr_pds_info *const pds_info = &descriptor_state->pds_info; struct pvr_pds_descriptor_program_input program = { 0 }; struct pvr_const_map_entry *new_entries; ASSERTED uint32_t code_size_in_dwords; uint32_t staging_buffer_size; uint32_t *staging_buffer; VkResult result; *pds_info = (struct pvr_pds_info){ 0 }; for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) { const struct pvr_descriptor_set_layout *set_layout = vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]); const pco_descriptor_set_data *desc_set_data = &data->common.desc_sets[desc_set]; const pco_range *desc_set_range = &desc_set_data->range; /* If the descriptor set isn't for this stage or is unused, skip it. */ if (!(BITFIELD_BIT(stage) & set_layout->stage_flags)) { assert(!desc_set_data->used); continue; } if (!desc_set_data->used) continue; program.descriptor_sets[program.descriptor_set_count] = (struct pvr_pds_descriptor_set){ .descriptor_set = desc_set, .size_in_dwords = desc_set_range->count, .destination = desc_set_range->start, }; program.descriptor_set_count++; } pds_info->entries = vk_alloc2(&device->vk.alloc, allocator, const_entries_size_in_bytes, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!pds_info->entries) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_free_static_consts; } pds_info->entries_size_in_bytes = const_entries_size_in_bytes; pvr_pds_generate_descriptor_upload_program(&program, NULL, pds_info); code_size_in_dwords = pds_info->code_size_in_dwords; staging_buffer_size = PVR_DW_TO_BYTES(pds_info->code_size_in_dwords); if (!staging_buffer_size) { vk_free2(&device->vk.alloc, allocator, pds_info->entries); *descriptor_state = (struct pvr_stage_allocation_descriptor_state){ 0 }; return VK_SUCCESS; } staging_buffer = vk_alloc2(&device->vk.alloc, allocator, staging_buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (!staging_buffer) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_free_entries; } pvr_pds_generate_descriptor_upload_program(&program, staging_buffer, pds_info); assert(pds_info->code_size_in_dwords <= code_size_in_dwords); /* FIXME: use vk_realloc2() ? */ new_entries = vk_realloc((!allocator) ? &device->vk.alloc : allocator, pds_info->entries, pds_info->entries_written_size_in_bytes, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!new_entries) { result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); goto err_free_staging_buffer; } pds_info->entries = new_entries; pds_info->entries_size_in_bytes = pds_info->entries_written_size_in_bytes; /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, NULL, 0, 0, staging_buffer, pds_info->code_size_in_dwords, 16, 16, &descriptor_state->pds_code); if (result != VK_SUCCESS) goto err_free_staging_buffer; vk_free2(&device->vk.alloc, allocator, staging_buffer); return VK_SUCCESS; err_free_staging_buffer: vk_free2(&device->vk.alloc, allocator, staging_buffer); err_free_entries: vk_free2(&device->vk.alloc, allocator, pds_info->entries); err_free_static_consts: pvr_bo_suballoc_free(descriptor_state->static_consts); return result; } static void pvr_pds_descriptor_program_destroy( struct pvr_device *const device, const struct VkAllocationCallbacks *const allocator, struct pvr_stage_allocation_descriptor_state *const descriptor_state) { if (!descriptor_state) return; pvr_bo_suballoc_free(descriptor_state->pds_code.pvr_bo); vk_free2(&device->vk.alloc, allocator, descriptor_state->pds_info.entries); pvr_bo_suballoc_free(descriptor_state->static_consts); } static void pvr_pds_compute_program_setup( const struct pvr_device_info *dev_info, const uint32_t local_input_regs[static const PVR_WORKGROUP_DIMENSIONS], const uint32_t work_group_input_regs[static const PVR_WORKGROUP_DIMENSIONS], uint32_t barrier_coefficient, bool add_base_workgroup, uint32_t usc_temps, pvr_dev_addr_t usc_shader_dev_addr, struct pvr_pds_compute_shader_program *const program) { pvr_pds_compute_shader_program_init(program); program->local_input_regs[0] = local_input_regs[0]; program->local_input_regs[1] = local_input_regs[1]; program->local_input_regs[2] = local_input_regs[2]; program->work_group_input_regs[0] = work_group_input_regs[0]; program->work_group_input_regs[1] = work_group_input_regs[1]; program->work_group_input_regs[2] = work_group_input_regs[2]; program->barrier_coefficient = barrier_coefficient; program->add_base_workgroup = add_base_workgroup; program->flattened_work_groups = true; program->kick_usc = true; STATIC_ASSERT(ARRAY_SIZE(program->local_input_regs) == PVR_WORKGROUP_DIMENSIONS); STATIC_ASSERT(ARRAY_SIZE(program->work_group_input_regs) == PVR_WORKGROUP_DIMENSIONS); STATIC_ASSERT(ARRAY_SIZE(program->global_input_regs) == PVR_WORKGROUP_DIMENSIONS); pvr_pds_setup_doutu(&program->usc_task_control, usc_shader_dev_addr.addr, usc_temps, ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE, false); pvr_pds_compute_shader(program, NULL, PDS_GENERATE_SIZES, dev_info); } /* FIXME: See if pvr_device_init_compute_pds_program() and this could be merged. */ static VkResult pvr_pds_compute_program_create_and_upload( struct pvr_device *const device, const VkAllocationCallbacks *const allocator, const uint32_t local_input_regs[static const PVR_WORKGROUP_DIMENSIONS], const uint32_t work_group_input_regs[static const PVR_WORKGROUP_DIMENSIONS], uint32_t barrier_coefficient, uint32_t usc_temps, pvr_dev_addr_t usc_shader_dev_addr, struct pvr_pds_upload *const pds_upload_out, struct pvr_pds_info *const pds_info_out) { struct pvr_device_info *dev_info = &device->pdevice->dev_info; struct pvr_pds_compute_shader_program program; uint32_t staging_buffer_size; uint32_t *staging_buffer; VkResult result; pvr_pds_compute_program_setup(dev_info, local_input_regs, work_group_input_regs, barrier_coefficient, false, usc_temps, usc_shader_dev_addr, &program); /* FIXME: According to pvr_device_init_compute_pds_program() the code size * is in bytes. Investigate this. */ staging_buffer_size = PVR_DW_TO_BYTES(program.code_size + program.data_size); staging_buffer = vk_alloc2(&device->vk.alloc, allocator, staging_buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (!staging_buffer) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); /* FIXME: pvr_pds_compute_shader doesn't implement * PDS_GENERATE_CODEDATA_SEGMENTS. */ pvr_pds_compute_shader(&program, &staging_buffer[0], PDS_GENERATE_CODE_SEGMENT, dev_info); pvr_pds_compute_shader(&program, &staging_buffer[program.code_size], PDS_GENERATE_DATA_SEGMENT, dev_info); /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, &staging_buffer[program.code_size], program.data_size, 16, &staging_buffer[0], program.code_size, 16, 16, pds_upload_out); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, staging_buffer); return result; } *pds_info_out = (struct pvr_pds_info){ .temps_required = program.highest_temp, .code_size_in_dwords = program.code_size, .data_size_in_dwords = program.data_size, }; vk_free2(&device->vk.alloc, allocator, staging_buffer); return VK_SUCCESS; }; static void pvr_pds_compute_program_destroy( struct pvr_device *const device, const struct VkAllocationCallbacks *const allocator, struct pvr_pds_upload *const pds_program, struct pvr_pds_info *const pds_info) { /* We don't allocate an entries buffer so we don't need to free it */ pvr_bo_suballoc_free(pds_program->pvr_bo); } /* This only uploads the code segment. The data segment will need to be patched * with the base workgroup before uploading. */ static VkResult pvr_pds_compute_base_workgroup_variant_program_init( struct pvr_device *const device, const VkAllocationCallbacks *const allocator, const uint32_t local_input_regs[static const PVR_WORKGROUP_DIMENSIONS], const uint32_t work_group_input_regs[static const PVR_WORKGROUP_DIMENSIONS], uint32_t barrier_coefficient, uint32_t usc_temps, pvr_dev_addr_t usc_shader_dev_addr, struct pvr_pds_base_workgroup_program *program_out) { struct pvr_device_info *dev_info = &device->pdevice->dev_info; struct pvr_pds_compute_shader_program program; uint32_t buffer_size; uint32_t *buffer; VkResult result; pvr_pds_compute_program_setup(dev_info, local_input_regs, work_group_input_regs, barrier_coefficient, true, usc_temps, usc_shader_dev_addr, &program); /* FIXME: According to pvr_device_init_compute_pds_program() the code size * is in bytes. Investigate this. */ buffer_size = PVR_DW_TO_BYTES(MAX2(program.code_size, program.data_size)); buffer = vk_alloc2(&device->vk.alloc, allocator, buffer_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!buffer) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); pvr_pds_compute_shader(&program, &buffer[0], PDS_GENERATE_CODE_SEGMENT, dev_info); /* FIXME: Figure out the define for alignment of 16. */ result = pvr_gpu_upload_pds(device, NULL, 0, 0, buffer, program.code_size, 16, 16, &program_out->code_upload); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, buffer); return result; } pvr_pds_compute_shader(&program, buffer, PDS_GENERATE_DATA_SEGMENT, dev_info); program_out->data_section = buffer; /* We'll need to patch the base workgroup in the PDS data section before * dispatch so we save the offsets at which to patch. We only need to save * the offset for the first workgroup id since the workgroup ids are stored * contiguously in the data segment. */ program_out->base_workgroup_data_patching_offset = program.base_workgroup_constant_offset_in_dwords[0]; program_out->info = (struct pvr_pds_info){ .temps_required = program.highest_temp, .code_size_in_dwords = program.code_size, .data_size_in_dwords = program.data_size, }; return VK_SUCCESS; } static void pvr_pds_compute_base_workgroup_variant_program_finish( struct pvr_device *device, const VkAllocationCallbacks *const allocator, struct pvr_pds_base_workgroup_program *const state) { pvr_bo_suballoc_free(state->code_upload.pvr_bo); vk_free2(&device->vk.alloc, allocator, state->data_section); } /****************************************************************************** Generic pipeline functions ******************************************************************************/ static void pvr_pipeline_init(struct pvr_device *device, enum pvr_pipeline_type type, const VkPipelineLayout layout, struct pvr_pipeline *const pipeline) { vk_object_base_init(&device->vk, &pipeline->base, VK_OBJECT_TYPE_PIPELINE); pipeline->type = type; assert(!pipeline->layout); pipeline->layout = vk_pipeline_layout_from_handle(layout); vk_pipeline_layout_ref(pipeline->layout); } static void pvr_pipeline_finish(struct pvr_device *device, struct pvr_pipeline *pipeline) { vk_pipeline_layout_unref(&device->vk, pipeline->layout); vk_object_base_finish(&pipeline->base); } /* How many shared regs it takes to store a pvr_dev_addr_t. * Each shared reg is 32 bits. */ #define PVR_DEV_ADDR_SIZE_IN_SH_REGS \ DIV_ROUND_UP(sizeof(pvr_dev_addr_t), sizeof(uint32_t)) /****************************************************************************** Compute pipeline functions ******************************************************************************/ /* Compiles and uploads shaders and PDS programs. */ static VkResult pvr_compute_pipeline_compile( struct pvr_device *const device, struct vk_pipeline_cache *cache, const VkComputePipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *const allocator, struct pvr_compute_pipeline *const compute_pipeline) { struct vk_pipeline_layout *layout = compute_pipeline->base.layout; uint32_t work_group_input_regs[PVR_WORKGROUP_DIMENSIONS]; uint32_t local_input_regs[PVR_WORKGROUP_DIMENSIONS]; uint32_t barrier_coefficient; uint32_t usc_temps; VkResult result; compute_pipeline->shader_state.const_shared_reg_count = 0; /* FIXME: Compile and upload the shader. */ /* FIXME: Initialize the shader state and setup build info. */ UNREACHABLE("finishme: compute support"); result = pvr_pds_descriptor_program_create_and_upload( device, allocator, layout, MESA_SHADER_COMPUTE, NULL, &compute_pipeline->descriptor_state); if (result != VK_SUCCESS) goto err_free_shader; result = pvr_pds_compute_program_create_and_upload( device, allocator, local_input_regs, work_group_input_regs, barrier_coefficient, usc_temps, compute_pipeline->shader_state.bo->dev_addr, &compute_pipeline->primary_program, &compute_pipeline->primary_program_info); if (result != VK_SUCCESS) goto err_free_descriptor_program; /* If the workgroup ID is required, then we require the base workgroup * variant of the PDS compute program as well. */ compute_pipeline->flags.base_workgroup = work_group_input_regs[0] != PVR_PDS_REG_UNUSED || work_group_input_regs[1] != PVR_PDS_REG_UNUSED || work_group_input_regs[2] != PVR_PDS_REG_UNUSED; if (compute_pipeline->flags.base_workgroup) { result = pvr_pds_compute_base_workgroup_variant_program_init( device, allocator, local_input_regs, work_group_input_regs, barrier_coefficient, usc_temps, compute_pipeline->shader_state.bo->dev_addr, &compute_pipeline->primary_base_workgroup_variant_program); if (result != VK_SUCCESS) goto err_destroy_compute_program; } return VK_SUCCESS; err_destroy_compute_program: pvr_pds_compute_program_destroy(device, allocator, &compute_pipeline->primary_program, &compute_pipeline->primary_program_info); err_free_descriptor_program: pvr_pds_descriptor_program_destroy(device, allocator, &compute_pipeline->descriptor_state); err_free_shader: pvr_bo_suballoc_free(compute_pipeline->shader_state.bo); return result; } static VkResult pvr_compute_pipeline_init(struct pvr_device *device, struct vk_pipeline_cache *cache, const VkComputePipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *allocator, struct pvr_compute_pipeline *compute_pipeline) { VkResult result; pvr_pipeline_init(device, PVR_PIPELINE_TYPE_COMPUTE, pCreateInfo->layout, &compute_pipeline->base); result = pvr_compute_pipeline_compile(device, cache, pCreateInfo, allocator, compute_pipeline); if (result != VK_SUCCESS) { pvr_pipeline_finish(device, &compute_pipeline->base); return result; } return VK_SUCCESS; } static VkResult pvr_compute_pipeline_create(struct pvr_device *device, struct vk_pipeline_cache *cache, const VkComputePipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *allocator, VkPipeline *const pipeline_out) { struct pvr_compute_pipeline *compute_pipeline; VkResult result; compute_pipeline = vk_zalloc2(&device->vk.alloc, allocator, sizeof(*compute_pipeline), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!compute_pipeline) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); /* Compiles and uploads shaders and PDS programs. */ result = pvr_compute_pipeline_init(device, cache, pCreateInfo, allocator, compute_pipeline); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, compute_pipeline); return result; } *pipeline_out = pvr_pipeline_to_handle(&compute_pipeline->base); return VK_SUCCESS; } static void pvr_compute_pipeline_destroy( struct pvr_device *const device, const VkAllocationCallbacks *const allocator, struct pvr_compute_pipeline *const compute_pipeline) { if (compute_pipeline->flags.base_workgroup) { pvr_pds_compute_base_workgroup_variant_program_finish( device, allocator, &compute_pipeline->primary_base_workgroup_variant_program); } pvr_pds_compute_program_destroy(device, allocator, &compute_pipeline->primary_program, &compute_pipeline->primary_program_info); pvr_pds_descriptor_program_destroy(device, allocator, &compute_pipeline->descriptor_state); pvr_bo_suballoc_free(compute_pipeline->shader_state.bo); pvr_pipeline_finish(device, &compute_pipeline->base); vk_free2(&device->vk.alloc, allocator, compute_pipeline); } VkResult pvr_CreateComputePipelines(VkDevice _device, VkPipelineCache pipelineCache, uint32_t createInfoCount, const VkComputePipelineCreateInfo *pCreateInfos, const VkAllocationCallbacks *pAllocator, VkPipeline *pPipelines) { VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache); PVR_FROM_HANDLE(pvr_device, device, _device); VkResult result = VK_SUCCESS; for (uint32_t i = 0; i < createInfoCount; i++) { const VkResult local_result = pvr_compute_pipeline_create(device, cache, &pCreateInfos[i], pAllocator, &pPipelines[i]); if (local_result != VK_SUCCESS) { result = local_result; pPipelines[i] = VK_NULL_HANDLE; } } return result; } /****************************************************************************** Graphics pipeline functions ******************************************************************************/ static void pvr_pipeline_destroy_shader_data(pco_data *data) { for (unsigned u = 0; u < ARRAY_SIZE(data->common.desc_sets); ++u) if (data->common.desc_sets[u].bindings) ralloc_free(data->common.desc_sets[u].bindings); } static void pvr_graphics_pipeline_destroy(struct pvr_device *const device, const VkAllocationCallbacks *const allocator, struct pvr_graphics_pipeline *const gfx_pipeline) { const uint32_t num_vertex_attrib_programs = ARRAY_SIZE(gfx_pipeline->shader_state.vertex.pds_attrib_programs); pvr_pds_descriptor_program_destroy( device, allocator, &gfx_pipeline->shader_state.fragment.descriptor_state); pvr_pds_descriptor_program_destroy( device, allocator, &gfx_pipeline->shader_state.vertex.descriptor_state); for (uint32_t i = 0; i < num_vertex_attrib_programs; i++) { struct pvr_pds_attrib_program *const attrib_program = &gfx_pipeline->shader_state.vertex.pds_attrib_programs[i]; pvr_pds_vertex_attrib_program_destroy(device, allocator, attrib_program); } pvr_bo_suballoc_free( gfx_pipeline->shader_state.fragment.pds_fragment_program.pvr_bo); pvr_bo_suballoc_free( gfx_pipeline->shader_state.fragment.pds_coeff_program.pvr_bo); pvr_bo_suballoc_free(gfx_pipeline->shader_state.fragment.bo); pvr_bo_suballoc_free(gfx_pipeline->shader_state.vertex.bo); pvr_pipeline_finish(device, &gfx_pipeline->base); pvr_pipeline_destroy_shader_data(&gfx_pipeline->vs_data); pvr_pipeline_destroy_shader_data(&gfx_pipeline->fs_data); vk_free2(&device->vk.alloc, allocator, gfx_pipeline); } static void pvr_vertex_state_save(struct pvr_graphics_pipeline *gfx_pipeline, pco_shader *vs) { struct pvr_vertex_shader_state *vertex_state = &gfx_pipeline->shader_state.vertex; const pco_data *shader_data = pco_shader_data(vs); memcpy(&gfx_pipeline->vs_data, shader_data, sizeof(*shader_data)); /* This ends up unused since we'll use the temp_usage for the PDS program we * end up selecting, and the descriptor PDS program doesn't use any temps. * Let's set it to ~0 in case it ever gets used. */ vertex_state->stage_state.pds_temps_count = ~0; } static void pvr_fragment_state_save(struct pvr_graphics_pipeline *gfx_pipeline, pco_shader *fs) { struct pvr_fragment_shader_state *fragment_state = &gfx_pipeline->shader_state.fragment; const pco_data *shader_data = pco_shader_data(fs); memcpy(&gfx_pipeline->fs_data, shader_data, sizeof(*shader_data)); /* TODO: add selection for other values of pass type and sample rate. */ fragment_state->pass_type = ROGUE_TA_PASSTYPE_OPAQUE; fragment_state->sample_rate = ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE; /* We can't initialize it yet since we still need to generate the PDS * programs so set it to `~0` to make sure that we set this up later on. */ fragment_state->stage_state.pds_temps_count = ~0; } static bool pvr_blend_factor_requires_consts(VkBlendFactor factor) { switch (factor) { case VK_BLEND_FACTOR_CONSTANT_COLOR: case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR: case VK_BLEND_FACTOR_CONSTANT_ALPHA: case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA: return true; default: return false; } } /** * \brief Indicates whether dynamic blend constants are needed. * * If the user has specified the blend constants to be dynamic, they might not * necessarily be using them. This function makes sure that they are being used * in order to determine whether we need to upload them later on for the shader * to access them. */ static bool pvr_graphics_pipeline_requires_dynamic_blend_consts( const struct pvr_graphics_pipeline *gfx_pipeline) { const struct vk_dynamic_graphics_state *const state = &gfx_pipeline->dynamic_state; if (BITSET_TEST(state->set, MESA_VK_DYNAMIC_CB_BLEND_CONSTANTS)) return false; for (uint32_t i = 0; i < state->cb.attachment_count; i++) { const struct vk_color_blend_attachment_state *attachment = &state->cb.attachments[i]; const bool has_color_write = attachment->write_mask & (VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT); const bool has_alpha_write = attachment->write_mask & VK_COLOR_COMPONENT_A_BIT; if (!attachment->blend_enable || attachment->write_mask == 0) continue; if (has_color_write) { const uint8_t src_color_blend_factor = attachment->src_color_blend_factor; const uint8_t dst_color_blend_factor = attachment->dst_color_blend_factor; if (pvr_blend_factor_requires_consts(src_color_blend_factor) || pvr_blend_factor_requires_consts(dst_color_blend_factor)) { return true; } } if (has_alpha_write) { const uint8_t src_alpha_blend_factor = attachment->src_alpha_blend_factor; const uint8_t dst_alpha_blend_factor = attachment->dst_alpha_blend_factor; if (pvr_blend_factor_requires_consts(src_alpha_blend_factor) || pvr_blend_factor_requires_consts(dst_alpha_blend_factor)) { return true; } } } return false; } #undef PVR_DEV_ADDR_SIZE_IN_SH_REGS static void pvr_graphics_pipeline_setup_vertex_dma( struct pvr_graphics_pipeline *gfx_pipeline, const VkPipelineVertexInputStateCreateInfo *const vertex_input_state, struct pvr_pds_vertex_dma *const dma_descriptions, uint32_t *const dma_count) { pco_vs_data *vs_data = &gfx_pipeline->vs_data.vs; const VkVertexInputBindingDescription *sorted_bindings[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; const VkVertexInputAttributeDescription *sorted_attributes[PVR_MAX_VERTEX_INPUT_BINDINGS] = { 0 }; /* Vertex attributes map to the `layout(location = x)` annotation in the * shader where `x` is the attribute's location. * Vertex bindings have NO relation to the shader. They have nothing to do * with the `layout(set = x, binding = y)` notation. They instead indicate * where the data for a collection of vertex attributes comes from. The * application binds a VkBuffer with vkCmdBindVertexBuffers() to a specific * binding number and based on that we'll know which buffer to DMA the data * from, to fill in the collection of vertex attributes. */ for (uint32_t i = 0; i < vertex_input_state->vertexBindingDescriptionCount; i++) { const VkVertexInputBindingDescription *binding_desc = &vertex_input_state->pVertexBindingDescriptions[i]; sorted_bindings[binding_desc->binding] = binding_desc; } for (uint32_t i = 0; i < vertex_input_state->vertexAttributeDescriptionCount; i++) { const VkVertexInputAttributeDescription *attribute_desc = &vertex_input_state->pVertexAttributeDescriptions[i]; sorted_attributes[attribute_desc->location] = attribute_desc; } for (uint32_t i = 0; i < vertex_input_state->vertexAttributeDescriptionCount; i++) { const VkVertexInputAttributeDescription *attribute = sorted_attributes[i]; if (!attribute) continue; gl_vert_attrib location = attribute->location + VERT_ATTRIB_GENERIC0; const VkVertexInputBindingDescription *binding = sorted_bindings[attribute->binding]; struct pvr_pds_vertex_dma *dma_desc = &dma_descriptions[*dma_count]; const struct util_format_description *fmt_description = vk_format_description(attribute->format); const pco_range *attrib_range = &vs_data->attribs[location]; /* Skip unused attributes. */ if (!attrib_range->count) continue; /* DMA setup. */ /* The PDS program sets up DDMADs to DMA attributes into vtxin regs. * * DDMAD -> Multiply, add, and DOUTD (i.e. DMA from that address). * DMA source addr = src0 * src1 + src2 * DMA params = src3 * * In the PDS program we setup src0 with the binding's stride and src1 * with either the instance id or vertex id (both of which get filled by * the hardware). We setup src2 later on once we know which VkBuffer to * DMA the data from so it's saved for later when we patch the data * section. */ /* TODO: Right now we're setting up a DMA per attribute. In a case where * there are multiple attributes packed into a single binding with * adjacent locations we'd still be DMAing them separately. This is not * great so the DMA setup should be smarter and could do with some * optimization. */ *dma_desc = (struct pvr_pds_vertex_dma){ 0 }; /* In relation to the Vulkan spec. 22.4. Vertex Input Address Calculation * this corresponds to `attribDesc.offset`. * The PDS program doesn't do anything with it but just save it in the * PDS program entry. */ dma_desc->offset = attribute->offset; /* In relation to the Vulkan spec. 22.4. Vertex Input Address Calculation * this corresponds to `bindingDesc.stride`. * The PDS program will calculate the `effectiveVertexOffset` with this * and add it to the address provided in the patched data segment. */ dma_desc->stride = binding->stride; if (binding->inputRate == VK_VERTEX_INPUT_RATE_INSTANCE) dma_desc->flags = PVR_PDS_VERTEX_DMA_FLAGS_INSTANCE_RATE; else dma_desc->flags = 0; /* Size to DMA per vertex attribute. Used to setup src3 in the DDMAD. */ dma_desc->size_in_dwords = attrib_range->count; /* Vtxin reg offset to start DMAing into. */ dma_desc->destination = attrib_range->start; /* Will be used by the driver to figure out buffer address to patch in the * data section. I.e. which binding we should DMA from. */ dma_desc->binding_index = attribute->binding; /* We don't currently support VK_EXT_vertex_attribute_divisor so no * repeating of instance-rate vertex attributes needed. We should always * move on to the next vertex attribute. */ assert(binding->inputRate != VK_VERTEX_INPUT_RATE_INSTANCE); dma_desc->divisor = 1; /* Will be used to generate PDS code that takes care of robust buffer * access, and later on by the driver to write the correct robustness * buffer address to DMA the fallback values from. */ dma_desc->robustness_buffer_offset = pvr_get_robustness_buffer_format_offset(attribute->format); /* Used by later on by the driver to figure out if the buffer is being * accessed out of bounds, for robust buffer access. */ dma_desc->component_size_in_bytes = fmt_description->block.bits / fmt_description->nr_channels / 8; ++*dma_count; } } static void pvr_graphics_pipeline_setup_fragment_coeff_program( struct pvr_graphics_pipeline *gfx_pipeline, nir_shader *fs, struct pvr_pds_coeff_loading_program *frag_coeff_program) { uint64_t varyings_used = fs->info.inputs_read & BITFIELD64_RANGE(VARYING_SLOT_VAR0, MAX_VARYING); pco_vs_data *vs_data = &gfx_pipeline->vs_data.vs; pco_fs_data *fs_data = &gfx_pipeline->fs_data.fs; unsigned fpu = 0; unsigned dest = 0; if (fs_data->uses.z) { pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu], PDSINST_DOUT_FIELDS_DOUTI_SRC, douti_src) { /* TODO: define instead of sizeof(uint16_t). */ douti_src.f32_offset = fs_data->uses.w ? 1 * sizeof(uint16_t) : 0; douti_src.f16_offset = douti_src.f32_offset; douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD; douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D; } frag_coeff_program->destination[fpu++] = dest++; } if (fs_data->uses.w) { pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu], PDSINST_DOUT_FIELDS_DOUTI_SRC, douti_src) { douti_src.f32_offset = 0; douti_src.f16_offset = douti_src.f32_offset; douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD; douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D; } frag_coeff_program->destination[fpu++] = dest++; } if (fs_data->uses.pntc) { pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu], PDSINST_DOUT_FIELDS_DOUTI_SRC, douti_src) { douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD; douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_2D; douti_src.pointsprite = true; } frag_coeff_program->destination[fpu++] = dest; dest += 2; } u_foreach_bit64 (varying, varyings_used) { nir_variable *var = nir_find_variable_with_location(fs, nir_var_shader_in, varying); assert(var); pco_range *cf_range = &fs_data->varyings[varying]; assert(cf_range->count > 0); assert(!(cf_range->start % ROGUE_USC_COEFFICIENT_SET_SIZE)); assert(!(cf_range->count % ROGUE_USC_COEFFICIENT_SET_SIZE)); pco_range *vtxout_range = &vs_data->varyings[varying]; assert(vtxout_range->count > 0); assert(vtxout_range->start >= 4); assert(vtxout_range->count == cf_range->count / ROGUE_USC_COEFFICIENT_SET_SIZE); unsigned count = vtxout_range->count; unsigned vtxout = vtxout_range->start; /* pos.x, pos.y unused. */ vtxout -= 2; /* pos.z unused. */ if (!fs_data->uses.z) vtxout -= 1; /* pos.w unused. */ if (!fs_data->uses.w) vtxout -= 1; pvr_csb_pack (&frag_coeff_program->FPU_iterators[fpu], PDSINST_DOUT_FIELDS_DOUTI_SRC, douti_src) { /* TODO: define instead of sizeof(uint16_t). */ douti_src.f32_offset = vtxout * sizeof(uint16_t); /* TODO: f16 support. */ douti_src.f16 = false; douti_src.f16_offset = douti_src.f32_offset; switch (var->data.interpolation) { case INTERP_MODE_SMOOTH: douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD; douti_src.perspective = true; break; case INTERP_MODE_NOPERSPECTIVE: douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_GOURUAD; break; case INTERP_MODE_FLAT: /* TODO: triangle fan, provoking vertex last. */ douti_src.shademodel = ROGUE_PDSINST_DOUTI_SHADEMODEL_FLAT_VERTEX0; break; default: UNREACHABLE("Unimplemented interpolation type."); } douti_src.size = ROGUE_PDSINST_DOUTI_SIZE_1D + count - 1; } frag_coeff_program->destination[fpu++] = cf_range->start / ROGUE_USC_COEFFICIENT_SET_SIZE; } frag_coeff_program->num_fpu_iterators = fpu; } static void set_var(pco_range *allocation_list, unsigned to, nir_variable *var, unsigned dwords_each) { unsigned slots = glsl_count_dword_slots(var->type, false); allocation_list[var->data.location] = (pco_range){ .start = to, .count = slots * dwords_each, }; } static void allocate_var(pco_range *allocation_list, unsigned *counter, nir_variable *var, unsigned dwords_each) { unsigned slots = glsl_count_dword_slots(var->type, false); allocation_list[var->data.location] = (pco_range){ .start = *counter, .count = slots * dwords_each, }; *counter += slots * dwords_each; } static void try_allocate_var(pco_range *allocation_list, unsigned *counter, nir_shader *nir, uint64_t bitset, nir_variable_mode mode, int location, unsigned dwords_each) { nir_variable *var = nir_find_variable_with_location(nir, mode, location); if (!(bitset & BITFIELD64_BIT(location))) return; assert(var); allocate_var(allocation_list, counter, var, dwords_each); } static void try_allocate_vars(pco_range *allocation_list, unsigned *counter, nir_shader *nir, uint64_t *bitset, nir_variable_mode mode, bool f16, enum glsl_interp_mode interp_mode, unsigned dwords_each) { uint64_t skipped = 0; while (*bitset) { int location = u_bit_scan64(bitset); nir_variable *var = nir_find_variable_with_location(nir, mode, location); assert(var); if (glsl_type_is_16bit(glsl_without_array_or_matrix(var->type)) != f16 || var->data.interpolation != interp_mode) { skipped |= BITFIELD64_BIT(location); continue; } allocate_var(allocation_list, counter, var, dwords_each); } *bitset |= skipped; } static void allocate_val(pco_range *allocation_list, unsigned *counter, unsigned location, unsigned dwords_each) { allocation_list[location] = (pco_range){ .start = *counter, .count = dwords_each, }; *counter += dwords_each; } static void pvr_alloc_vs_sysvals(pco_data *data, nir_shader *nir) { BITSET_DECLARE(system_values_read, SYSTEM_VALUE_MAX); BITSET_COPY(system_values_read, nir->info.system_values_read); gl_system_value sys_vals[] = { SYSTEM_VALUE_VERTEX_ID, SYSTEM_VALUE_INSTANCE_ID, SYSTEM_VALUE_BASE_INSTANCE, SYSTEM_VALUE_BASE_VERTEX, SYSTEM_VALUE_DRAW_ID, }; for (unsigned u = 0; u < ARRAY_SIZE(sys_vals); ++u) { if (BITSET_TEST(system_values_read, sys_vals[u])) { nir_intrinsic_op op = nir_intrinsic_from_system_value(sys_vals[u]); unsigned dwords = nir_intrinsic_infos[op].dest_components; assert(dwords > 0); allocate_val(data->common.sys_vals, &data->common.vtxins, sys_vals[u], dwords); BITSET_CLEAR(system_values_read, sys_vals[u]); } } assert(BITSET_IS_EMPTY(system_values_read)); } static void pvr_init_vs_attribs( pco_data *data, const VkPipelineVertexInputStateCreateInfo *const vertex_input_state) { for (unsigned u = 0; u < vertex_input_state->vertexAttributeDescriptionCount; ++u) { const VkVertexInputAttributeDescription *attrib = &vertex_input_state->pVertexAttributeDescriptions[u]; gl_vert_attrib location = attrib->location + VERT_ATTRIB_GENERIC0; data->vs.attrib_formats[location] = vk_format_to_pipe_format(attrib->format); } } static void pvr_alloc_vs_attribs(pco_data *data, nir_shader *nir) { /* TODO NEXT: this should be based on the format size. */ nir_foreach_shader_in_variable (var, nir) { allocate_var(data->vs.attribs, &data->common.vtxins, var, 1); } } static void pvr_alloc_vs_varyings(pco_data *data, nir_shader *nir) { uint64_t vars_mask = nir->info.outputs_written & BITFIELD64_RANGE(VARYING_SLOT_VAR0, MAX_VARYING); /* Output position must be present. */ assert(nir_find_variable_with_location(nir, nir_var_shader_out, VARYING_SLOT_POS)); /* Varying ordering is specific. */ try_allocate_var(data->vs.varyings, &data->vs.vtxouts, nir, nir->info.outputs_written, nir_var_shader_out, VARYING_SLOT_POS, 1); /* Save varying counts. */ u_foreach_bit64 (location, vars_mask) { nir_variable *var = nir_find_variable_with_location(nir, nir_var_shader_out, location); assert(var); /* TODO: f16 support. */ bool f16 = glsl_type_is_16bit(glsl_without_array_or_matrix(var->type)); assert(!f16); unsigned components = glsl_get_components(var->type); switch (var->data.interpolation) { case INTERP_MODE_SMOOTH: if (f16) data->vs.f16_smooth += components; else data->vs.f32_smooth += components; break; case INTERP_MODE_FLAT: if (f16) data->vs.f16_flat += components; else data->vs.f32_flat += components; break; case INTERP_MODE_NOPERSPECTIVE: if (f16) data->vs.f16_npc += components; else data->vs.f32_npc += components; break; default: UNREACHABLE(""); } } for (unsigned f16 = 0; f16 <= 1; ++f16) { for (enum glsl_interp_mode interp_mode = INTERP_MODE_SMOOTH; interp_mode <= INTERP_MODE_NOPERSPECTIVE; ++interp_mode) { try_allocate_vars(data->vs.varyings, &data->vs.vtxouts, nir, &vars_mask, nir_var_shader_out, f16, interp_mode, 1); } } assert(!vars_mask); const gl_varying_slot last_slots[] = { VARYING_SLOT_PSIZ, VARYING_SLOT_VIEWPORT, VARYING_SLOT_LAYER, }; for (unsigned u = 0; u < ARRAY_SIZE(last_slots); ++u) { try_allocate_var(data->vs.varyings, &data->vs.vtxouts, nir, nir->info.outputs_written, nir_var_shader_out, last_slots[u], 1); } } static void pvr_alloc_fs_sysvals(pco_data *data, nir_shader *nir) { /* TODO */ } static void pvr_alloc_fs_varyings(pco_data *data, nir_shader *nir) { assert(!data->common.coeffs); /* Save the z/w locations. */ unsigned zw_count = !!data->fs.uses.z + !!data->fs.uses.w; allocate_val(data->fs.varyings, &data->common.coeffs, VARYING_SLOT_POS, zw_count * ROGUE_USC_COEFFICIENT_SET_SIZE); /* If point coords are used, they come after z/w (if present). */ nir_variable *var = nir_find_variable_with_location(nir, nir_var_shader_in, VARYING_SLOT_PNTC); if (var) { assert(!var->data.location_frac); unsigned count = glsl_get_components(var->type); assert(count == 2); allocate_var(data->fs.varyings, &data->common.coeffs, var, ROGUE_USC_COEFFICIENT_SET_SIZE); data->fs.uses.pntc = true; } /* Allocate the rest of the input varyings. */ nir_foreach_shader_in_variable (var, nir) { /* Already handled. */ if (var->data.location == VARYING_SLOT_POS || var->data.location == VARYING_SLOT_PNTC) continue; allocate_var(data->fs.varyings, &data->common.coeffs, var, ROGUE_USC_COEFFICIENT_SET_SIZE); } } static void pvr_init_fs_outputs(pco_data *data, const struct pvr_render_pass *pass, const struct pvr_render_subpass *const subpass, const struct pvr_renderpass_hwsetup_subpass *hw_subpass) { for (unsigned u = 0; u < subpass->color_count; ++u) { unsigned idx = subpass->color_attachments[u]; if (idx == VK_ATTACHMENT_UNUSED) continue; gl_frag_result location = FRAG_RESULT_DATA0 + u; VkFormat vk_format = pass->attachments[idx].vk_format; data->fs.output_formats[location] = vk_format_to_pipe_format(vk_format); } /* TODO: z-replicate. */ } static void pvr_setup_fs_outputs(pco_data *data, nir_shader *nir, const struct pvr_render_subpass *const subpass, const struct pvr_renderpass_hwsetup_subpass *hw_subpass) { ASSERTED unsigned num_outputs = hw_subpass->setup.num_render_targets; assert(num_outputs == subpass->color_count); uint64_t outputs_written = nir->info.outputs_written; assert(util_bitcount64(outputs_written) == num_outputs); for (unsigned u = 0; u < subpass->color_count; ++u) { gl_frag_result location = FRAG_RESULT_DATA0 + u; unsigned idx = subpass->color_attachments[u]; const struct usc_mrt_resource *mrt_resource; ASSERTED bool output_reg; enum pipe_format format; unsigned format_bits; nir_variable *var; if (idx == VK_ATTACHMENT_UNUSED) continue; assert(u == idx); /* TODO: not sure if this is true or not... */ mrt_resource = &hw_subpass->setup.mrt_resources[u]; output_reg = mrt_resource->type == USC_MRT_RESOURCE_TYPE_OUTPUT_REG; assert(output_reg); /* TODO: tile buffer support. */ var = nir_find_variable_with_location(nir, nir_var_shader_out, location); assert(var); format = data->fs.output_formats[location]; format_bits = util_format_get_blocksizebits(format); /* TODO: other sized formats. */ assert(!(format_bits % 32)); assert(mrt_resource->intermediate_size == format_bits / 8); set_var(data->fs.outputs, mrt_resource->reg.output_reg, var, format_bits / 32); data->fs.output_reg[location] = output_reg; outputs_written &= ~BITFIELD64_BIT(location); } /* TODO: z-replicate. */ assert(!outputs_written); } static void pvr_init_fs_input_attachments( pco_data *data, const struct pvr_render_subpass *const subpass, const struct pvr_renderpass_hwsetup_subpass *hw_subpass) { pvr_finishme("pvr_init_fs_input_attachments"); } static void pvr_setup_fs_input_attachments( pco_data *data, nir_shader *nir, const struct pvr_render_subpass *const subpass, const struct pvr_renderpass_hwsetup_subpass *hw_subpass) { pvr_finishme("pvr_setup_fs_input_attachments"); } static void pvr_init_descriptors(pco_data *data, nir_shader *nir, struct vk_pipeline_layout *layout) { for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) { const struct pvr_descriptor_set_layout *set_layout = vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]); pco_descriptor_set_data *desc_set_data = &data->common.desc_sets[desc_set]; /* If the descriptor set isn't for this stage, skip it. */ if (!(BITFIELD_BIT(nir->info.stage) & set_layout->stage_flags)) continue; desc_set_data->binding_count = set_layout->binding_count; desc_set_data->bindings = rzalloc_array_size(NULL, sizeof(*desc_set_data->bindings), set_layout->binding_count); } } static void pvr_setup_descriptors(pco_data *data, nir_shader *nir, struct vk_pipeline_layout *layout) { gl_shader_stage stage = nir->info.stage; for (unsigned desc_set = 0; desc_set < layout->set_count; ++desc_set) { const struct pvr_descriptor_set_layout *set_layout = vk_to_pvr_descriptor_set_layout(layout->set_layouts[desc_set]); const unsigned desc_set_size_dw = set_layout->size / sizeof(uint32_t); pco_descriptor_set_data *desc_set_data = &data->common.desc_sets[desc_set]; pco_range *desc_set_range = &desc_set_data->range; assert(!(set_layout->size % sizeof(uint32_t))); /* If the descriptor set isn't for this stage or is unused, skip it. */ if (!(BITFIELD_BIT(stage) & set_layout->stage_flags)) { assert(!desc_set_data->used); continue; } if (!desc_set_data->used) continue; desc_set_range->start = data->common.shareds; desc_set_range->count = desc_set_size_dw; data->common.shareds += desc_set_size_dw; for (unsigned binding = 0; binding < set_layout->binding_count; ++binding) { const struct pvr_descriptor_set_layout_binding *layout_binding = &set_layout->bindings[binding]; pco_binding_data *binding_data = &desc_set_data->bindings[binding]; binding_data->range = (pco_range){ .start = desc_set_range->start + (layout_binding->offset / sizeof(uint32_t)), .count = (layout_binding->stride * layout_binding->descriptor_count) / sizeof(uint32_t), .stride = layout_binding->stride / sizeof(uint32_t), }; } } assert(data->common.shareds < 256); } static void pvr_preprocess_shader_data(pco_data *data, nir_shader *nir, const VkGraphicsPipelineCreateInfo *pCreateInfo, struct vk_pipeline_layout *layout) { switch (nir->info.stage) { case MESA_SHADER_VERTEX: { const VkPipelineVertexInputStateCreateInfo *const vertex_input_state = pCreateInfo->pVertexInputState; pvr_init_vs_attribs(data, vertex_input_state); break; } case MESA_SHADER_FRAGMENT: { PVR_FROM_HANDLE(pvr_render_pass, pass, pCreateInfo->renderPass); const struct pvr_render_subpass *const subpass = &pass->subpasses[pCreateInfo->subpass]; const struct pvr_renderpass_hw_map *subpass_map = &pass->hw_setup->subpass_map[pCreateInfo->subpass]; const struct pvr_renderpass_hwsetup_subpass *hw_subpass = &pass->hw_setup->renders[subpass_map->render] .subpasses[subpass_map->subpass]; pvr_init_fs_outputs(data, pass, subpass, hw_subpass); pvr_init_fs_input_attachments(data, subpass, hw_subpass); /* TODO: push consts, blend consts, dynamic state, etc. */ break; } default: UNREACHABLE(""); } pvr_init_descriptors(data, nir, layout); /* TODO: common things, like large constants being put into shareds. */ } static void pvr_postprocess_shader_data(pco_data *data, nir_shader *nir, const VkGraphicsPipelineCreateInfo *pCreateInfo, struct vk_pipeline_layout *layout) { switch (nir->info.stage) { case MESA_SHADER_VERTEX: { pvr_alloc_vs_sysvals(data, nir); pvr_alloc_vs_attribs(data, nir); pvr_alloc_vs_varyings(data, nir); break; } case MESA_SHADER_FRAGMENT: { PVR_FROM_HANDLE(pvr_render_pass, pass, pCreateInfo->renderPass); const struct pvr_render_subpass *const subpass = &pass->subpasses[pCreateInfo->subpass]; const struct pvr_renderpass_hw_map *subpass_map = &pass->hw_setup->subpass_map[pCreateInfo->subpass]; const struct pvr_renderpass_hwsetup_subpass *hw_subpass = &pass->hw_setup->renders[subpass_map->render] .subpasses[subpass_map->subpass]; pvr_alloc_fs_sysvals(data, nir); pvr_alloc_fs_varyings(data, nir); pvr_setup_fs_outputs(data, nir, subpass, hw_subpass); pvr_setup_fs_input_attachments(data, nir, subpass, hw_subpass); /* TODO: push consts, blend consts, dynamic state, etc. */ break; } default: UNREACHABLE(""); } pvr_setup_descriptors(data, nir, layout); /* TODO: common things, like large constants being put into shareds. */ } /* Compiles and uploads shaders and PDS programs. */ static VkResult pvr_graphics_pipeline_compile(struct pvr_device *const device, struct vk_pipeline_cache *cache, const VkGraphicsPipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *const allocator, struct pvr_graphics_pipeline *const gfx_pipeline) { struct vk_pipeline_layout *layout = gfx_pipeline->base.layout; const uint32_t cache_line_size = rogue_get_slc_cache_line_size(&device->pdevice->dev_info); VkResult result; struct pvr_vertex_shader_state *vertex_state = &gfx_pipeline->shader_state.vertex; struct pvr_fragment_shader_state *fragment_state = &gfx_pipeline->shader_state.fragment; pco_ctx *pco_ctx = device->pdevice->pco_ctx; nir_shader *producer = NULL; nir_shader *consumer = NULL; pco_data shader_data[MESA_SHADER_STAGES] = { 0 }; nir_shader *nir_shaders[MESA_SHADER_STAGES] = { 0 }; pco_shader *pco_shaders[MESA_SHADER_STAGES] = { 0 }; pco_shader **vs = &pco_shaders[MESA_SHADER_VERTEX]; pco_shader **fs = &pco_shaders[MESA_SHADER_FRAGMENT]; void *shader_mem_ctx = ralloc_context(NULL); struct pvr_pds_vertex_dma vtx_dma_descriptions[PVR_MAX_VERTEX_ATTRIB_DMAS]; uint32_t vtx_dma_count = 0; struct pvr_pds_coeff_loading_program frag_coeff_program = { 0 }; for (gl_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) { size_t stage_index = gfx_pipeline->stage_indices[stage]; /* Skip unused/inactive stages. */ if (stage_index == ~0) continue; result = vk_pipeline_shader_stage_to_nir(&device->vk, gfx_pipeline->base.pipeline_flags, &pCreateInfo->pStages[stage_index], pco_spirv_options(), pco_nir_options(), shader_mem_ctx, &nir_shaders[stage]); if (result != VK_SUCCESS) goto err_free_build_context; pco_preprocess_nir(pco_ctx, nir_shaders[stage]); } for (gl_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) { if (!nir_shaders[stage]) continue; if (producer) pco_link_nir(pco_ctx, producer, nir_shaders[stage]); producer = nir_shaders[stage]; } for (gl_shader_stage stage = MESA_SHADER_STAGES; stage-- > 0;) { if (!nir_shaders[stage]) continue; if (consumer) pco_rev_link_nir(pco_ctx, nir_shaders[stage], consumer); consumer = nir_shaders[stage]; } for (gl_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) { if (!nir_shaders[stage]) continue; pvr_preprocess_shader_data(&shader_data[stage], nir_shaders[stage], pCreateInfo, layout); pco_lower_nir(pco_ctx, nir_shaders[stage], &shader_data[stage]); pco_postprocess_nir(pco_ctx, nir_shaders[stage], &shader_data[stage]); pvr_postprocess_shader_data(&shader_data[stage], nir_shaders[stage], pCreateInfo, layout); } for (gl_shader_stage stage = 0; stage < MESA_SHADER_STAGES; ++stage) { pco_shader **pco = &pco_shaders[stage]; /* Skip unused/inactive stages. */ if (!nir_shaders[stage]) continue; *pco = pco_trans_nir(pco_ctx, nir_shaders[stage], &shader_data[stage], shader_mem_ctx); if (!*pco) { result = VK_ERROR_INITIALIZATION_FAILED; goto err_free_build_context; } pco_process_ir(pco_ctx, *pco); pco_encode_ir(pco_ctx, *pco); } pvr_vertex_state_save(gfx_pipeline, *vs); pvr_graphics_pipeline_setup_vertex_dma(gfx_pipeline, pCreateInfo->pVertexInputState, vtx_dma_descriptions, &vtx_dma_count); result = pvr_gpu_upload_usc(device, pco_shader_binary_data(*vs), pco_shader_binary_size(*vs), cache_line_size, &vertex_state->bo); if (result != VK_SUCCESS) goto err_free_build_context; if (*fs) { pvr_fragment_state_save(gfx_pipeline, *fs); pvr_graphics_pipeline_setup_fragment_coeff_program( gfx_pipeline, nir_shaders[MESA_SHADER_FRAGMENT], &frag_coeff_program); result = pvr_gpu_upload_usc(device, pco_shader_binary_data(*fs), pco_shader_binary_size(*fs), cache_line_size, &fragment_state->bo); if (result != VK_SUCCESS) goto err_free_vertex_bo; result = pvr_pds_coeff_program_create_and_upload(device, allocator, &frag_coeff_program, fragment_state); if (result != VK_SUCCESS) goto err_free_fragment_bo; result = pvr_pds_fragment_program_create_and_upload(device, allocator, *fs, fragment_state); if (result != VK_SUCCESS) goto err_free_coeff_program; result = pvr_pds_descriptor_program_create_and_upload( device, allocator, layout, MESA_SHADER_FRAGMENT, &gfx_pipeline->fs_data, &fragment_state->descriptor_state); if (result != VK_SUCCESS) goto err_free_frag_program; /* If not, we need to MAX2() and set * `fragment_state->stage_state.pds_temps_count` appropriately. */ assert(fragment_state->descriptor_state.pds_info.temps_required == 0); } result = pvr_pds_vertex_attrib_programs_create_and_upload( device, allocator, &gfx_pipeline->vs_data, vtx_dma_descriptions, vtx_dma_count, &vertex_state->pds_attrib_programs); if (result != VK_SUCCESS) goto err_free_frag_descriptor_program; result = pvr_pds_descriptor_program_create_and_upload( device, allocator, layout, MESA_SHADER_VERTEX, &gfx_pipeline->vs_data, &vertex_state->descriptor_state); if (result != VK_SUCCESS) goto err_free_vertex_attrib_program; /* FIXME: When the temp_buffer_total_size is non-zero we need to allocate a * scratch buffer for both vertex and fragment stage. * Figure out the best place to do this. */ /* assert(pvr_pds_descriptor_program_variables.temp_buff_total_size == 0); */ /* TODO: Implement spilling with the above. */ ralloc_free(shader_mem_ctx); return VK_SUCCESS; err_free_vertex_attrib_program: for (uint32_t i = 0; i < ARRAY_SIZE(vertex_state->pds_attrib_programs); i++) { struct pvr_pds_attrib_program *const attrib_program = &vertex_state->pds_attrib_programs[i]; pvr_pds_vertex_attrib_program_destroy(device, allocator, attrib_program); } err_free_frag_descriptor_program: pvr_pds_descriptor_program_destroy(device, allocator, &fragment_state->descriptor_state); err_free_frag_program: pvr_bo_suballoc_free(fragment_state->pds_fragment_program.pvr_bo); err_free_coeff_program: pvr_bo_suballoc_free(fragment_state->pds_coeff_program.pvr_bo); err_free_fragment_bo: pvr_bo_suballoc_free(fragment_state->bo); err_free_vertex_bo: pvr_bo_suballoc_free(vertex_state->bo); err_free_build_context: ralloc_free(shader_mem_ctx); return result; } static struct vk_render_pass_state pvr_create_renderpass_state(const VkGraphicsPipelineCreateInfo *const info) { PVR_FROM_HANDLE(pvr_render_pass, pass, info->renderPass); const struct pvr_render_subpass *const subpass = &pass->subpasses[info->subpass]; enum vk_rp_attachment_flags attachments = 0; assert(info->subpass < pass->subpass_count); for (uint32_t i = 0; i < subpass->color_count; i++) { if (pass->attachments[subpass->color_attachments[i]].aspects) attachments |= MESA_VK_RP_ATTACHMENT_COLOR_0_BIT << i; } if (subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED) { VkImageAspectFlags ds_aspects = pass->attachments[subpass->depth_stencil_attachment].aspects; if (ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) attachments |= MESA_VK_RP_ATTACHMENT_DEPTH_BIT; if (ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT) attachments |= MESA_VK_RP_ATTACHMENT_STENCIL_BIT; } return (struct vk_render_pass_state){ .attachments = attachments, /* TODO: This is only needed for VK_KHR_create_renderpass2 (or core 1.2), * which is not currently supported. */ .view_mask = 0, }; } static VkResult pvr_graphics_pipeline_init(struct pvr_device *device, struct vk_pipeline_cache *cache, const VkGraphicsPipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *allocator, struct pvr_graphics_pipeline *gfx_pipeline) { struct vk_dynamic_graphics_state *const dynamic_state = &gfx_pipeline->dynamic_state; const struct vk_render_pass_state rp_state = pvr_create_renderpass_state(pCreateInfo); struct vk_graphics_pipeline_all_state all_state; struct vk_graphics_pipeline_state state = { 0 }; VkResult result; pvr_pipeline_init(device, PVR_PIPELINE_TYPE_GRAPHICS, pCreateInfo->layout, &gfx_pipeline->base); result = vk_graphics_pipeline_state_fill(&device->vk, &state, pCreateInfo, &rp_state, 0, &all_state, NULL, 0, NULL); if (result != VK_SUCCESS) goto err_pipeline_finish; vk_dynamic_graphics_state_init(dynamic_state); /* Load static state into base dynamic state holder. */ vk_dynamic_graphics_state_fill(dynamic_state, &state); /* The value of ms.rasterization_samples is undefined when * rasterizer_discard_enable is set, but we need a specific value. * Fill that in here. */ if (state.rs->rasterizer_discard_enable) dynamic_state->ms.rasterization_samples = VK_SAMPLE_COUNT_1_BIT; memset(gfx_pipeline->stage_indices, ~0, sizeof(gfx_pipeline->stage_indices)); for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) { VkShaderStageFlagBits vk_stage = pCreateInfo->pStages[i].stage; gl_shader_stage gl_stage = vk_to_mesa_shader_stage(vk_stage); /* From the Vulkan 1.2.192 spec for VkPipelineShaderStageCreateInfo: * * "stage must not be VK_SHADER_STAGE_ALL_GRAPHICS, * or VK_SHADER_STAGE_ALL." * * So we don't handle that. * * We also don't handle VK_SHADER_STAGE_TESSELLATION_* and * VK_SHADER_STAGE_GEOMETRY_BIT stages as 'tessellationShader' and * 'geometryShader' are set to false in the VkPhysicalDeviceFeatures * structure returned by the driver. */ switch (pCreateInfo->pStages[i].stage) { case VK_SHADER_STAGE_VERTEX_BIT: case VK_SHADER_STAGE_FRAGMENT_BIT: gfx_pipeline->stage_indices[gl_stage] = i; break; default: UNREACHABLE("Unsupported stage."); } } /* Compiles and uploads shaders and PDS programs. */ result = pvr_graphics_pipeline_compile(device, cache, pCreateInfo, allocator, gfx_pipeline); if (result != VK_SUCCESS) goto err_pipeline_finish; return VK_SUCCESS; err_pipeline_finish: pvr_pipeline_finish(device, &gfx_pipeline->base); return result; } /* If allocator == NULL, the internal one will be used. */ static VkResult pvr_graphics_pipeline_create(struct pvr_device *device, struct vk_pipeline_cache *cache, const VkGraphicsPipelineCreateInfo *pCreateInfo, const VkAllocationCallbacks *allocator, VkPipeline *const pipeline_out) { struct pvr_graphics_pipeline *gfx_pipeline; VkResult result; gfx_pipeline = vk_zalloc2(&device->vk.alloc, allocator, sizeof(*gfx_pipeline), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); if (!gfx_pipeline) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); /* Compiles and uploads shaders and PDS programs too. */ result = pvr_graphics_pipeline_init(device, cache, pCreateInfo, allocator, gfx_pipeline); if (result != VK_SUCCESS) { vk_free2(&device->vk.alloc, allocator, gfx_pipeline); return result; } *pipeline_out = pvr_pipeline_to_handle(&gfx_pipeline->base); return VK_SUCCESS; } VkResult pvr_CreateGraphicsPipelines(VkDevice _device, VkPipelineCache pipelineCache, uint32_t createInfoCount, const VkGraphicsPipelineCreateInfo *pCreateInfos, const VkAllocationCallbacks *pAllocator, VkPipeline *pPipelines) { VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache); PVR_FROM_HANDLE(pvr_device, device, _device); VkResult result = VK_SUCCESS; for (uint32_t i = 0; i < createInfoCount; i++) { const VkResult local_result = pvr_graphics_pipeline_create(device, cache, &pCreateInfos[i], pAllocator, &pPipelines[i]); if (local_result != VK_SUCCESS) { result = local_result; pPipelines[i] = VK_NULL_HANDLE; } } return result; } /***************************************************************************** Other functions *****************************************************************************/ void pvr_DestroyPipeline(VkDevice _device, VkPipeline _pipeline, const VkAllocationCallbacks *pAllocator) { PVR_FROM_HANDLE(pvr_pipeline, pipeline, _pipeline); PVR_FROM_HANDLE(pvr_device, device, _device); if (!pipeline) return; switch (pipeline->type) { case PVR_PIPELINE_TYPE_GRAPHICS: { struct pvr_graphics_pipeline *const gfx_pipeline = to_pvr_graphics_pipeline(pipeline); pvr_graphics_pipeline_destroy(device, pAllocator, gfx_pipeline); break; } case PVR_PIPELINE_TYPE_COMPUTE: { struct pvr_compute_pipeline *const compute_pipeline = to_pvr_compute_pipeline(pipeline); pvr_compute_pipeline_destroy(device, pAllocator, compute_pipeline); break; } default: UNREACHABLE("Unknown pipeline type."); } }