/* * Copyright 2023 Alyssa Rosenzweig * Copyright 2023 Valve Corporation * SPDX-License-Identifier: MIT */ #include "asahi/lib/agx_abi.h" #include "compiler/libcl/libcl_vk.h" #include "util/macros.h" #include "util/u_math.h" #include "geometry.h" #include "libagx_intrinsics.h" #include "query.h" #include "tessellator.h" /* Swap the two non-provoking vertices in odd triangles. This generates a vertex * ID list with a consistent winding order. * * Holding prim and flatshade_first constant, the map : [0, 1, 2] -> [0, 1, 2] * is its own inverse. It is hence used both vertex fetch and transform * feedback. */ uint libagx_map_vertex_in_tri_strip(uint prim, uint vert, bool flatshade_first) { unsigned pv = flatshade_first ? 0 : 2; bool even = (prim & 1) == 0; bool provoking = vert == pv; return (provoking || even) ? vert : ((3 - pv) - vert); } static inline uint xfb_prim(uint id, uint n, uint copy) { return sub_sat(id, n - 1u) + copy; } /* * Determine whether an output vertex has an n'th copy in the transform feedback * buffer. This is written weirdly to let constant folding remove unnecessary * stores when length is known statically. */ bool libagx_xfb_vertex_copy_in_strip(uint n, uint id, uint length, uint copy) { uint prim = xfb_prim(id, n, copy); int num_prims = length - (n - 1); return copy == 0 || (prim < num_prims && id >= copy && copy < num_prims); } uint libagx_xfb_vertex_offset(uint n, uint invocation_base_prim, uint strip_base_prim, uint id_in_strip, uint copy, bool flatshade_first) { uint prim = xfb_prim(id_in_strip, n, copy); uint vert_0 = min(id_in_strip, n - 1); uint vert = vert_0 - copy; if (n == 3) { vert = libagx_map_vertex_in_tri_strip(prim, vert, flatshade_first); } /* Tally up in the whole buffer */ uint base_prim = invocation_base_prim + strip_base_prim; uint base_vertex = base_prim * n; return base_vertex + (prim * n) + vert; } uint64_t libagx_xfb_vertex_address(constant struct agx_geometry_params *p, uint index, uint buffer, uint stride, uint output_offset) { uint xfb_offset = (index * stride) + output_offset; return (uintptr_t)(p->xfb_base[buffer]) + xfb_offset; } uint libagx_vertex_id_for_line_loop(uint prim, uint vert, uint num_prims) { /* (0, 1), (1, 2), (2, 0) */ if (prim == (num_prims - 1) && vert == 1) return 0; else return prim + vert; } uint libagx_vertex_id_for_line_class(enum mesa_prim mode, uint prim, uint vert, uint num_prims) { /* Line list, line strip, or line loop */ if (mode == MESA_PRIM_LINE_LOOP && prim == (num_prims - 1) && vert == 1) return 0; if (mode == MESA_PRIM_LINES) prim *= 2; return prim + vert; } uint libagx_vertex_id_for_tri_fan(uint prim, uint vert, bool flatshade_first) { /* Vulkan spec section 20.1.7 gives (i + 1, i + 2, 0) for a provoking * first. OpenGL instead wants (0, i + 1, i + 2) with a provoking last. * Piglit clipflat expects us to switch between these orders depending on * provoking vertex, to avoid trivializing the fan. * * Rotate accordingly. */ if (flatshade_first) { vert = (vert == 2) ? 0 : (vert + 1); } /* The simpler form assuming last is provoking. */ return (vert == 0) ? 0 : prim + vert; } uint libagx_vertex_id_for_tri_class(enum mesa_prim mode, uint prim, uint vert, bool flatshade_first) { if (flatshade_first && mode == MESA_PRIM_TRIANGLE_FAN) { vert = vert + 1; vert = (vert == 3) ? 0 : vert; } if (mode == MESA_PRIM_TRIANGLE_FAN && vert == 0) return 0; if (mode == MESA_PRIM_TRIANGLES) prim *= 3; /* Triangle list, triangle strip, or triangle fan */ if (mode == MESA_PRIM_TRIANGLE_STRIP) { unsigned pv = flatshade_first ? 0 : 2; bool even = (prim & 1) == 0; bool provoking = vert == pv; vert = ((provoking || even) ? vert : ((3 - pv) - vert)); } return prim + vert; } uint libagx_vertex_id_for_line_adj_class(enum mesa_prim mode, uint prim, uint vert) { /* Line list adj or line strip adj */ if (mode == MESA_PRIM_LINES_ADJACENCY) prim *= 4; return prim + vert; } uint libagx_vertex_id_for_tri_strip_adj(uint prim, uint vert, uint num_prims, bool flatshade_first) { /* See Vulkan spec section 20.1.11 "Triangle Strips With Adjancency". * * There are different cases for first/middle/last/only primitives and for * odd/even primitives. Determine which case we're in. */ bool last = prim == (num_prims - 1); bool first = prim == 0; bool even = (prim & 1) == 0; bool even_or_first = even || first; /* When the last vertex is provoking, we rotate the primitives * accordingly. This seems required for OpenGL. */ if (!flatshade_first && !even_or_first) { vert = (vert + 4u) % 6u; } /* Offsets per the spec. The spec lists 6 cases with 6 offsets. Luckily, * there are lots of patterns we can exploit, avoiding a full 6x6 LUT. * * Here we assume the first vertex is provoking, the Vulkan default. */ uint offsets[6] = { 0, first ? 1 : (even ? -2 : 3), even_or_first ? 2 : 4, last ? 5 : 6, even_or_first ? 4 : 2, even_or_first ? 3 : -2, }; /* Ensure NIR can see thru the local array */ uint offset = 0; for (uint i = 1; i < 6; ++i) { if (i == vert) offset = offsets[i]; } /* Finally add to the base of the primitive */ return (prim * 2) + offset; } uint libagx_vertex_id_for_tri_adj_class(enum mesa_prim mode, uint prim, uint vert, uint nr, bool flatshade_first) { /* Tri adj list or tri adj strip */ if (mode == MESA_PRIM_TRIANGLE_STRIP_ADJACENCY) { return libagx_vertex_id_for_tri_strip_adj(prim, vert, nr, flatshade_first); } else { return (6 * prim) + vert; } } static uint vertex_id_for_topology(enum mesa_prim mode, bool flatshade_first, uint prim, uint vert, uint num_prims) { switch (mode) { case MESA_PRIM_POINTS: case MESA_PRIM_LINES: case MESA_PRIM_TRIANGLES: case MESA_PRIM_LINES_ADJACENCY: case MESA_PRIM_TRIANGLES_ADJACENCY: /* Regular primitive: every N vertices defines a primitive */ return (prim * mesa_vertices_per_prim(mode)) + vert; case MESA_PRIM_LINE_LOOP: return libagx_vertex_id_for_line_loop(prim, vert, num_prims); case MESA_PRIM_LINE_STRIP: case MESA_PRIM_LINE_STRIP_ADJACENCY: /* (i, i + 1) or (i, ..., i + 3) */ return prim + vert; case MESA_PRIM_TRIANGLE_STRIP: { /* Order depends on the provoking vert. * * First: (0, 1, 2), (1, 3, 2), (2, 3, 4). * Last: (0, 1, 2), (2, 1, 3), (2, 3, 4). * * Pull the (maybe swapped) vert from the corresponding primitive */ return prim + libagx_map_vertex_in_tri_strip(prim, vert, flatshade_first); } case MESA_PRIM_TRIANGLE_FAN: return libagx_vertex_id_for_tri_fan(prim, vert, flatshade_first); case MESA_PRIM_TRIANGLE_STRIP_ADJACENCY: return libagx_vertex_id_for_tri_strip_adj(prim, vert, num_prims, flatshade_first); default: return 0; } } uint libagx_map_to_line_adj(uint id) { /* Sequence (1, 2), (5, 6), (9, 10), ... */ return ((id & ~1) * 2) + (id & 1) + 1; } uint libagx_map_to_line_strip_adj(uint id) { /* Sequence (1, 2), (2, 3), (4, 5), .. */ uint prim = id / 2; uint vert = id & 1; return prim + vert + 1; } uint libagx_map_to_tri_strip_adj(uint id) { /* Sequence (0, 2, 4), (2, 6, 4), (4, 6, 8), (6, 10, 8) * * Although tri strips with adjacency have 6 cases in general, after * disregarding the vertices only available in a geometry shader, there are * only even/odd cases. In other words, it's just a triangle strip subject to * extra padding. * * Dividing through by two, the sequence is: * * (0, 1, 2), (1, 3, 2), (2, 3, 4), (3, 5, 4) */ uint prim = id / 3; uint vtx = id % 3; /* Flip the winding order of odd triangles */ if ((prim % 2) == 1) { if (vtx == 1) vtx = 2; else if (vtx == 2) vtx = 1; } return 2 * (prim + vtx); } static void store_index(uintptr_t index_buffer, uint index_size_B, uint id, uint value) { global uint32_t *out_32 = (global uint32_t *)index_buffer; global uint16_t *out_16 = (global uint16_t *)index_buffer; global uint8_t *out_8 = (global uint8_t *)index_buffer; if (index_size_B == 4) out_32[id] = value; else if (index_size_B == 2) out_16[id] = value; else out_8[id] = value; } static uint load_index(uintptr_t index_buffer, uint32_t index_buffer_range_el, uint id, uint index_size) { bool oob = id >= index_buffer_range_el; /* If the load would be out-of-bounds, load the first element which is * assumed valid. If the application index buffer is empty with robustness2, * index_buffer will point to a zero sink where only the first is valid. */ if (oob) { id = 0; } uint el; if (index_size == 1) { el = ((constant uint8_t *)index_buffer)[id]; } else if (index_size == 2) { el = ((constant uint16_t *)index_buffer)[id]; } else { el = ((constant uint32_t *)index_buffer)[id]; } /* D3D robustness semantics. TODO: Optimize? */ if (oob) { el = 0; } return el; } uint libagx_load_index_buffer(constant struct agx_ia_state *p, uint id, uint index_size) { return load_index(p->index_buffer, p->index_buffer_range_el, id, index_size); } static void increment_counters(global uint32_t *a, global uint32_t *b, global uint32_t *c, uint count) { global uint32_t *ptr[] = {a, b, c}; for (uint i = 0; i < 3; ++i) { if (ptr[i]) { *(ptr[i]) += count; } } } static unsigned decomposed_prims_for_vertices_with_tess(enum mesa_prim prim, int vertices, unsigned verts_per_patch) { if (prim >= MESA_PRIM_PATCHES) { return vertices / verts_per_patch; } else { return u_decomposed_prims_for_vertices(prim, vertices); } } KERNEL(1) libagx_increment_ia(global uint32_t *ia_vertices, global uint32_t *ia_primitives, global uint32_t *vs_invocations, global uint32_t *c_prims, global uint32_t *c_invs, constant uint32_t *draw, enum mesa_prim prim, unsigned verts_per_patch) { increment_counters(ia_vertices, vs_invocations, NULL, draw[0] * draw[1]); uint prims = decomposed_prims_for_vertices_with_tess(prim, draw[0], verts_per_patch) * draw[1]; increment_counters(ia_primitives, c_prims, c_invs, prims); } KERNEL(1024) libagx_increment_ia_restart(global uint32_t *ia_vertices, global uint32_t *ia_primitives, global uint32_t *vs_invocations, global uint32_t *c_prims, global uint32_t *c_invs, constant uint32_t *draw, uint64_t index_buffer, uint32_t index_buffer_range_el, uint32_t restart_index, uint32_t index_size_B, enum mesa_prim prim, unsigned verts_per_patch) { uint tid = cl_global_id.x; unsigned count = draw[0]; local uint scratch; uint start = draw[2]; uint partial = 0; /* Count non-restart indices */ for (uint i = tid; i < count; i += 1024) { uint index = load_index(index_buffer, index_buffer_range_el, start + i, index_size_B); if (index != restart_index) partial++; } /* Accumulate the partials across the workgroup */ scratch = 0; barrier(CLK_LOCAL_MEM_FENCE); atomic_add(&scratch, partial); barrier(CLK_LOCAL_MEM_FENCE); /* Elect a single thread from the workgroup to increment the counters */ if (tid == 0) { increment_counters(ia_vertices, vs_invocations, NULL, scratch * draw[1]); } /* TODO: We should vectorize this */ if ((ia_primitives || c_prims || c_invs) && tid == 0) { uint accum = 0; int last_restart = -1; for (uint i = 0; i < count; ++i) { uint index = load_index(index_buffer, index_buffer_range_el, start + i, index_size_B); if (index == restart_index) { accum += decomposed_prims_for_vertices_with_tess( prim, i - last_restart - 1, verts_per_patch); last_restart = i; } } { accum += decomposed_prims_for_vertices_with_tess( prim, count - last_restart - 1, verts_per_patch); } increment_counters(ia_primitives, c_prims, c_invs, accum * draw[1]); } } /* * Return the ID of the first thread in the workgroup where cond is true, or * 1024 if cond is false across the workgroup. */ static uint first_true_thread_in_workgroup(bool cond, local uint *scratch) { barrier(CLK_LOCAL_MEM_FENCE); scratch[get_sub_group_id()] = sub_group_ballot(cond)[0]; barrier(CLK_LOCAL_MEM_FENCE); uint first_group = ctz(sub_group_ballot(scratch[get_sub_group_local_id()])[0]); uint off = ctz(first_group < 32 ? scratch[first_group] : 0); return (first_group * 32) + off; } /* * When unrolling the index buffer for a draw, we translate the old indirect * draws to new indirect draws. This routine allocates the new index buffer and * sets up most of the new draw descriptor. */ static global void * setup_unroll_for_draw(global struct agx_heap *heap, constant uint *in_draw, global uint *out, enum mesa_prim mode, uint index_size_B) { /* Determine an upper bound on the memory required for the index buffer. * Restarts only decrease the unrolled index buffer size, so the maximum size * is the unrolled size when the input has no restarts. */ uint max_prims = u_decomposed_prims_for_vertices(mode, in_draw[0]); uint max_verts = max_prims * mesa_vertices_per_prim(mode); uint alloc_size = max_verts * index_size_B; /* Allocate unrolled index buffer. * * TODO: For multidraw, should be atomic. But multidraw+unroll isn't * currently wired up in any driver. */ uint old_heap_bottom_B = agx_heap_alloc_nonatomic_offs(heap, alloc_size); /* Setup most of the descriptor. Count will be determined after unroll. */ out[1] = in_draw[1]; /* instance count */ out[2] = old_heap_bottom_B / index_size_B; /* index offset */ out[3] = in_draw[3]; /* index bias */ out[4] = in_draw[4]; /* base instance */ /* Return the index buffer we allocated */ return (global uchar *)heap->base + old_heap_bottom_B; } KERNEL(1024) libagx_unroll_restart(global struct agx_heap *heap, uint64_t index_buffer, constant uint *in_draw, global uint32_t *out_draw, uint32_t max_draws, uint32_t restart_index, uint32_t index_buffer_size_el, uint32_t index_size_log2, uint32_t flatshade_first, uint mode__11) { uint32_t index_size_B = 1 << index_size_log2; enum mesa_prim mode = libagx_uncompact_prim(mode__11); uint tid = cl_local_id.x; uint count = in_draw[0]; local uintptr_t out_ptr; if (tid == 0) { out_ptr = (uintptr_t)setup_unroll_for_draw(heap, in_draw, out_draw, mode, index_size_B); } barrier(CLK_LOCAL_MEM_FENCE); uintptr_t in_ptr = (uintptr_t)(libagx_index_buffer( index_buffer, index_buffer_size_el, in_draw[2], index_size_B)); local uint scratch[32]; uint out_prims = 0; uint needle = 0; uint per_prim = mesa_vertices_per_prim(mode); while (needle < count) { /* Search for next restart or the end. Lanes load in parallel. */ uint next_restart = needle; for (;;) { uint idx = next_restart + tid; bool restart = idx >= count || load_index(in_ptr, index_buffer_size_el, idx, index_size_B) == restart_index; uint next_offs = first_true_thread_in_workgroup(restart, scratch); next_restart += next_offs; if (next_offs < 1024) break; } /* Emit up to the next restart. Lanes output in parallel */ uint subcount = next_restart - needle; uint subprims = u_decomposed_prims_for_vertices(mode, subcount); uint out_prims_base = out_prims; for (uint i = tid; i < subprims; i += 1024) { for (uint vtx = 0; vtx < per_prim; ++vtx) { uint id = vertex_id_for_topology(mode, flatshade_first, i, vtx, subprims); uint offset = needle + id; uint x = ((out_prims_base + i) * per_prim) + vtx; uint y = load_index(in_ptr, index_buffer_size_el, offset, index_size_B); store_index(out_ptr, index_size_B, x, y); } } out_prims += subprims; needle = next_restart + 1; } if (tid == 0) out_draw[0] = out_prims * per_prim; } uint libagx_setup_xfb_buffer(global struct agx_geometry_params *p, uint i, uint stride, uint max_output_end, uint vertices_per_prim) { uint xfb_offset = *(p->xfb_offs_ptrs[i]); p->xfb_base[i] = p->xfb_base_original[i] + xfb_offset; /* Let output_end = output_offset + output_size. * * Primitive P will write up to (but not including) offset: * * xfb_offset + ((P - 1) * (verts_per_prim * stride)) * + ((verts_per_prim - 1) * stride) * + output_end * * To fit all outputs for P, that value must be less than the XFB * buffer size for the output with maximal output_end, as everything * else is constant here across outputs within a buffer/primitive: * * floor(P) <= (stride + size - xfb_offset - output_end) * // (stride * verts_per_prim) */ int numer_s = p->xfb_size[i] + (stride - max_output_end) - xfb_offset; uint numer = max(numer_s, 0); return numer / (stride * vertices_per_prim); } void libagx_write_strip(GLOBAL uint32_t *index_buffer, uint32_t inv_index_offset, uint32_t prim_index_offset, uint32_t vertex_offset, uint32_t verts_in_prim, uint3 info) { _libagx_write_strip(index_buffer, inv_index_offset + prim_index_offset, vertex_offset, verts_in_prim, info.x, info.y, info.z); } void libagx_pad_index_gs(global int *index_buffer, uint inv_index_offset, uint nr_indices, uint alloc) { for (uint i = nr_indices; i < alloc; ++i) { index_buffer[inv_index_offset + i] = -1; } } KERNEL(1) libagx_gs_setup_indirect( uint64_t index_buffer, constant uint *draw, global uintptr_t *vertex_buffer /* output */, global struct agx_ia_state *ia /* output */, global struct agx_geometry_params *p /* output */, global struct agx_heap *heap, uint64_t vs_outputs /* Vertex (TES) output mask */, uint32_t index_size_B /* 0 if no index bffer */, uint32_t index_buffer_range_el, uint32_t prim /* Input primitive type, enum mesa_prim */, int is_prefix_summing, uint max_indices, enum agx_gs_shape shape) { /* Determine the (primitives, instances) grid size. */ uint vertex_count = draw[0]; uint instance_count = draw[1]; ia->verts_per_instance = vertex_count; /* Calculate number of primitives input into the GS */ uint prim_per_instance = u_decomposed_prims_for_vertices(prim, vertex_count); p->input_primitives = prim_per_instance * instance_count; /* Invoke VS as (vertices, instances); GS as (primitives, instances) */ p->vs_grid[0] = vertex_count; p->vs_grid[1] = instance_count; p->gs_grid[0] = prim_per_instance; p->gs_grid[1] = instance_count; p->primitives_log2 = util_logbase2_ceil(prim_per_instance); /* If indexing is enabled, the third word is the offset into the index buffer * in elements. Apply that offset now that we have it. For a hardware * indirect draw, the hardware would do this for us, but for software input * assembly we need to do it ourselves. */ if (index_size_B) { ia->index_buffer = libagx_index_buffer( index_buffer, index_buffer_range_el, draw[2], index_size_B); ia->index_buffer_range_el = libagx_index_buffer_range_el(index_buffer_range_el, draw[2]); } /* We need to allocate VS and GS count buffers, do so now */ uint vertex_buffer_size = libagx_tcs_in_size(vertex_count * instance_count, vs_outputs); if (is_prefix_summing) { p->count_buffer = agx_heap_alloc_nonatomic( heap, p->input_primitives * p->count_buffer_stride); } p->input_buffer = (uintptr_t)agx_heap_alloc_nonatomic(heap, vertex_buffer_size); *vertex_buffer = p->input_buffer; p->input_mask = vs_outputs; /* Allocate the index buffer and write the draw consuming it */ global VkDrawIndexedIndirectCommand *cmd = (global void *)p->indirect_desc; *cmd = (VkDrawIndexedIndirectCommand){ .indexCount = agx_gs_rast_vertices(shape, max_indices, prim_per_instance, instance_count), .instanceCount = agx_gs_rast_instances(shape, max_indices, prim_per_instance, instance_count), }; if (shape == AGX_GS_SHAPE_DYNAMIC_INDEXED) { cmd->firstIndex = agx_heap_alloc_nonatomic_offs(heap, cmd->indexCount * 4) / 4; p->output_index_buffer = (global uint *)(heap->base + (cmd->firstIndex * 4)); } } /* * Returns (work_group_scan_inclusive_add(x), work_group_sum(x)). Implemented * manually with subgroup ops and local memory since Mesa doesn't do those * lowerings yet. */ static uint2 libagx_work_group_scan_inclusive_add(uint x, local uint *scratch) { uint sg_id = get_sub_group_id(); /* Partial prefix sum of the subgroup */ uint sg = sub_group_scan_inclusive_add(x); /* Reduction (sum) for the subgroup */ uint sg_sum = sub_group_broadcast(sg, 31); /* Write out all the subgroups sums */ barrier(CLK_LOCAL_MEM_FENCE); scratch[sg_id] = sg_sum; barrier(CLK_LOCAL_MEM_FENCE); /* Read all the subgroup sums. Thread T in subgroup G reads the sum of all * threads in subgroup T. */ uint other_sum = scratch[get_sub_group_local_id()]; /* Exclusive sum the subgroup sums to get the total before the current group, * which can be added to the total for the current group. */ uint other_sums = sub_group_scan_exclusive_add(other_sum); uint base = sub_group_broadcast(other_sums, sg_id); uint prefix = base + sg; /* Reduce the workgroup using the prefix sum we already did */ uint reduction = sub_group_broadcast(other_sums + other_sum, 31); return (uint2)(prefix, reduction); } KERNEL(1024) _libagx_prefix_sum(global uint *buffer, uint len, uint words, uint word) { local uint scratch[32]; uint tid = cl_local_id.x; /* Main loop: complete workgroups processing 1024 values at once */ uint i, count = 0; uint len_remainder = len % 1024; uint len_rounded_down = len - len_remainder; for (i = tid; i < len_rounded_down; i += 1024) { global uint *ptr = &buffer[(i * words) + word]; uint value = *ptr; uint2 sums = libagx_work_group_scan_inclusive_add(value, scratch); *ptr = count + sums[0]; count += sums[1]; } /* The last iteration is special since we won't have a full subgroup unless * the length is divisible by the subgroup size, and we don't advance count. */ global uint *ptr = &buffer[(i * words) + word]; uint value = (tid < len_remainder) ? *ptr : 0; uint scan = libagx_work_group_scan_inclusive_add(value, scratch)[0]; if (tid < len_remainder) { *ptr = count + scan; } } KERNEL(1024) libagx_prefix_sum_geom(constant struct agx_geometry_params *p) { _libagx_prefix_sum(p->count_buffer, p->input_primitives, p->count_buffer_stride / 4, cl_group_id.x); } KERNEL(1024) libagx_prefix_sum_tess(global struct libagx_tess_args *p, global uint *c_prims, global uint *c_invs, uint increment_stats__2) { _libagx_prefix_sum(p->counts, p->nr_patches, 1 /* words */, 0 /* word */); /* After prefix summing, we know the total # of indices, so allocate the * index buffer now. Elect a thread for the allocation. */ barrier(CLK_LOCAL_MEM_FENCE); if (cl_local_id.x != 0) return; /* The last element of an inclusive prefix sum is the total sum */ uint total = p->nr_patches > 0 ? p->counts[p->nr_patches - 1] : 0; /* Allocate 4-byte indices */ uint32_t elsize_B = sizeof(uint32_t); uint32_t size_B = total * elsize_B; uint alloc_B = agx_heap_alloc_nonatomic_offs(p->heap, size_B); p->index_buffer = (global uint32_t *)(((uintptr_t)p->heap->base) + alloc_B); /* ...and now we can generate the API indexed draw */ global uint32_t *desc = p->out_draws; desc[0] = total; /* count */ desc[1] = 1; /* instance_count */ desc[2] = alloc_B / elsize_B; /* start */ desc[3] = 0; /* index_bias */ desc[4] = 0; /* start_instance */ /* If necessary, increment clipper statistics too. This is only used when * there's no geometry shader following us. See agx_nir_lower_gs.c for more * info on the emulation. We just need to calculate the # of primitives * tessellated. */ if (increment_stats__2) { uint prims = p->points_mode ? total : p->isolines ? (total / 2) : (total / 3); increment_counters(c_prims, c_invs, NULL, prims); } } uintptr_t libagx_vertex_output_address(uintptr_t buffer, uint64_t mask, uint vtx, gl_varying_slot location) { /* Written like this to let address arithmetic work */ return buffer + ((uintptr_t)libagx_tcs_in_offs_el(vtx, location, mask)) * 16; } uintptr_t libagx_geometry_input_address(constant struct agx_geometry_params *p, uint vtx, gl_varying_slot location) { return libagx_vertex_output_address(p->input_buffer, p->input_mask, vtx, location); } unsigned libagx_input_vertices(constant struct agx_ia_state *ia) { return ia->verts_per_instance; } global uint * libagx_load_xfb_count_address(constant struct agx_geometry_params *p, int index, int count_words, uint unrolled_id) { return &p->count_buffer[(unrolled_id * count_words) + index]; } uint libagx_previous_xfb_primitives(global struct agx_geometry_params *p, int static_count, int count_index, int count_words, bool prefix_sum, uint unrolled_id) { if (static_count >= 0) { /* If the number of outputted vertices per invocation is known statically, * we can calculate the base. */ return unrolled_id * static_count; } else { /* Otherwise, load from the count buffer buffer. Note that the sums are * inclusive, so index 0 is nonzero. This requires a little fixup here. We * use a saturating unsigned subtraction so we don't read out-of-bounds. * * If we didn't prefix sum, there's only one element. */ uint prim_minus_1 = prefix_sum ? sub_sat(unrolled_id, 1u) : 0; uint count = p->count_buffer[(prim_minus_1 * count_words) + count_index]; return unrolled_id == 0 ? 0 : count; } } /* Like u_foreach_bit, specialized for XFB to enable loop unrolling */ #define libagx_foreach_xfb(word, index) \ for (uint i = 0; i < 4; ++i) \ if (word & BITFIELD_BIT(i)) void libagx_pre_gs(global struct agx_geometry_params *p, uint streams, uint buffers_written, uint4 buffer_to_stream, int4 count_index, uint4 stride, uint4 output_end, int4 static_count, uint invocations, uint vertices_per_prim, global uint *gs_invocations, global uint *gs_primitives, global uint *c_primitives, global uint *c_invocations) { unsigned count_words = !!(count_index[0] >= 0) + !!(count_index[1] >= 0) + !!(count_index[2] >= 0) + !!(count_index[3] >= 0); bool prefix_sum = count_words && buffers_written; uint unrolled_in_prims = p->input_primitives; /* Determine the number of primitives generated in each stream */ uint4 in_prims = 0; libagx_foreach_xfb(streams, i) { in_prims[i] = libagx_previous_xfb_primitives( p, static_count[i], count_index[i], count_words, prefix_sum, unrolled_in_prims); *(p->prims_generated_counter[i]) += in_prims[i]; } uint4 prims = in_prims; uint emitted_prims = prims[0] + prims[1] + prims[2] + prims[3]; if (buffers_written) { libagx_foreach_xfb(buffers_written, i) { uint max_prims = libagx_setup_xfb_buffer( p, i, stride[i], output_end[i], vertices_per_prim); unsigned stream = buffer_to_stream[i]; prims[stream] = min(prims[stream], max_prims); } int4 overflow = prims < in_prims; libagx_foreach_xfb(streams, i) { p->xfb_verts[i] = prims[i] * vertices_per_prim; *(p->xfb_overflow[i]) += (bool)overflow[i]; *(p->xfb_prims_generated_counter[i]) += prims[i]; } *(p->xfb_any_overflow) += any(overflow); /* Update XFB counters */ libagx_foreach_xfb(buffers_written, i) { uint32_t prim_stride_B = stride[i] * vertices_per_prim; unsigned stream = buffer_to_stream[i]; global uint *ptr = p->xfb_offs_ptrs[i]; if ((uintptr_t)ptr == AGX_ZERO_PAGE_ADDRESS) { ptr = (global uint *)AGX_SCRATCH_PAGE_ADDRESS; } *ptr += prims[stream] * prim_stride_B; } } /* The geometry shader is invoked once per primitive (after unrolling * primitive restart). From the spec: * * In case of instanced geometry shaders (see section 11.3.4.2) the * geometry shader invocations count is incremented for each separate * instanced invocation. */ *gs_invocations += unrolled_in_prims * invocations; *gs_primitives += emitted_prims; /* Clipper queries are not well-defined, so we can emulate them in lots of * silly ways. We need the hardware counters to implement them properly. For * now, just consider all primitives emitted as passing through the clipper. * This satisfies spec text: * * The number of primitives that reach the primitive clipping stage. * * and * * If at least one vertex of the primitive lies inside the clipping * volume, the counter is incremented by one or more. Otherwise, the * counter is incremented by zero or more. */ *c_primitives += emitted_prims; *c_invocations += emitted_prims; }