diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index 8b297b4830ff..3d6e811a6ed5 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1648,7 +1648,7 @@ def store(name, srcs, indices=[], flags=[]): # without the binding because the hardware expects a single flattened index # rather than a (binding, index) pair. We may also want to use this with GL. # Note that this doesn't actually turn into a HW instruction. -intrinsic("bindless_resource_ir3", [1], dest_comp=1, indices=[DESC_SET], flags=[CAN_ELIMINATE, CAN_REORDER]) +intrinsic("bindless_resource_ir3", [1], dest_comp=1, indices=[DESC_SET, ACCESS], flags=[CAN_ELIMINATE, CAN_REORDER]) # IR3-specific intrinsics for shader preamble. These are meant to be used like # this: @@ -1680,12 +1680,12 @@ def store(name, srcs, indices=[], flags=[]): # IR3-specific intrinsic for ldc.k. Copies UBO to constant file. # base is the const file base in components, range is the amount to copy in # vec4's. -intrinsic("copy_ubo_to_uniform_ir3", [1, 1], indices=[BASE, RANGE]) +intrinsic("copy_ubo_to_uniform_ir3", [1, 1], indices=[ACCESS, BASE, RANGE]) # IR3-specific intrinsic for ldg.k. # base is an offset to apply to the address in bytes, range_base is the # const file base in components, range is the amount to copy in vec4's. -intrinsic("copy_global_to_uniform_ir3", [1], indices=[BASE, RANGE_BASE, RANGE]) +intrinsic("copy_global_to_uniform_ir3", [1], indices=[ACCESS, BASE, RANGE_BASE, RANGE]) # IR3-specific intrinsic for stsc. Loads from push consts to constant file # Should be used in the shader preamble. @@ -1710,9 +1710,9 @@ def store(name, srcs, indices=[], flags=[]): bit_sizes=src0, flags=[CAN_ELIMINATE]) # IR3-specific intrinsics for prefetching descriptors in preambles. -intrinsic("prefetch_sam_ir3", [1, 1], flags=[CAN_REORDER]) -intrinsic("prefetch_tex_ir3", [1], flags=[CAN_REORDER]) -intrinsic("prefetch_ubo_ir3", [1], flags=[CAN_REORDER]) +intrinsic("prefetch_sam_ir3", [1, 1], indices=[ACCESS], flags=[CAN_REORDER]) +intrinsic("prefetch_tex_ir3", [1], indices=[ACCESS], flags=[CAN_REORDER]) +intrinsic("prefetch_ubo_ir3", [1], indices=[ACCESS], flags=[CAN_REORDER]) intrinsic("resbase_ir3", src_comp=[1], dest_comp=2, flags=[CAN_ELIMINATE, CAN_REORDER]) diff --git a/src/freedreno/common/freedreno_dev_info.h b/src/freedreno/common/freedreno_dev_info.h index 4302b8a4bb98..bd6f80ac395b 100644 --- a/src/freedreno/common/freedreno_dev_info.h +++ b/src/freedreno/common/freedreno_dev_info.h @@ -474,20 +474,21 @@ struct fd_dev_info { * expected: */ bool has_salu_int_narrowing_quirk; - /* Whether the device supports the image processing opcode */ bool has_image_processing; - /* The amount of valid draw state IDs. */ uint32_t max_draw_states; - /* Whether GRAS_CL_INTERP_CNTL has FACENESS/CENTERRHW and thus * being able to avoid setting ij_linear_sample for FragFace/FragCoord. */ bool has_implicit_fragface_fragcoord_ij_linear; - uint32_t max_texel_buffer_range_elements; uint32_t max_storage_buffer_range_bytes; + /* If GMEM needs to be disabled for this GPU */ + bool disable_gmem; + + /* GMEM size in bytes */ + uint32_t gmem_size; } props; }; diff --git a/src/freedreno/common/freedreno_devices.py b/src/freedreno/common/freedreno_devices.py index 408bd0b09647..652cd358cc1d 100644 --- a/src/freedreno/common/freedreno_devices.py +++ b/src/freedreno/common/freedreno_devices.py @@ -40,7 +40,7 @@ tile_max_w = 992, # max_bitfield_val(4, 0, 5) tile_max_h = max_bitfield_val(9, 5, 5), num_vsc_pipes = 8, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, num_sp_cores = 0, # TODO wave_granularity = 2, fibers_per_sp = 0, # TODO @@ -58,7 +58,7 @@ tile_max_w = 1024, # max_bitfield_val(4, 0, 5) tile_max_h = max_bitfield_val(9, 5, 5), num_vsc_pipes = 8, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, num_sp_cores = 0, # TODO wave_granularity = 2, fibers_per_sp = 0, # TODO @@ -77,7 +77,7 @@ tile_max_w = 1024, # max_bitfield_val(7, 0, 5) tile_max_h = max_bitfield_val(16, 9, 5), num_vsc_pipes = 16, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, num_sp_cores = 1, wave_granularity = 2, fibers_per_sp = 64 * 16, # Lowest number that didn't fault on spillall fs-varying-array-mat4-col-row-rd. @@ -95,7 +95,7 @@ tile_max_w = 1024, # max_bitfield_val(7, 0, 5) tile_max_h = max_bitfield_val(16, 9, 5), num_vsc_pipes = 16, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, num_sp_cores = 2, wave_granularity = 2, fibers_per_sp = 64 * 16, # Lowest number that didn't fault on spillall fs-varying-array-mat4-col-row-rd. @@ -113,7 +113,7 @@ tile_max_w = 1024, # max_bitfield_val(7, 0, 5) tile_max_h = max_bitfield_val(16, 9, 5), num_vsc_pipes = 16, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, num_sp_cores = 4, wave_granularity = 2, fibers_per_sp = 64 * 16, # Lowest number that didn't fault on spillall fs-varying-array-mat4-col-row-rd. @@ -287,7 +287,7 @@ GPUId(605), # TODO: Test it, based only on libwrapfake dumps GPUId(610), GPUId(612), # TODO: Test it, based only on libwrapfake dumps - GPUId(613), + GPUId(613), # Snapdragon 4 gen 2 unofficial support ], A6xxGPUInfo( CHIP.A6XX, [a6xx_base, a6xx_gen1_low], @@ -343,7 +343,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 16, highest_bank_bit = 14, @@ -378,7 +378,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 16, magic_regs = dict( @@ -412,7 +412,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict( @@ -446,7 +446,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 16, highest_bank_bit = 15, @@ -481,7 +481,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 4 * 16, highest_bank_bit = 15, @@ -516,7 +516,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 4 * 16, highest_bank_bit = 15, @@ -551,7 +551,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -592,7 +592,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 14, @@ -626,7 +626,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -661,7 +661,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 4 * 16, magic_regs = dict( @@ -695,7 +695,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -988,6 +988,53 @@ [A6XXRegs.REG_A6XX_UCHE_UNKNOWN_0E12, 0], ] + +# Adreno 710/720 are not supported by the upstream, but some hacks float on the internet adding their support. +# These hacks simply reuse A730 entry with different ids and looks like it works in some extent +# Let's do the same in our patchset +add_gpus([ + GPUId(chip_id=0x07010000, name="FD710"), # KGSL, no speedbin data + GPUId(chip_id=0xffff07010000, name="FD710"), # Default no-speedbin fallback + ], A6xxGPUInfo( + CHIP.A7XX, + [a7xx_base, a7xx_gen1], + num_ccu = 4, + tile_align_w = 64, + tile_align_h = 32, + tile_max_w = 1024, + tile_max_h = 1024, + num_vsc_pipes = 32, + cs_shared_mem_size = 64 * 1024, + wave_granularity = 2, + fibers_per_sp = 128 * 2 * 16, + highest_bank_bit = 16, + magic_regs = a730_magic_regs, + raw_magic_regs = a730_raw_magic_regs, + )) + +# Adreno 720 +add_gpus([ + GPUId(chip_id=0x43020000, name="FD720"), # KGSL, no speedbin data + GPUId(chip_id=0xffff43020000, name="FD720"), # Default no-speedbin fallback + ], A6xxGPUInfo( + CHIP.A7XX, + [a7xx_base, a7xx_gen1], + num_ccu = 4, + tile_align_w = 64, + tile_align_h = 32, + tile_max_w = 1024, + tile_max_h = 1024, + num_vsc_pipes = 32, + cs_shared_mem_size = 64 * 1024, + wave_granularity = 2, + fibers_per_sp = 128 * 2 * 16, + highest_bank_bit = 16, + magic_regs = a730_magic_regs, + raw_magic_regs = a730_raw_magic_regs, + )) + + + add_gpus([ # These are named as Adreno730v3 or Adreno725v1. GPUId(chip_id=0x07030002, name="FD725"), @@ -1001,7 +1048,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -1021,7 +1068,7 @@ tile_max_w = 1024, tile_max_h = 1024, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -1104,7 +1151,7 @@ tile_max_w = 2016, tile_max_h = 2032, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict( @@ -1170,7 +1217,7 @@ tile_max_w = 2016, tile_max_h = 2032, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -1191,7 +1238,7 @@ tile_max_w = 2016, tile_max_h = 2032, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = a740_magic_regs, @@ -1251,7 +1298,7 @@ tile_max_w = 2016, tile_max_h = 2032, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict( @@ -1274,7 +1321,7 @@ tile_max_w = 2016, tile_max_h = 2032, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, highest_bank_bit = 16, @@ -1333,7 +1380,7 @@ has_dp2acc = False, reg_size_vec4 = 96, has_rt_workaround = False, - supports_double_threadsize = False, + supports_double_threadsize = True, has_dual_wave_dispatch = True, round_robin_errata = False, max_texel_buffer_range_elements = (1 << 29) - 1, @@ -1418,21 +1465,37 @@ ], A6xxGPUInfo( CHIP.A8XX, [a7xx_base, a7xx_gen3, a8xx_base, a8xx_gen1, GPUProps( + sysmem_vpc_attr_buf_size = 131072, + sysmem_vpc_pos_buf_size = 65536, + sysmem_vpc_bv_pos_buf_size = 32768, + sysmem_ccu_color_cache_fraction = CCUColorCacheFraction.FULL.value, + sysmem_per_ccu_color_cache_size = 64 * 1024, + sysmem_ccu_depth_cache_fraction = CCUColorCacheFraction.THREE_QUARTER.value, + sysmem_per_ccu_depth_cache_size = 64 * 1024, + gmem_ccu_color_cache_fraction = CCUColorCacheFraction.EIGHTH.value, + gmem_per_ccu_color_cache_size = 32 * 1024, + gmem_ccu_depth_cache_fraction = CCUColorCacheFraction.FULL.value, + gmem_per_ccu_depth_cache_size = 48 * 1024, + gmem_vpc_attr_buf_size = 16384, gmem_vpc_pos_buf_size = 12288, gmem_vpc_bv_pos_buf_size = 20480, - # This is possibly also needed for a830 (and all of a8xx), - # move to a8xx_base if confirmed needed for a830. + + gmem_size = 576 * 1024, + has_ray_intersection = False, + has_sw_fuse = False, has_fs_tex_prefetch = False, + has_salu_int_narrowing_quirk = True, + shading_rate_matches_vk = True, )], num_ccu = 1, num_slices = 1, - tile_align_w = 32, - tile_align_h = 16, + tile_align_w = 64, + tile_align_h = 32, tile_max_w = 16384, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), @@ -1452,7 +1515,45 @@ tile_max_w = 16416, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, + wave_granularity = 2, + fibers_per_sp = 128 * 2 * 16, + magic_regs = dict(), + raw_magic_regs = a8xx_base_raw_magic_regs, + )) + +# gen8_6_0 +add_gpus([ + GPUId(chip_id=0x44030000, name="Adreno (TM) 825"), + ], A6xxGPUInfo( + CHIP.A8XX, + [a7xx_base, a7xx_gen3, a8xx_base, a8xx_gen1, GPUProps( + gmem_ccu_color_cache_fraction = CCUColorCacheFraction.HALF.value, + gmem_per_ccu_color_cache_size = 128 * 1024, + gmem_ccu_depth_cache_fraction = CCUColorCacheFraction.HALF.value, + gmem_per_ccu_depth_cache_size = 128 * 1024, + + sysmem_ccu_color_cache_fraction = CCUColorCacheFraction.FULL.value, + sysmem_per_ccu_color_cache_size = 128 * 1024, + sysmem_ccu_depth_cache_fraction = CCUColorCacheFraction.THREE_QUARTER.value, + sysmem_per_ccu_depth_cache_size = 96 * 1024, + + gmem_vpc_attr_buf_size = 49152, + gmem_vpc_pos_buf_size = 24576, + gmem_vpc_bv_pos_buf_size = 32768, + + disable_gmem = False, + gmem_size = 2 * 1024 * 1024, + shading_rate_matches_vk = True, + )], + num_ccu = 4, + num_slices = 2, + tile_align_w = 64, + tile_align_h = 32, + tile_max_w = 16416, + tile_max_h = 16384, + num_vsc_pipes = 32, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), @@ -1465,8 +1566,26 @@ CHIP.A8XX, [a7xx_base, a7xx_gen3, a8xx_base, a8xx_gen2, GPUProps( - shading_rate_matches_vk = True, # TODO confirm this - sysmem_vpc_bv_pos_buf_size = 24576, + sysmem_vpc_bv_pos_buf_size = 24576, + sysmem_ccu_color_cache_fraction = CCUColorCacheFraction.FULL.value, + sysmem_per_ccu_color_cache_size = 128 * 1024, + sysmem_ccu_depth_cache_fraction = CCUColorCacheFraction.HALF.value, + sysmem_per_ccu_depth_cache_size = 128 * 1024, + + gmem_vpc_attr_buf_size = 49152, + gmem_vpc_pos_buf_size = 24576, + gmem_vpc_bv_pos_buf_size = 32768, + + gmem_ccu_color_cache_fraction = CCUColorCacheFraction.HALF.value, + gmem_per_ccu_color_cache_size = 128 * 1024, + gmem_ccu_depth_cache_fraction = CCUColorCacheFraction.HALF.value, + gmem_per_ccu_depth_cache_size = 128 * 1024, + + has_fs_tex_prefetch = False, + has_salu_int_narrowing_quirk = True, + shading_rate_matches_vk = True, + gmem_size = 2 * 1024 * 1024, + enable_tp_ubwc_flag_hint = True, )], num_ccu = 4, num_slices = 2, @@ -1475,7 +1594,7 @@ tile_max_w = 16384, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), @@ -1520,7 +1639,7 @@ tile_max_w = 16416, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), @@ -1539,7 +1658,7 @@ tile_max_w = 16416, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), @@ -1558,7 +1677,7 @@ tile_max_w = 16384, tile_max_h = 16384, num_vsc_pipes = 32, - cs_shared_mem_size = 32 * 1024, + cs_shared_mem_size = 64 * 1024, wave_granularity = 2, fibers_per_sp = 128 * 2 * 16, magic_regs = dict(), diff --git a/src/freedreno/drm-shim/freedreno_noop.c b/src/freedreno/drm-shim/freedreno_noop.c index 010145229fc5..21362629e4aa 100644 --- a/src/freedreno/drm-shim/freedreno_noop.c +++ b/src/freedreno/drm-shim/freedreno_noop.c @@ -327,6 +327,21 @@ static const struct msm_device_info device_infos[] = { .chip_id = 0x44050000, .gmem_size = 12 * 1024 * 1024, }, + { + .gpu_id = 825, + .chip_id = 0x44030000, + .gmem_size = 2 * 1024 * 1024, + }, + { + .gpu_id = 829, + .chip_id = 0x44030A20, + .gmem_size = 2 * 1024 * 1024, + }, + { + .gpu_id = 810, + .chip_id = 0x44010000, + .gmem_size = 576 * 1024, + } }; static void diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h index 8d6170f2cbb8..3514e395988d 100644 --- a/src/freedreno/ir3/ir3.h +++ b/src/freedreno/ir3/ir3.h @@ -2299,7 +2299,8 @@ void ir3_ra_predicates(struct ir3_shader_variant *v); bool ir3_lower_subgroups(struct ir3 *ir); /* legalize: */ -bool ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary); +bool ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary, + bool is_preamble_speculatable); bool ir3_legalize_relative(struct ir3 *ir); static inline bool diff --git a/src/freedreno/ir3/ir3_compiler.c b/src/freedreno/ir3/ir3_compiler.c index df7f68d8f3bc..24a935e3ad4c 100644 --- a/src/freedreno/ir3/ir3_compiler.c +++ b/src/freedreno/ir3/ir3_compiler.c @@ -57,6 +57,34 @@ DEBUG_GET_ONCE_OPTION(ir3_shader_override_path, "IR3_SHADER_OVERRIDE_PATH", enum ir3_shader_debug ir3_shader_debug = 0; const char *ir3_shader_override_path = NULL; +struct ir3_gpu_profile +ir3_get_gpu_profile(uint32_t chip_id) +{ + switch (chip_id) { + case 0x44010000: /* Adreno 810 */ + return (struct ir3_gpu_profile){90, 4, 4, false}; + case 0x44030000: /* Adreno 825 */ + return (struct ir3_gpu_profile){85, 8, 8, true}; + case 0x44030A20: /* Adreno 829 */ + return (struct ir3_gpu_profile){80, 10, 8, true}; + case 0x44050001: /* Adreno 830 */ + return (struct ir3_gpu_profile){75, 16, 12, true}; + case 0x43050A31: /* Adreno 830 variant */ + return (struct ir3_gpu_profile){75, 16, 12, true}; + case 0x43050A32: /* Adreno 840 */ + return (struct ir3_gpu_profile){70, 20, 16, true}; + default: + return (struct ir3_gpu_profile){85, 8, 8, false}; + } +} + +uint32_t +ir3_effective_reg_size(struct ir3_compiler *compiler) +{ + struct ir3_gpu_profile profile = ir3_get_gpu_profile(compiler->dev_id->chip_id); + return compiler->reg_size_vec4 * profile.reg_efficiency / 100; +} + void ir3_compiler_destroy(struct ir3_compiler *compiler) { @@ -217,6 +245,12 @@ ir3_compiler_debug_init(void) util_call_once(&once, __debug_init); } +static inline bool +ir3_is_a810(const struct fd_dev_id *dev_id) +{ + return dev_id->chip_id == 0x44010000; +} + struct ir3_compiler * ir3_compiler_create(struct fd_device *dev, const struct fd_dev_id *dev_id, const struct fd_dev_info *dev_info, @@ -233,6 +267,9 @@ ir3_compiler_create(struct fd_device *dev, const struct fd_dev_id *dev_id, compiler->options = *options; compiler->info = dev_info; + if (ir3_is_a810(dev_id)) + ir3_shader_debug |= IR3_DBG_NODESCPREFETCH; + /* TODO see if older GPU's were different here */ compiler->branchstack_size = dev_info->props.has_dual_wave_dispatch ? 512 : 256; compiler->max_branchstack = 64; @@ -382,6 +419,14 @@ ir3_compiler_create(struct fd_device *dev, const struct fd_dev_id *dev_id, compiler->has_bitwise_triops = compiler->gen >= 5; compiler->cat3_rel_offset_0_quirk = compiler->gen <= 5; + /* + * Adreno 810 has a much smaller cache/GMEM budget and substantially lower + * external memory bandwidth than the larger A8xx parts. Let the UBO + * promotion pass spend a few extra const-file slots merging nearby ranges + * so hot shader code issues fewer memory-backed UBO reads. + */ + compiler->coalesce_ubo_push_ranges = dev_id->chip_id == 0xffff44010000ull; + /* The driver can't request this unless preambles are supported. */ if (options->push_ubo_with_preamble) assert(compiler->has_preamble); @@ -456,4 +501,4 @@ const char * ir3_shader_debug_as_string() { return debug_dump_flags(shader_debug_options, ir3_shader_debug); -} +} \ No newline at end of file diff --git a/src/freedreno/ir3/ir3_compiler.h b/src/freedreno/ir3/ir3_compiler.h index 2a581a26a509..7499076bc11a 100644 --- a/src/freedreno/ir3/ir3_compiler.h +++ b/src/freedreno/ir3/ir3_compiler.h @@ -58,6 +58,13 @@ struct ir3_compiler_options { uint64_t uche_trap_base; }; +struct ir3_gpu_profile { + uint32_t reg_efficiency; + uint32_t max_sy_inflight; + uint32_t max_ss_inflight; + bool force_double_threadsize; +}; + struct ir3_compiler { struct fd_device *dev; const struct fd_dev_id *dev_id; @@ -239,6 +246,13 @@ struct ir3_compiler { bool cat3_rel_offset_0_quirk; + /* + * Some low-bandwidth parts benefit from spending a little more constant + * file space to merge nearby promoted UBO ranges. This reduces the number + * of memory-backed UBO fetch windows left in hot shader code. + */ + bool coalesce_ubo_push_ranges; + struct { /* The number of cycles needed for the result of one ALU operation to be * available to another ALU operation. Only valid when the halfness of the @@ -389,6 +403,52 @@ void ir3_shader_bisect_dump_id(struct ir3_shader_variant *v); bool ir3_shader_bisect_select(struct ir3_shader_variant *v); bool ir3_shader_bisect_disasm_select(struct ir3_shader_variant *v); +/* ========== A8XX HELPER FUNCTIONS ========== */ + +struct ir3_gpu_profile; +struct ir3_gpu_profile ir3_get_gpu_profile(uint32_t chip_id); +uint32_t ir3_effective_reg_size(struct ir3_compiler *compiler); + +static inline bool +ir3_force_double_threadsize(struct ir3_compiler *compiler) +{ + if (compiler->gen >= 8) { + struct ir3_gpu_profile profile = ir3_get_gpu_profile(compiler->dev_id->chip_id); + return profile.force_double_threadsize; + } + return compiler->info->props.supports_double_threadsize; +} + +/* Helper function to check if A8XX should use aggressive const limits */ +static inline bool +ir3_use_aggressive_const_limits(struct ir3_compiler *compiler) +{ + return compiler->gen >= 8; +} + +/* Helper function to get optimal delay slots for current gen */ +static inline unsigned +ir3_get_alu_to_alu_delay(struct ir3_compiler *compiler) +{ + if (compiler->gen >= 8) + return 1; + if (compiler->gen >= 7) + return 2; + return 3; +} + +static inline unsigned +ir3_get_non_alu_delay(struct ir3_compiler *compiler) +{ + if (compiler->gen >= 8) + return 4; + if (compiler->gen >= 7) + return 5; + return 6; +} + +/* ========== END A8XX HELPER FUNCTIONS ========== */ + ENDC; -#endif /* IR3_COMPILER_H_ */ +#endif /* IR3_COMPILER_H_ */ \ No newline at end of file diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index a06d6f85bbf6..736d5a7498eb 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -5799,6 +5799,8 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, goto out; } + bool is_preamble_speculatable = ir3_nir_is_preamble_speculatable(ctx->s); + emit_instructions(ctx); if (ctx->error) { @@ -6187,7 +6189,7 @@ ir3_compile_shader_nir(struct ir3_compiler *compiler, /* We need to do legalize after (for frag shader's) the "bary.f" * offsets (inloc) have been assigned. */ - IR3_PASS(ir, ir3_legalize, so, &max_bary); + IR3_PASS(ir, ir3_legalize, so, &max_bary, is_preamble_speculatable); if (ctx->compiler->cs_lock_unlock_quirk && ir3_shader_compute(so)) { struct ir3_instruction *end = ir3_find_end(so->ir); diff --git a/src/freedreno/ir3/ir3_legalize.c b/src/freedreno/ir3/ir3_legalize.c index 02c3812264aa..b316b2b29666 100644 --- a/src/freedreno/ir3/ir3_legalize.c +++ b/src/freedreno/ir3/ir3_legalize.c @@ -2454,7 +2454,8 @@ align_aliases(struct ir3 *ir) } bool -ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary) +ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary, + bool can_speculate_preamble) { struct ir3_legalize_ctx *ctx = rzalloc(ir, struct ir3_legalize_ctx); bool progress; @@ -2534,7 +2535,7 @@ ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary) } } - so->early_preamble = has_preamble && !gpr_in_preamble && + so->early_preamble = can_speculate_preamble && has_preamble && !gpr_in_preamble && !pred_in_preamble && !relative_in_preamble && ir->compiler->info->props.has_early_preamble && !(ir3_shader_debug & IR3_DBG_NOEARLYPREAMBLE); diff --git a/src/freedreno/ir3/ir3_nir.c b/src/freedreno/ir3/ir3_nir.c index c7430bdd041b..b32275179fb0 100644 --- a/src/freedreno/ir3/ir3_nir.c +++ b/src/freedreno/ir3/ir3_nir.c @@ -90,6 +90,7 @@ load_driver_ubo(nir_builder *b, unsigned components, nir_def *ubo, unsigned offs { return nir_load_ubo(b, components, 32, ubo, nir_imm_int(b, offset * sizeof(uint32_t)), + .access = ACCESS_CAN_SPECULATE, .align_mul = 16, .align_offset = (offset % 4) * sizeof(uint32_t), .range_base = offset * sizeof(uint32_t), @@ -2085,3 +2086,56 @@ ir3_nir_get_global_offset(nir_builder *b, struct ir3_compiler *compiler, .shift = offset_shift, }; } + +/* Early preamble may execute even if the shader doesn't. In order for this to + * be safe, every instruction must be speculatable, i.e. it cannot cause faults + * no matter what data the user throws at it. Generally this means descriptors + * are in-bounds and (if loading from descriptors) they contain valid data. + */ + +bool +ir3_nir_is_preamble_speculatable(nir_shader *s) +{ + nir_function_impl *entrypoint = nir_shader_get_entrypoint(s); + + bool in_preamble = false; + nir_foreach_block (block, entrypoint) { + nir_foreach_instr (instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); + + if (intrin->intrinsic == nir_intrinsic_preamble_start_ir3) { + in_preamble = true; + continue; + } + + /* We've reached the end of the preamble without finding a + * non-speculatable instruction. + */ + if (intrin->intrinsic == nir_intrinsic_preamble_end_ir3) + return true; + + /* As a special case, copy_push_const_to_uniform isn't marked + * can_reorder but it's speculatable anyway. We don't currently have + * a way to mark always-speculatable-but-not-reorderable intrinsics. + * Ignore elect_any_ir3 as it can only be part of the scaffolding we + * emit for the preamble. + */ + if (intrin->intrinsic == nir_intrinsic_copy_push_const_to_uniform_ir3 || + intrin->intrinsic == nir_intrinsic_elect_any_ir3) + continue; + + /* Ignore anything outside the preamble. */ + if (!in_preamble) + continue; + + if (nir_intrinsic_has_access(intrin) && + !(nir_intrinsic_access(intrin) & ACCESS_CAN_SPECULATE)) + return false; + } + } + + return true; +} diff --git a/src/freedreno/ir3/ir3_nir.h b/src/freedreno/ir3/ir3_nir.h index 95817d5c73cc..85fae65da740 100644 --- a/src/freedreno/ir3/ir3_nir.h +++ b/src/freedreno/ir3/ir3_nir.h @@ -215,6 +215,8 @@ nir_io_offset ir3_nir_get_global_offset(nir_builder *b, struct ir3_compiler *compiler, nir_def *offset, unsigned offset_shift); +bool ir3_nir_is_preamble_speculatable(nir_shader *s); + ENDC; #endif /* IR3_NIR_H_ */ diff --git a/src/freedreno/ir3/ir3_nir_analyze_ubo_ranges.c b/src/freedreno/ir3/ir3_nir_analyze_ubo_ranges.c index aa6b04693f3a..7e6cba3734ba 100644 --- a/src/freedreno/ir3/ir3_nir_analyze_ubo_ranges.c +++ b/src/freedreno/ir3/ir3_nir_analyze_ubo_ranges.c @@ -64,6 +64,7 @@ get_ubo_info(nir_intrinsic_instr *instr, struct ir3_ubo_info *ubo) return true; } } + ubo->can_speculate = nir_intrinsic_access(instr) & ACCESS_CAN_SPECULATE; return false; } @@ -96,7 +97,8 @@ get_existing_range(nir_intrinsic_instr *instr, * newly updated range. */ static void -merge_neighbors(struct ir3_ubo_analysis_state *state, int index) +merge_neighbors(struct ir3_ubo_analysis_state *state, int index, + uint32_t max_coalesce_gap) { struct ir3_ubo_range *a = &state->range[index]; @@ -108,7 +110,13 @@ merge_neighbors(struct ir3_ubo_analysis_state *state, int index) if (memcmp(&a->ubo, &b->ubo, sizeof(a->ubo))) continue; - if (a->start > b->end || a->end < b->start) + uint32_t gap = 0; + if (a->end < b->start) + gap = b->start - a->end; + else if (b->end < a->start) + gap = a->start - b->end; + + if (gap > max_coalesce_gap) continue; /* Merge B into A. */ @@ -130,7 +138,7 @@ merge_neighbors(struct ir3_ubo_analysis_state *state, int index) static void gather_ubo_ranges(nir_shader *nir, nir_intrinsic_instr *instr, struct ir3_ubo_analysis_state *state, uint32_t alignment, - uint32_t *upload_remaining) + uint32_t max_coalesce_gap, uint32_t *upload_remaining) { struct ir3_ubo_info ubo = {}; if (!get_ubo_info(instr, &ubo)) @@ -146,10 +154,19 @@ gather_ubo_ranges(nir_shader *nir, nir_intrinsic_instr *instr, if (memcmp(&plan_r->ubo, &ubo, sizeof(ubo))) continue; - /* Don't extend existing uploads unless they're - * neighboring/overlapping. + /* Don't extend existing uploads unless they're neighboring/overlapping. + * On bandwidth-limited GPUs, allow a small hole between ranges. The + * preamble may copy a few unused dwords, but hot shader code can then + * source nearby UBO loads from the constant file instead of issuing more + * memory-backed UBO reads. */ - if (r.start > plan_r->end || r.end < plan_r->start) + uint32_t gap = 0; + if (plan_r->end < r.start) + gap = r.start - plan_r->end; + else if (r.end < plan_r->start) + gap = plan_r->start - r.end; + + if (gap > max_coalesce_gap) continue; r.start = MIN2(r.start, plan_r->start); @@ -163,7 +180,7 @@ gather_ubo_ranges(nir_shader *nir, nir_intrinsic_instr *instr, plan_r->end = r.end; *upload_remaining -= added; - merge_neighbors(state, i); + merge_neighbors(state, i, max_coalesce_gap); return; } @@ -410,7 +427,8 @@ copy_global_to_uniform(nir_shader *nir, struct ir3_ubo_analysis_state *state) for (unsigned offset = 0; offset < size; offset += 256 * 16) { unsigned const_offset = range->offset / 4 + offset / 4; nir_copy_global_to_uniform_ir3( - b, base, .base = start + offset, .range_base = const_offset, + b, base, .access = range->ubo.can_speculate ? ACCESS_CAN_SPECULATE : 0, + .base = start + offset, .range_base = const_offset, .range = MIN2(256, (size - offset) / 16)); } } @@ -434,8 +452,11 @@ copy_ubo_to_uniform(nir_shader *nir, const struct ir3_const_state *const_state) const struct ir3_ubo_range *range = &state->range[i]; nir_def *ubo = nir_imm_int(b, range->ubo.block); + enum gl_access_qualifier access = + range->ubo.can_speculate ? ACCESS_CAN_SPECULATE : 0; if (range->ubo.bindless) { ubo = nir_bindless_resource_ir3(b, 32, ubo, + .access = access, .desc_set = range->ubo.bindless_base); } @@ -446,6 +467,7 @@ copy_ubo_to_uniform(nir_shader *nir, const struct ir3_const_state *const_state) for (unsigned offset = 0; offset < size; offset += 256) { nir_copy_ubo_to_uniform_ir3(b, ubo, nir_imm_int(b, range->start / 16 + offset), + .access = access, .base = range->offset / 4 + offset * 4, .range = MIN2(size - offset, 256)); } @@ -465,7 +487,12 @@ instr_is_load_ubo(nir_instr *instr) /* nir_lower_ubo_vec4 happens after this pass. */ assert(op != nir_intrinsic_load_ubo_vec4); - return op == nir_intrinsic_load_ubo; + if (op != nir_intrinsic_load_ubo) + return false; + + return instr->block->cf_node.parent->type == nir_cf_node_function || + (nir_intrinsic_access(nir_instr_as_intrinsic(instr)) & + ACCESS_CAN_SPECULATE); } static bool @@ -561,7 +588,7 @@ ir3_nir_lower_const_global_loads(nir_shader *nir, struct ir3_shader_variant *v) if (instr_is_load_const(instr) && ir3_def_is_rematerializable_for_preamble(nir_instr_as_intrinsic(instr)->src[0].ssa, NULL)) gather_ubo_ranges(nir, nir_instr_as_intrinsic(instr), &state, - align_vec4, + align_vec4, 0, &upload_remaining); } } @@ -635,6 +662,11 @@ ir3_nir_analyze_ubo_ranges(nir_shader *nir, struct ir3_shader_variant *v) return; uint32_t upload_remaining = max_upload; + /* A810 is especially external-bandwidth constrained. When promoting UBO + * loads on that GPU, coalesce ranges separated by up to 128 bytes (8 vec4s) + * so repeated per-invocation loads are more likely to hit the const file. + */ + uint32_t max_coalesce_gap = compiler->coalesce_ubo_push_ranges ? 128 : 0; bool push_ubos = compiler->options.push_ubo_with_preamble; nir_foreach_function (function, nir) { @@ -643,7 +675,7 @@ ir3_nir_analyze_ubo_ranges(nir_shader *nir, struct ir3_shader_variant *v) nir_foreach_instr (instr, block) { if (instr_is_load_ubo(instr)) gather_ubo_ranges(nir, nir_instr_as_intrinsic(instr), state, - align_vec4, + align_vec4, max_coalesce_gap, &upload_remaining); } } diff --git a/src/freedreno/ir3/ir3_nir_opt_preamble.c b/src/freedreno/ir3/ir3_nir_opt_preamble.c index 3fe3d4d66eae..fdbe7aa6aa1a 100644 --- a/src/freedreno/ir3/ir3_nir_opt_preamble.c +++ b/src/freedreno/ir3/ir3_nir_opt_preamble.c @@ -160,11 +160,11 @@ instr_cost(nir_instr *instr, const void *data) nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_ubo: { - /* If the UBO and offset are constant, then UBO lowering should do a - * better job trying to lower this, and opt_preamble shouldn't try to - * duplicate it. However if it has a non-constant offset then we can - * avoid setting up a0.x etc. in the main shader and potentially have - * to push less. + /* If the UBO and offset are constant and it is speculatable, then UBO + * lowering should do a better job trying to lower this, and + * opt_preamble shouldn't try to duplicate it. However if it has a + * non-constant offset then we can avoid setting up a0.x etc. in the + * main shader and potentially have to push less. */ bool const_ubo = nir_src_is_const(intrin->src[0]); if (!const_ubo) { @@ -173,7 +173,9 @@ instr_cost(nir_instr *instr, const void *data) const_ubo = nir_src_is_const(rsrc->src[0]); } - if (const_ubo && nir_src_is_const(intrin->src[1])) + if (const_ubo && nir_src_is_const(intrin->src[1]) && + (instr->block->cf_node.parent->type == nir_cf_node_function || + (nir_intrinsic_access(intrin) & ACCESS_CAN_SPECULATE))) return 0; /* TODO: get actual numbers for ldc */ @@ -268,44 +270,6 @@ avoid_instr(const nir_instr *instr, const void *data) return intrin->intrinsic == nir_intrinsic_bindless_resource_ir3; } -static bool -set_speculate(nir_builder *b, nir_instr *instr, UNUSED void *_) -{ - if (instr->type == nir_instr_type_tex) { - nir_instr_as_tex(instr)->can_speculate = true; - return true; - } - - if (instr->type != nir_instr_type_intrinsic) - return false; - - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - - switch (intr->intrinsic) { - /* These instructions go through bounds-checked hardware descriptors so - * should be safe to speculate. - * - * TODO: This isn't necessarily true in Vulkan, where descriptors don't need - * to be filled out and bindless descriptor offsets aren't bounds checked. - * We may need to plumb this information through from turnip for correctness - * to avoid regressing freedreno codegen. - */ - case nir_intrinsic_load_ubo: - case nir_intrinsic_load_ubo_vec4: - case nir_intrinsic_image_load: - case nir_intrinsic_image_samples_identical: - case nir_intrinsic_bindless_image_load: - case nir_intrinsic_load_ssbo: - case nir_intrinsic_load_ssbo_ir3: - nir_intrinsic_set_access(intr, nir_intrinsic_access(intr) | - ACCESS_CAN_SPECULATE); - return true; - - default: - return false; - } -} - bool ir3_nir_opt_preamble(nir_shader *nir, struct ir3_shader_variant *v) { @@ -323,9 +287,6 @@ ir3_nir_opt_preamble(nir_shader *nir, struct ir3_shader_variant *v) if (max_size == 0) return false; - bool progress = nir_shader_instructions_pass(nir, set_speculate, - nir_metadata_control_flow, NULL); - nir_opt_preamble_options options = { .drawid_uniform = true, .subgroup_size_uniform = true, @@ -339,7 +300,7 @@ ir3_nir_opt_preamble(nir_shader *nir, struct ir3_shader_variant *v) }; unsigned size = 0; - progress |= nir_opt_preamble(nir, &options, &size); + bool progress = nir_opt_preamble(nir, &options, &size); if (!v->binning_pass) { uint32_t preamble_size_vec4 = @@ -596,6 +557,33 @@ get_descriptors(nir_instr *instr, nir_def **descs) } } +static bool +is_descriptor_prefetch_speculatable(nir_def *desc) +{ + nir_instr *instr = nir_def_instr(desc); + + /* Non-bindless descriptors are always speculatable */ + if (instr->type != nir_instr_type_intrinsic) + return true; + + nir_intrinsic_instr *bindless = nir_instr_as_intrinsic(nir_def_instr(desc)); + + if (bindless->intrinsic != nir_intrinsic_bindless_resource_ir3) + return true; + + return nir_intrinsic_access(bindless) & ACCESS_CAN_SPECULATE; +} + +static bool +is_descriptor_prefetchable(nir_def *desc) +{ + nir_instr *instr = nir_def_instr(desc); + + return instr->block->cf_node.parent->type == nir_cf_node_function || + is_descriptor_prefetch_speculatable(desc); +} + + #define MAX_PREFETCHES 32 struct prefetches { @@ -627,10 +615,12 @@ struct prefetch_state { static bool emit_descriptor_prefetch(nir_builder *b, nir_instr *instr, nir_def **descs, - struct prefetch_state *state) + struct prefetch_state *state, bool can_speculate) { nir_block *insert_block = nir_def_block(descs[0]); + enum gl_access_qualifier access = can_speculate ? ACCESS_CAN_SPECULATE : 0; + if (descs[1]) { insert_block = find_insert_block_for_defs(descs, 2); @@ -679,13 +669,13 @@ emit_descriptor_prefetch(nir_builder *b, nir_instr *instr, nir_def **descs, if (!sampler_already_prefetched) add_prefetch(&state->sampler, descs[1]); - nir_prefetch_sam_ir3(b, descs[0], descs[1]); + nir_prefetch_sam_ir3(b, descs[0], descs[1], .access = access); } else { if (tex_already_prefetched) return false; add_prefetch(&state->tex, descs[0]); - nir_prefetch_tex_ir3(b, descs[0]); + nir_prefetch_tex_ir3(b, descs[0], .access = access); } } else { assert(instr->type == nir_instr_type_intrinsic); @@ -700,9 +690,9 @@ emit_descriptor_prefetch(nir_builder *b, nir_instr *instr, nir_def **descs, nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (intrin->intrinsic == nir_intrinsic_load_ubo) - nir_prefetch_ubo_ir3(b, descs[0]); + nir_prefetch_ubo_ir3(b, descs[0], .access = access); else - nir_prefetch_tex_ir3(b, descs[0]); + nir_prefetch_tex_ir3(b, descs[0], .access = access); } return true; @@ -776,29 +766,20 @@ ir3_nir_opt_prefetch_descriptors(nir_shader *nir, struct ir3_shader_variant *v) should_prefetch_descriptor(descs[1]))) continue; - /* The instruction itself must be hoistable. - * TODO: If the descriptor is statically referenced and in-bounds, then - * we should be able to hoist the descriptor load even if the - * descriptor contents aren't guaranteed. This would require more - * plumbing. - * TODO: Textures. This is broken in nir_opt_preamble at the moment and - * handling them would also require more plumbing. - */ - if (instr->type == nir_instr_type_intrinsic && - nir_intrinsic_has_access(nir_instr_as_intrinsic(instr)) && - !(nir_intrinsic_access(nir_instr_as_intrinsic(instr)) & - ACCESS_CAN_SPECULATE) && - block->cf_node.parent->type != nir_cf_node_function) - continue; - - /* Each descriptor must be rematerializable */ + /* Each descriptor must be rematerializable and speculatable */ if (descs[0] && - !ir3_def_is_rematerializable_for_preamble(descs[0], preamble_defs)) + (!is_descriptor_prefetchable(descs[0]) || + !ir3_def_is_rematerializable_for_preamble(descs[0], preamble_defs))) continue; if (descs[1] && - !ir3_def_is_rematerializable_for_preamble(descs[1], preamble_defs)) + (!is_descriptor_prefetchable(descs[1]) || + !ir3_def_is_rematerializable_for_preamble(descs[1], preamble_defs))) continue; + bool is_speculatable = + (!descs[0] || is_descriptor_prefetch_speculatable(descs[0])) && + (!descs[1] || is_descriptor_prefetch_speculatable(descs[1])); + /* If the preamble hasn't been created then this descriptor isn't a * duplicate and we will definitely insert an instruction, so create * the preamble if it hasn't already been created. @@ -849,7 +830,8 @@ ir3_nir_opt_prefetch_descriptors(nir_shader *nir, struct ir3_shader_variant *v) preamble_defs); } - progress |= emit_descriptor_prefetch(&b, instr, preamble_descs, &state); + progress |= emit_descriptor_prefetch(&b, instr, preamble_descs, &state, + is_speculatable); if (state.sampler.num_prefetches == MAX_PREFETCHES && state.tex.num_prefetches == MAX_PREFETCHES) diff --git a/src/freedreno/ir3/ir3_shader.h b/src/freedreno/ir3/ir3_shader.h index b23bc24f22ad..c9eb4b2822db 100644 --- a/src/freedreno/ir3/ir3_shader.h +++ b/src/freedreno/ir3/ir3_shader.h @@ -153,6 +153,7 @@ struct ir3_ubo_info { uint16_t bindless_base; /* For bindless, which base register is used */ bool bindless; bool global; + bool can_speculate; }; /** diff --git a/src/freedreno/ir3/tests/delay.c b/src/freedreno/ir3/tests/delay.c index 516bd414f666..f955f6472cb7 100644 --- a/src/freedreno/ir3/tests/delay.c +++ b/src/freedreno/ir3/tests/delay.c @@ -184,7 +184,7 @@ main(int argc, char **argv) } int max_bary; - ir3_legalize(ir, shader->variants, &max_bary); + ir3_legalize(ir, shader->variants, &max_bary, false); unsigned n = calc_nops(block, last); diff --git a/src/freedreno/vulkan/tu_cmd_buffer.cc b/src/freedreno/vulkan/tu_cmd_buffer.cc index cbbad92a20d0..f072d3ea8b86 100644 --- a/src/freedreno/vulkan/tu_cmd_buffer.cc +++ b/src/freedreno/vulkan/tu_cmd_buffer.cc @@ -1372,6 +1372,12 @@ use_sysmem_rendering(struct tu_cmd_buffer *cmd, return true; } + bool no_gmem = cmd->device->physical_device->dev_info.props.disable_gmem; + if (no_gmem) { + cmd->state.rp.gmem_disable_reason = "Unsupported GPU"; + return true; + } + /* can't fit attachments into gmem */ if (!cmd->state.tiling->possible) { cmd->state.rp.gmem_disable_reason = "Can't fit attachments into gmem"; diff --git a/src/freedreno/vulkan/tu_descriptor_set.cc b/src/freedreno/vulkan/tu_descriptor_set.cc index 2ff682643517..ca771d299f67 100644 --- a/src/freedreno/vulkan/tu_descriptor_set.cc +++ b/src/freedreno/vulkan/tu_descriptor_set.cc @@ -196,6 +196,17 @@ tu_CreateDescriptorSetLayout( set_layout->binding[b].offset = set_layout->size; set_layout->binding[b].dynamic_offset_offset = dynamic_offset_size; set_layout->binding[b].shader_stages = binding->stageFlags; + set_layout->binding[b].partially_bound = + /* Descriptor buffer implies PARTIALLY_BOUND. From a NOTE in the + * spec: "The requirements above imply that all descriptor bindings + * have been defined with the equivalent of ... + * VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT" + */ + (pCreateInfo->flags & + VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT) || + (variable_flags && j < variable_flags->bindingCount && + (variable_flags->pBindingFlags[j] & + VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT)); bool has_subsampled_sampler = false; if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || @@ -473,6 +484,7 @@ blake3_update_descriptor_set_binding_layout(blake3_hasher *ctx, BLAKE3_UPDATE_VALUE(ctx, layout->array_size); BLAKE3_UPDATE_VALUE(ctx, layout->dynamic_offset_offset); BLAKE3_UPDATE_VALUE(ctx, layout->immutable_samplers_offset); + BLAKE3_UPDATE_VALUE(ctx, layout->partially_bound); const struct tu_sampler *samplers = tu_immutable_samplers(set_layout, layout); diff --git a/src/freedreno/vulkan/tu_descriptor_set.h b/src/freedreno/vulkan/tu_descriptor_set.h index 6a6a1ff9e59a..1f6e52447a93 100644 --- a/src/freedreno/vulkan/tu_descriptor_set.h +++ b/src/freedreno/vulkan/tu_descriptor_set.h @@ -61,6 +61,12 @@ struct tu_descriptor_set_binding_layout /* Shader stages that use this binding */ uint32_t shader_stages; + + /* Whether statically accessing this binding guarantees that the descriptor + * accessed is valid. If true, only dynamically accessed descriptors are + * valid and we cannot speculate descriptor access. + */ + bool partially_bound; }; struct tu_descriptor_set_layout diff --git a/src/freedreno/vulkan/tu_device.cc b/src/freedreno/vulkan/tu_device.cc index 416c10e1af1e..bc8f40903f67 100644 --- a/src/freedreno/vulkan/tu_device.cc +++ b/src/freedreno/vulkan/tu_device.cc @@ -996,6 +996,12 @@ tu_get_physical_device_properties_1_2(struct tu_physical_device *pdevice, }; } + if (TU_DEBUG(DECK_EMU)) { + p->driverID = VK_DRIVER_ID_MESA_RADV; + memset(p->driverName, 0, sizeof(p->driverName)); + snprintf(p->driverName, VK_MAX_DRIVER_NAME_SIZE, "radv"); + } + p->denormBehaviorIndependence = VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL; p->roundingModeIndependence = @@ -1283,17 +1289,35 @@ tu_get_properties(struct tu_physical_device *pdevice, props->optimalBufferCopyRowPitchAlignment = 128; props->nonCoherentAtomSize = 64; - props->apiVersion = + /* HACK: Expose Vulkan 1.3 on devices without multiview. + * Current Minecraft renderer checks for VK1.2 presence and refuses to + * start on VK1.0 exposed here in the case if the device has no multiview + * This makes it boot on these devices. This is not conformant, but I don't + * care. Sigh. + */ + + /*props->apiVersion = tu_has_multiview(pdevice) ? ((pdevice->info->chip >= 7) ? TU_API_VERSION : VK_MAKE_VERSION(1, 3, VK_HEADER_VERSION)) : VK_MAKE_VERSION(1, 0, VK_HEADER_VERSION); + */ + + props->apiVersion = pdevice->info->chip >= 7 ? + TU_API_VERSION : // expose current api on gen7 devices anyway + VK_MAKE_VERSION(1, 3, VK_HEADER_VERSION); + props->driverVersion = vk_get_driver_version(); props->vendorID = pdevice->instance->drirc.debug.force_vk_vendor != 0 ? pdevice->instance->drirc.debug.force_vk_vendor : 0x5143; props->deviceID = pdevice->dev_id.chip_id; props->deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; + if (TU_DEBUG(DECK_EMU)) { + props->vendorID = 0x1002; + props->deviceID = 0x163F; + } + /* Vulkan 1.4 */ props->dynamicRenderingLocalReadDepthStencilAttachments = true; props->dynamicRenderingLocalReadMultisampledAttachments = true; @@ -1310,6 +1334,10 @@ tu_get_properties(struct tu_physical_device *pdevice, pdevice->instance->drirc.debug.force_vk_devicename : pdevice->name); memcpy(props->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE); + if (TU_DEBUG(DECK_EMU)) { + strcpy(props->deviceName, "AMD Custom GPU 0405 (RADV VANGOGH)"); + } + tu_get_physical_device_properties_1_1(pdevice, props); tu_get_physical_device_properties_1_2(pdevice, props); tu_get_physical_device_properties_1_3(pdevice, props); @@ -2836,6 +2864,9 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice, vk_device_dispatch_table_from_entrypoints( &dispatch_table, &tu_device_entrypoints_a8xx, false); } + + /* HACK: disable concurrent binning for now */ + tu_env.debug |= TU_DEBUG_NO_CONCURRENT_BINNING; vk_device_dispatch_table_from_entrypoints( &dispatch_table, &wsi_device_entrypoints, false); diff --git a/src/freedreno/vulkan/tu_pipeline.cc b/src/freedreno/vulkan/tu_pipeline.cc index d69d23db89b0..b31e1a183899 100644 --- a/src/freedreno/vulkan/tu_pipeline.cc +++ b/src/freedreno/vulkan/tu_pipeline.cc @@ -1743,6 +1743,13 @@ tu_pipeline_builder_compile_shaders(struct tu_pipeline_builder *builder, }; VkPipelineCreationFeedback stage_feedbacks[MESA_SHADER_STAGES] = { 0 }; + const uint64_t chip_id = builder->device->physical_device->dev_id.chip_id; + const bool is_a810 = chip_id == 0x44010000ull; + const bool is_a825 = chip_id == 0x44030000ull; + const bool is_a829 = chip_id == 0x44030A20ull; + const bool is_a830 = chip_id == 0xffff44050000 || 0x44050001; + const bool is_target_gpu = is_a810 || is_a825 || is_a829 || is_a830; + const bool executable_info = builder->create_flags & VK_PIPELINE_CREATE_2_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR; @@ -1903,7 +1910,8 @@ tu_pipeline_builder_compile_shaders(struct tu_pipeline_builder *builder, } } - keys[last_pre_rast_stage].fdm_per_layer = builder->fdm_per_layer; + keys[last_pre_rast_stage].fdm_per_layer = + is_target_gpu ? false : builder->fdm_per_layer; } if (builder->state & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT) { @@ -1945,7 +1953,7 @@ tu_pipeline_builder_compile_shaders(struct tu_pipeline_builder *builder, * tu_shader_key::force_sample_interp in a bit. */ keys[MESA_SHADER_FRAGMENT].force_sample_interp = - !builder->rasterizer_discard && msaa_info && msaa_info->sampleShadingEnable; + is_target_gpu ? false : (!builder->rasterizer_discard && msaa_info && msaa_info->sampleShadingEnable); } unsigned char pipeline_blake3[BLAKE3_KEY_LEN]; diff --git a/src/freedreno/vulkan/tu_shader.cc b/src/freedreno/vulkan/tu_shader.cc index 8c228e9ff668..33d1e4712d8c 100644 --- a/src/freedreno/vulkan/tu_shader.cc +++ b/src/freedreno/vulkan/tu_shader.cc @@ -461,8 +461,13 @@ lower_ssbo_ubo_intrinsic(struct tu_device *dev, nir_def *results[MAX_SETS] = { NULL }; if (nir_scalar_is_const(scalar_idx)) { + bool can_speculate_descriptor = intrin->instr.pass_flags; nir_def *bindless = - nir_bindless_resource_ir3(b, 32, descriptor_idx, .desc_set = nir_scalar_as_uint(scalar_idx)); + nir_bindless_resource_ir3(b, 32, descriptor_idx, + .desc_set = nir_scalar_as_uint(scalar_idx), + .access = can_speculate_descriptor ? + ACCESS_CAN_SPECULATE : + (gl_access_qualifier)0); nir_src_rewrite(&intrin->src[buffer_src], bindless); return true; } @@ -524,14 +529,17 @@ build_bindless(struct tu_device *dev, nir_builder *b, struct tu_shader *shader, const struct tu_pipeline_layout *layout, uint32_t read_only_input_attachments, - bool dynamic_renderpass) + bool dynamic_renderpass, + bool *descriptor_valid) { nir_variable *var = nir_deref_instr_get_variable(deref); unsigned set = var->data.descriptor_set; unsigned binding = var->data.binding; + const struct tu_descriptor_set_layout *set_layout = + layout->set[set].layout; const struct tu_descriptor_set_binding_layout *bind_layout = - &layout->set[set].layout->binding[binding]; + &set_layout->binding[binding]; /* input attachments use non bindless workaround */ if (bind_layout->type == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT && @@ -590,15 +598,29 @@ build_bindless(struct tu_device *dev, nir_builder *b, offset); descriptor_stride = bind_layout->size / (4 * FDL6_TEX_CONST_DWORDS); + bool can_speculate_descriptor = true; + if (deref->deref_type != nir_deref_type_var) { assert(deref->deref_type == nir_deref_type_array); nir_def *arr_index = deref->arr.index.ssa; desc_offset = nir_iadd(b, desc_offset, nir_imul_imm(b, arr_index, descriptor_stride)); + if (!nir_src_is_const(deref->arr.index) || + (set_layout->has_variable_descriptors && + binding == set_layout->binding_count - 1) || + nir_src_as_uint(deref->arr.index) >= bind_layout->array_size) + can_speculate_descriptor = false; } - return nir_bindless_resource_ir3(b, 32, desc_offset, .desc_set = set); + *descriptor_valid = !bind_layout->partially_bound && + can_speculate_descriptor; + + return nir_bindless_resource_ir3(b, 32, desc_offset, + .desc_set = set, + .access = can_speculate_descriptor ? + ACCESS_CAN_SPECULATE : + (gl_access_qualifier)0); } static nir_def * @@ -680,8 +702,20 @@ lower_image_deref(struct tu_device *dev, nir_builder *b, nir_intrinsic_instr *instr, struct tu_shader *shader, const struct tu_pipeline_layout *layout) { + bool descriptor_valid = true; nir_deref_instr *deref = nir_src_as_deref(instr->src[0]); - nir_def *bindless = build_bindless(dev, b, deref, 0, shader, layout, 0, false); + nir_def *bindless = build_bindless(dev, b, deref, 0, shader, layout, 0, false, + &descriptor_valid); + if ((instr->intrinsic == nir_intrinsic_image_deref_load || + instr->intrinsic == nir_intrinsic_image_deref_sparse_load || + instr->intrinsic == nir_intrinsic_image_size || + instr->intrinsic == nir_intrinsic_image_samples) && + descriptor_valid) { + nir_intrinsic_set_access(instr, + (gl_access_qualifier)(nir_intrinsic_access(instr) | + ACCESS_CAN_SPECULATE)); + } + nir_rewrite_image_intrinsic(instr, bindless, nir_image_intrinsic_type_bindless); @@ -808,13 +842,15 @@ lower_tex_subsampled(const struct tu_sampler *sampler, b->cursor = nir_before_instr(&tex->instr); + bool descriptor_valid = true; + int tex_src_idx = nir_tex_instr_src_index(tex, nir_tex_src_texture_deref); assert(tex_src_idx >= 0); nir_deref_instr *deref = nir_src_as_deref(tex->src[tex_src_idx].src); nir_def *bindless = build_bindless(dev, b, deref, 2, shader, layout, 0, /* read_only_input_attachments (not used) */ - false /* dynamic_renderpass (not used)*/ - ); + false, /* dynamic_renderpass (not used)*/ + &descriptor_valid); nir_def *coord = nir_steal_tex_src(tex, nir_tex_src_coord); nir_def *coord_xy = nir_channels(b, coord, 0x3); @@ -841,7 +877,8 @@ lower_tex_subsampled(const struct tu_sampler *sampler, } nir_def *transformed_coord_xy = - tu_get_subsampled_coordinates(b, clamped_coord, bindless); + tu_get_subsampled_coordinates(b, clamped_coord, bindless, + descriptor_valid); /* Due to VUID-VkSamplerCreateInfo-flags-02577 we only have to handle * CLAMP_TO_EDGE and CLAMP_TO_BORDER. We implicitly do CLAMP_TO_EDGE to @@ -984,12 +1021,14 @@ lower_tex_impl(nir_builder *b, nir_tex_instr *tex, struct tu_device *dev, uint32_t read_only_input_attachments, bool dynamic_renderpass, bool ref) { + bool descriptor_valid = true; int sampler_src_idx = nir_tex_instr_src_index(tex, ref ? nir_tex_src_sampler_2_deref : nir_tex_src_sampler_deref); if (sampler_src_idx >= 0) { nir_deref_instr *deref = nir_src_as_deref(tex->src[sampler_src_idx].src); nir_def *bindless = build_bindless(dev, b, deref, 1, shader, layout, read_only_input_attachments, - dynamic_renderpass); + dynamic_renderpass, + &descriptor_valid); nir_src_rewrite(&tex->src[sampler_src_idx].src, bindless); tex->src[sampler_src_idx].src_type = ref ? nir_tex_src_sampler_2_handle : nir_tex_src_sampler_handle; } @@ -999,7 +1038,8 @@ lower_tex_impl(nir_builder *b, nir_tex_instr *tex, struct tu_device *dev, nir_deref_instr *deref = nir_src_as_deref(tex->src[tex_src_idx].src); nir_def *bindless = build_bindless(dev, b, deref, 0, shader, layout, read_only_input_attachments, - dynamic_renderpass); + dynamic_renderpass, + &descriptor_valid); nir_src_rewrite(&tex->src[tex_src_idx].src, bindless); tex->src[tex_src_idx].src_type = ref ? nir_tex_src_texture_2_handle : nir_tex_src_texture_handle; @@ -1013,6 +1053,9 @@ lower_tex_impl(nir_builder *b, nir_tex_instr *tex, struct tu_device *dev, lower_tex_texel_buffer_to_image(b, tex, tex_src_idx); } + if (!descriptor_valid) + tex->can_speculate = false; + return true; } @@ -1021,6 +1064,8 @@ lower_tex(nir_builder *b, nir_tex_instr *tex, struct tu_device *dev, struct tu_shader *shader, const struct tu_pipeline_layout *layout, uint32_t read_only_input_attachments, bool dynamic_renderpass) { + tex->can_speculate = true; + if (tex->op == nir_texop_block_match_sad_qcom || tex->op == nir_texop_block_match_ssd_qcom || tex->op == nir_texop_sample_weighted_qcom) { @@ -1157,6 +1202,113 @@ lower_inline_ubo(nir_builder *b, nir_intrinsic_instr *intrin, void *cb_data) return true; } +/* Instructions using descriptors are all bounds-checked, so they are valid to + * speculate as long as the descriptor is valid. There are two cases: + * + * 1. If the descriptor set is fully bound (i.e. no PARTIALLY_BOUND_BIT), then + * all descriptors statically used must be valid. That means the descriptor + * and load using the descriptor is free to speculate as long as it + * is always in-bounds. + * 2. If the descriptor set isn't fully bound, the descriptor may not be + * valid. However it may still be valid to speculatively prefetch the + * descriptor, as long as the descriptor is always in-bounds, + * since descriptors must have memory backing them if they are statically + * used. + */ + +static bool +can_speculate_resource(nir_def *def, + const struct tu_pipeline_layout *layout, + bool *can_speculate_descriptor) +{ + nir_instr *instr = nir_def_instr(def); + + *can_speculate_descriptor = false; + + if (instr->type != nir_instr_type_intrinsic) + return false; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic != nir_intrinsic_load_vulkan_descriptor) + return false; + + nir_instr *resource = nir_def_instr(intr->src[0].ssa); + if (resource->type != nir_instr_type_intrinsic) + return false; + + nir_intrinsic_instr *resource_intr = nir_instr_as_intrinsic(resource); + if (resource_intr->intrinsic != nir_intrinsic_vulkan_resource_index) + return false; + + unsigned set = nir_intrinsic_desc_set(resource_intr); + unsigned binding = nir_intrinsic_binding(resource_intr); + struct tu_descriptor_set_layout *set_layout = layout->set[set].layout; + struct tu_descriptor_set_binding_layout *bind_layout = + &set_layout->binding[binding]; + + *can_speculate_descriptor = nir_src_is_const(resource_intr->src[0]) && + (binding != set_layout->binding_count - 1 || + !set_layout->has_variable_descriptors) && + nir_src_as_uint(resource_intr->src[0]) < bind_layout->array_size; + + return *can_speculate_descriptor && !bind_layout->partially_bound; +} + +static bool +set_speculate_intrinsic(nir_intrinsic_instr *intrin, + const struct tu_pipeline_layout *layout) +{ + bool can_speculate = false, can_speculate_descriptor = false; + switch (intrin->intrinsic) { + case nir_intrinsic_load_ubo: + case nir_intrinsic_load_ssbo: + case nir_intrinsic_load_uav_ir3: + case nir_intrinsic_ssbo_atomic: + case nir_intrinsic_ssbo_atomic_swap: + case nir_intrinsic_get_ssbo_size: + can_speculate = can_speculate_resource(intrin->src[0].ssa, layout, + &can_speculate_descriptor); + break; + + case nir_intrinsic_store_ssbo: + can_speculate = can_speculate_resource(intrin->src[1].ssa, layout, + &can_speculate_descriptor); + break; + + default: + return false; + } + + if ((intrin->intrinsic == nir_intrinsic_load_ubo || + intrin->intrinsic == nir_intrinsic_load_ssbo || + intrin->intrinsic == nir_intrinsic_load_uav_ir3 || + intrin->intrinsic == nir_intrinsic_get_ssbo_size) && + can_speculate) { + nir_intrinsic_set_access(intrin, + (gl_access_qualifier)(nir_intrinsic_access(intrin) | + ACCESS_CAN_SPECULATE)); + } + + /* We need to communicate this to descriptor lowering, which happens in a + * separate pass afterwards and which destroys load_vulkan_descriptor + * intrinsics. We stuff the information in the pass_flags. + */ + intrin->instr.pass_flags = can_speculate_descriptor; + return true; +} + +static bool +set_speculate_instr(nir_builder *b, nir_instr *instr, void *cb_data) +{ + struct lower_instr_params *params = (struct lower_instr_params *) cb_data; + if (instr->type == nir_instr_type_intrinsic) { + return set_speculate_intrinsic(nir_instr_as_intrinsic(instr), + params->layout); + } + + return false; +} + /* Figure out the range of push constants that we're actually going to push to * the shader, and tell the backend to reserve this range when pushing UBO * constants. @@ -1399,6 +1551,11 @@ tu_lower_io(nir_shader *shader, struct tu_device *dev, ¶ms); } + progress |= nir_shader_instructions_pass(shader, + set_speculate_instr, + nir_metadata_none, + ¶ms); + progress |= nir_shader_instructions_pass(shader, lower_instr, nir_metadata_none, @@ -1555,7 +1712,8 @@ lower_ssbo_descriptor_instr(nir_builder *b, nir_intrinsic_instr *intrin, descriptor_idx = nir_iadd_imm(b, descriptor_idx, 1); nir_def *new_buffer = nir_bindless_resource_ir3(b, 32, descriptor_idx, - .desc_set = nir_intrinsic_desc_set(bindless)); + .desc_set = nir_intrinsic_desc_set(bindless), + .access = nir_intrinsic_access(bindless)); nir_src_rewrite(&intrin->src[buffer_src], new_buffer); return true; diff --git a/src/freedreno/vulkan/tu_subsampled_image.cc b/src/freedreno/vulkan/tu_subsampled_image.cc index 421e1843d9ff..4e197878479f 100644 --- a/src/freedreno/vulkan/tu_subsampled_image.cc +++ b/src/freedreno/vulkan/tu_subsampled_image.cc @@ -123,8 +123,10 @@ tu_emit_subsampled_metadata(struct tu_cmd_buffer *cmd, nir_def * tu_get_subsampled_coordinates(nir_builder *b, nir_def *coords, - nir_def *descriptor) + nir_def *descriptor, + bool can_speculate) { + gl_access_qualifier access = can_speculate ? ACCESS_CAN_SPECULATE : (gl_access_qualifier)0; nir_def *layer; if (coords->num_components > 2) layer = nir_f2u16(b, nir_channel(b, coords, 2)); @@ -137,11 +139,13 @@ tu_get_subsampled_coordinates(nir_builder *b, nir_def *hdr0 = nir_load_ubo(b, 4, 32, descriptor, nir_ishl_imm(b, nir_u2u32(b, layer_offset), 4), + .access = access, .align_mul = 16, .align_offset = 0, .range = TU_SUBSAMPLED_MAX_LAYERS * sizeof(struct tu_subsampled_metadata)); nir_def *bin_stride = nir_load_ubo(b, 1, 32, descriptor, nir_ishl_imm(b, nir_u2u32(b, nir_iadd_imm(b, layer_offset, 1)), 4), + .access = access, .align_mul = 16, .align_offset = 0, .range = TU_SUBSAMPLED_MAX_LAYERS * sizeof(struct tu_subsampled_metadata)); @@ -159,6 +163,7 @@ tu_get_subsampled_coordinates(nir_builder *b, nir_def *bin_data = nir_load_ubo(b, 4, 32, descriptor, nir_ishl_imm(b, nir_u2u32(b, bin_idx), 4), + .access = access, .align_mul = 16, .align_offset = 0, .range = TU_SUBSAMPLED_MAX_LAYERS * sizeof(struct tu_subsampled_metadata)); diff --git a/src/freedreno/vulkan/tu_subsampled_image.h b/src/freedreno/vulkan/tu_subsampled_image.h index 877b57983f8b..1c49e6a5ff0a 100644 --- a/src/freedreno/vulkan/tu_subsampled_image.h +++ b/src/freedreno/vulkan/tu_subsampled_image.h @@ -85,4 +85,5 @@ tu_calc_subsampled_aprons(VkRect2D *dst, nir_def * tu_get_subsampled_coordinates(nir_builder *b, nir_def *coords, - nir_def *descriptor); + nir_def *descriptor, + bool can_speculate); diff --git a/src/freedreno/vulkan/tu_util.cc b/src/freedreno/vulkan/tu_util.cc index cfc03d6eacd1..832466c92fc2 100644 --- a/src/freedreno/vulkan/tu_util.cc +++ b/src/freedreno/vulkan/tu_util.cc @@ -57,6 +57,7 @@ static const struct debug_control tu_debug_options[] = { { "nocb", TU_DEBUG_NO_CONCURRENT_BINNING }, { "forcecb", TU_DEBUG_FORCE_CONCURRENT_BINNING }, { "computeroundrobin", TU_DEBUG_COMPUTE_ROUND_ROBIN }, + { "deck_emu", TU_DEBUG_DECK_EMU }, { NULL, 0 } }; diff --git a/src/freedreno/vulkan/tu_util.h b/src/freedreno/vulkan/tu_util.h index f73c1f99a8aa..b23a241f91f6 100644 --- a/src/freedreno/vulkan/tu_util.h +++ b/src/freedreno/vulkan/tu_util.h @@ -75,6 +75,7 @@ enum tu_debug_flags : uint64_t TU_DEBUG_NO_CONCURRENT_BINNING = BITFIELD64_BIT(35), TU_DEBUG_FORCE_CONCURRENT_BINNING = BITFIELD64_BIT(36), TU_DEBUG_COMPUTE_ROUND_ROBIN = BITFIELD64_BIT(37), + TU_DEBUG_DECK_EMU = BITFIELD64_BIT(37), }; struct tu_env { diff --git a/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c index 225f6ddc364d..5d8c919e51ac 100644 --- a/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c +++ b/src/gallium/drivers/freedreno/ir3/ir3_descriptor.c @@ -13,6 +13,20 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) { unsigned desc_offset; + bool progress = false; + switch (intr->intrinsic) { + case nir_intrinsic_load_ubo: + case nir_intrinsic_load_ssbo: + case nir_intrinsic_get_ssbo_size: + nir_intrinsic_set_access(intr, + nir_intrinsic_access(intr) | ACCESS_CAN_SPECULATE); + progress = true; + break; + default: + break; + } + + switch (intr->intrinsic) { case nir_intrinsic_load_ssbo: case nir_intrinsic_store_ssbo: @@ -20,6 +34,7 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) case nir_intrinsic_ssbo_atomic_swap: case nir_intrinsic_get_ssbo_size: desc_offset = IR3_BINDLESS_SSBO_OFFSET; + progress = true; break; case nir_intrinsic_image_load: case nir_intrinsic_image_store: @@ -28,9 +43,10 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) case nir_intrinsic_image_size: case nir_intrinsic_image_samples: desc_offset = IR3_BINDLESS_IMAGE_OFFSET; + progress = true; break; default: - return false; + return progress; } unsigned buffer_src; @@ -53,7 +69,9 @@ lower_intrinsic(nir_builder *b, nir_intrinsic_instr *intr) * can avoid the dmesg spam and users thinking this is a driver bug: */ src = nir_umod_imm(b, src, IR3_BINDLESS_DESC_COUNT); - nir_def *bindless = nir_bindless_resource_ir3(b, 32, src, set); + nir_def *bindless = nir_bindless_resource_ir3(b, 32, src, + .desc_set = set, + .access = ACCESS_CAN_SPECULATE); nir_src_rewrite(&intr->src[buffer_src], bindless); return true; diff --git a/src/util/u_gralloc/u_gralloc_fallback.c b/src/util/u_gralloc/u_gralloc_fallback.c index 44fb32d8cfd5..bb6459c2e29c 100644 --- a/src/util/u_gralloc/u_gralloc_fallback.c +++ b/src/util/u_gralloc/u_gralloc_fallback.c @@ -148,12 +148,16 @@ fallback_gralloc_get_buffer_info(struct u_gralloc *gralloc, out->strides[0] = stride; #ifdef HAS_FREEDRENO - uint32_t gmsm = ('g' << 24) | ('m' << 16) | ('s' << 8) | 'm'; + /* uint32_t gmsm = ('g' << 24) | ('m' << 16) | ('s' << 8) | 'm'; if (hnd->handle->numInts >= 2 && hnd->handle->data[hnd->handle->numFds] == gmsm) { - /* This UBWC flag was introduced in a5xx. */ - bool ubwc = hnd->handle->data[hnd->handle->numFds + 1] & 0x08000000; - out->modifier = ubwc ? DRM_FORMAT_MOD_QCOM_COMPRESSED : DRM_FORMAT_MOD_LINEAR; + } + */ + + /* TODO: Actually find a way to detect a Qualcomm vendor allocated buffer */ + /* This UBWC flag was introduced in a5xx. */ + bool ubwc = hnd->handle->data[hnd->handle->numFds + 1] & 0x08000000; + out->modifier = ubwc ? DRM_FORMAT_MOD_QCOM_COMPRESSED : DRM_FORMAT_MOD_LINEAR; #endif return 0;