Add Chromium-only Blender WebEngine parity work
This commit is contained in:
18
blender-5.2.0/intern/cycles/kernel/device/gpu/block_sizes.h
Normal file
18
blender-5.2.0/intern/cycles/kernel/device/gpu/block_sizes.h
Normal file
@@ -0,0 +1,18 @@
|
||||
/* SPDX-FileCopyrightText: 2017-2025 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef __HIP__
|
||||
# define GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE 1024
|
||||
# define GPU_PARALLEL_PREFIX_SUM_DEFAULT_BLOCK_SIZE 1024
|
||||
# define GPU_PARALLEL_SORTED_INDEX_DEFAULT_BLOCK_SIZE 1024
|
||||
#else
|
||||
# define GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE 512
|
||||
# define GPU_PARALLEL_PREFIX_SUM_DEFAULT_BLOCK_SIZE 512
|
||||
# define GPU_PARALLEL_SORTED_INDEX_DEFAULT_BLOCK_SIZE 512
|
||||
#endif
|
||||
|
||||
#define GPU_PARALLEL_SORTED_INDEX_INACTIVE_KEY (~0)
|
||||
#define GPU_PARALLEL_SORT_BLOCK_SIZE 1024
|
||||
173
blender-5.2.0/intern/cycles/kernel/device/gpu/image.h
Normal file
173
blender-5.2.0/intern/cycles/kernel/device/gpu/image.h
Normal file
@@ -0,0 +1,173 @@
|
||||
/* SPDX-FileCopyrightText: 2017-2022 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "kernel/globals.h"
|
||||
#include "kernel/util/image_2d.h"
|
||||
#include "util/defines.h"
|
||||
#include "util/types_image.h"
|
||||
|
||||
CCL_NAMESPACE_BEGIN
|
||||
|
||||
ccl_device_inline float frac(const float x, ccl_private int *ix)
|
||||
{
|
||||
int i = float_to_int(x) - ((x < 0.0f) ? 1 : 0);
|
||||
*ix = i;
|
||||
return x - (float)i;
|
||||
}
|
||||
|
||||
/* w0, w1, w2, and w3 are the four cubic B-spline basis functions. */
|
||||
ccl_device float cubic_w0(const float a)
|
||||
{
|
||||
return (1.0f / 6.0f) * (a * (a * (-a + 3.0f) - 3.0f) + 1.0f);
|
||||
}
|
||||
ccl_device float cubic_w1(const float a)
|
||||
{
|
||||
return (1.0f / 6.0f) * (a * a * (3.0f * a - 6.0f) + 4.0f);
|
||||
}
|
||||
ccl_device float cubic_w2(const float a)
|
||||
{
|
||||
return (1.0f / 6.0f) * (a * (a * (-3.0f * a + 3.0f) + 3.0f) + 1.0f);
|
||||
}
|
||||
ccl_device float cubic_w3(const float a)
|
||||
{
|
||||
return (1.0f / 6.0f) * (a * a * a);
|
||||
}
|
||||
|
||||
/* g0 and g1 are the two amplitude functions. */
|
||||
ccl_device float cubic_g0(const float a)
|
||||
{
|
||||
return cubic_w0(a) + cubic_w1(a);
|
||||
}
|
||||
ccl_device float cubic_g1(const float a)
|
||||
{
|
||||
return cubic_w2(a) + cubic_w3(a);
|
||||
}
|
||||
|
||||
/* h0 and h1 are the two offset functions */
|
||||
ccl_device float cubic_h0(const float a)
|
||||
{
|
||||
return (cubic_w1(a) / cubic_g0(a)) - 1.0f;
|
||||
}
|
||||
ccl_device float cubic_h1(const float a)
|
||||
{
|
||||
return (cubic_w3(a) / cubic_g1(a)) + 1.0f;
|
||||
}
|
||||
|
||||
/* Fast bicubic texture lookup using 4 bilinear lookups, adapted from CUDA samples. */
|
||||
template<typename T>
|
||||
ccl_device_noinline T kernel_image_interp_bicubic(const ccl_global KernelImageInfo &info,
|
||||
const float2 uv)
|
||||
{
|
||||
ccl_gpu_image_object_2D tex = (ccl_gpu_image_object_2D)info.data;
|
||||
|
||||
const float x = (uv.x * (float)info.width) - 0.5f;
|
||||
const float y = (uv.y * (float)info.height) - 0.5f;
|
||||
|
||||
const float px = floorf(x);
|
||||
const float py = floorf(y);
|
||||
const float fx = x - px;
|
||||
const float fy = y - py;
|
||||
|
||||
const float g0x = cubic_g0(fx);
|
||||
const float g1x = cubic_g1(fx);
|
||||
/* Note +0.5 offset to compensate for CUDA linear filtering convention. */
|
||||
const float x0 = (px + cubic_h0(fx) + 0.5f) * info.inv_width;
|
||||
const float x1 = (px + cubic_h1(fx) + 0.5f) * info.inv_width;
|
||||
const float y0 = (py + cubic_h0(fy) + 0.5f) * info.inv_height;
|
||||
const float y1 = (py + cubic_h1(fy) + 0.5f) * info.inv_height;
|
||||
|
||||
return cubic_g0(fy) * (g0x * ccl_gpu_image_object_read_2D<T>(tex, x0, y0) +
|
||||
g1x * ccl_gpu_image_object_read_2D<T>(tex, x1, y0)) +
|
||||
cubic_g1(fy) * (g0x * ccl_gpu_image_object_read_2D<T>(tex, x0, y1) +
|
||||
g1x * ccl_gpu_image_object_read_2D<T>(tex, x1, y1));
|
||||
}
|
||||
|
||||
ccl_device float4 kernel_image_interp(KernelGlobals kg,
|
||||
ccl_private ShaderData *sd,
|
||||
const int image_texture_id,
|
||||
dual2 uv)
|
||||
{
|
||||
if (image_texture_id == KERNEL_IMAGE_NONE) {
|
||||
return IMAGE_MISSING_RGBA;
|
||||
}
|
||||
const ccl_global KernelImageTexture &tex = kernel_data_fetch(image_textures, image_texture_id);
|
||||
const ccl_global KernelImageInfo *info;
|
||||
|
||||
float2 sample_uv;
|
||||
|
||||
if (tex.tile_descriptor_offset != KERNEL_TILE_LOAD_NONE) {
|
||||
/* Wrapping. */
|
||||
if (!kernel_image_tile_wrap(ExtensionType(tex.extension), uv.val)) {
|
||||
return zero_float4();
|
||||
}
|
||||
|
||||
/* Tile mapping */
|
||||
float2 xy = zero_float2();
|
||||
const KernelTileDescriptor tile_descriptor = kernel_image_tile_map(
|
||||
kg, sd, tex, image_texture_id, uv, xy);
|
||||
|
||||
if (!kernel_tile_descriptor_loaded(tile_descriptor)) {
|
||||
return (tile_descriptor == KERNEL_TILE_LOAD_FAILED) ? IMAGE_MISSING_RGBA : tex.average_color;
|
||||
}
|
||||
|
||||
info = &kernel_data_fetch(image_info, kernel_tile_descriptor_image_info_id(tile_descriptor));
|
||||
|
||||
/* Convert to normalized space again. */
|
||||
sample_uv = make_float2(xy.x * info->inv_width, xy.y * info->inv_height);
|
||||
}
|
||||
else {
|
||||
/* Full image sampling. */
|
||||
if (tex.image_info_id == KERNEL_IMAGE_NONE) {
|
||||
return IMAGE_MISSING_RGBA;
|
||||
}
|
||||
|
||||
info = &kernel_data_fetch(image_info, tex.image_info_id);
|
||||
sample_uv = uv.val;
|
||||
}
|
||||
|
||||
/* float4, byte4, ushort4 and half4 */
|
||||
const int texture_type = info->data_type;
|
||||
if (texture_type == IMAGE_DATA_TYPE_FLOAT4 || texture_type == IMAGE_DATA_TYPE_BYTE4 ||
|
||||
texture_type == IMAGE_DATA_TYPE_HALF4 || texture_type == IMAGE_DATA_TYPE_USHORT4)
|
||||
{
|
||||
if (info->interpolation == INTERPOLATION_CUBIC || info->interpolation == INTERPOLATION_SMART) {
|
||||
return kernel_image_interp_bicubic<float4>(*info, sample_uv);
|
||||
}
|
||||
else {
|
||||
ccl_gpu_image_object_2D tex = (ccl_gpu_image_object_2D)info->data;
|
||||
return ccl_gpu_image_object_read_2D<float4>(tex, sample_uv.x, sample_uv.y);
|
||||
}
|
||||
}
|
||||
/* float, byte and half */
|
||||
else {
|
||||
float f;
|
||||
|
||||
if (info->interpolation == INTERPOLATION_CUBIC || info->interpolation == INTERPOLATION_SMART) {
|
||||
f = kernel_image_interp_bicubic<float>(*info, sample_uv);
|
||||
}
|
||||
else {
|
||||
ccl_gpu_image_object_2D tex = (ccl_gpu_image_object_2D)info->data;
|
||||
f = ccl_gpu_image_object_read_2D<float>(tex, sample_uv.x, sample_uv.y);
|
||||
}
|
||||
|
||||
return make_float4(f, f, f, 1.0f);
|
||||
}
|
||||
}
|
||||
|
||||
ccl_device_forceinline float4 kernel_image_interp_with_udim(KernelGlobals kg,
|
||||
ccl_private ShaderData *sd,
|
||||
const int udim_id,
|
||||
dual2 uv)
|
||||
{
|
||||
const int image_texture_id = kernel_image_udim_map(kg, udim_id, uv.val);
|
||||
if (image_texture_id == KERNEL_IMAGE_NONE) {
|
||||
return IMAGE_MISSING_RGBA;
|
||||
}
|
||||
|
||||
return kernel_image_interp(kg, sd, image_texture_id, uv);
|
||||
}
|
||||
|
||||
CCL_NAMESPACE_END
|
||||
1366
blender-5.2.0/intern/cycles/kernel/device/gpu/kernel.h
Normal file
1366
blender-5.2.0/intern/cycles/kernel/device/gpu/kernel.h
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
||||
/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
CCL_NAMESPACE_BEGIN
|
||||
|
||||
/* Given an array of states, build an array of indices for which the states
|
||||
* are active.
|
||||
*
|
||||
* Shared memory requirement is `sizeof(int) * (number_of_warps + 1)`. */
|
||||
|
||||
#include "kernel/device/gpu/block_sizes.h"
|
||||
#include "util/atomic.h"
|
||||
|
||||
/* TODO: abstract more device differences, define `ccl_gpu_local_syncthreads`,
|
||||
* `ccl_gpu_thread_warp`, `ccl_gpu_warp_index`, `ccl_gpu_num_warps` for all devices
|
||||
* and keep device specific code in `compat.h`. */
|
||||
|
||||
#ifdef __KERNEL_ONEAPI__
|
||||
|
||||
template<typename IsActiveOp>
|
||||
void gpu_parallel_active_index_array_impl(const uint num_states,
|
||||
ccl_global int *ccl_restrict indices,
|
||||
ccl_global int *ccl_restrict num_indices,
|
||||
IsActiveOp is_active_op)
|
||||
{
|
||||
const sycl::nd_item<1> &item_id = sycl::ext::oneapi::this_work_item::get_nd_item<1>();
|
||||
const uint blocksize = item_id.get_local_range(0);
|
||||
|
||||
sycl::multi_ptr<int[GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE + 1],
|
||||
sycl::access::address_space::local_space>
|
||||
ptr = sycl::ext::oneapi::group_local_memory<
|
||||
int[GPU_PARALLEL_ACTIVE_INDEX_DEFAULT_BLOCK_SIZE + 1]>(item_id.get_group());
|
||||
int *warp_offset = *ptr;
|
||||
|
||||
/* NOTE(@nsirgien): Here we calculate the same value as below but
|
||||
* faster for DPC++ : seems CUDA converting "%", "/", "*" based calculations below into
|
||||
* something faster already but DPC++ doesn't, so it's better to use
|
||||
* direct request of needed parameters - switching from this computation to computation below
|
||||
* will cause 2.5x performance slowdown. */
|
||||
const uint thread_index = item_id.get_local_id(0);
|
||||
const uint thread_warp = item_id.get_sub_group().get_local_id();
|
||||
|
||||
const uint warp_index = item_id.get_sub_group().get_group_id();
|
||||
const uint num_warps = item_id.get_sub_group().get_group_range()[0];
|
||||
|
||||
const uint state_index = item_id.get_global_id(0);
|
||||
|
||||
/* Test if state corresponding to this thread is active. */
|
||||
const uint is_active = (state_index < num_states) ? is_active_op(state_index) : 0;
|
||||
#else /* !__KERNEL__ONEAPI__ */
|
||||
# ifndef __KERNEL_METAL__
|
||||
template<typename IsActiveOp>
|
||||
__device__
|
||||
# endif
|
||||
void
|
||||
gpu_parallel_active_index_array_impl(const uint num_states,
|
||||
ccl_global int *indices,
|
||||
ccl_global int *num_indices,
|
||||
# ifdef __KERNEL_METAL__
|
||||
const uint is_active,
|
||||
const uint blocksize,
|
||||
const int thread_index,
|
||||
const uint state_index,
|
||||
const int ccl_gpu_warp_size,
|
||||
const int thread_warp,
|
||||
const int warp_index,
|
||||
const int num_warps,
|
||||
threadgroup int *warp_offset)
|
||||
{
|
||||
# else
|
||||
IsActiveOp is_active_op)
|
||||
{
|
||||
extern ccl_gpu_shared int warp_offset[];
|
||||
|
||||
# ifndef __KERNEL_METAL__
|
||||
const uint blocksize = ccl_gpu_block_dim_x;
|
||||
# endif
|
||||
|
||||
const uint thread_index = ccl_gpu_thread_idx_x;
|
||||
const uint thread_warp = thread_index % ccl_gpu_warp_size;
|
||||
|
||||
const uint warp_index = thread_index / ccl_gpu_warp_size;
|
||||
const uint num_warps = blocksize / ccl_gpu_warp_size;
|
||||
|
||||
const uint state_index = ccl_gpu_block_idx_x * blocksize + thread_index;
|
||||
|
||||
/* Test if state corresponding to this thread is active. */
|
||||
const uint is_active = (state_index < num_states) ? is_active_op(state_index) : 0;
|
||||
# endif
|
||||
#endif /* !__KERNEL_ONEAPI__ */
|
||||
/* For each thread within a warp compute how many other active states precede it. */
|
||||
#ifdef __KERNEL_ONEAPI__
|
||||
const uint thread_offset = sycl::exclusive_scan_over_group(
|
||||
item_id.get_sub_group(), is_active, std::plus<>());
|
||||
#else
|
||||
const uint thread_offset = popcount(ccl_gpu_ballot(is_active) &
|
||||
ccl_gpu_thread_mask(thread_warp));
|
||||
#endif
|
||||
|
||||
/* Last thread in warp stores number of active states for each warp. */
|
||||
#ifdef __KERNEL_ONEAPI__
|
||||
if (thread_warp == item_id.get_sub_group().get_local_range()[0] - 1) {
|
||||
#else
|
||||
if (thread_warp == ccl_gpu_warp_size - 1) {
|
||||
#endif
|
||||
warp_offset[warp_index] = thread_offset + is_active;
|
||||
}
|
||||
|
||||
#ifdef __KERNEL_ONEAPI__
|
||||
/* NOTE(@nsirgien): For us here only local memory writing (warp_offset) is important,
|
||||
* so faster local barriers can be used. */
|
||||
ccl_gpu_local_syncthreads();
|
||||
#else
|
||||
ccl_gpu_syncthreads();
|
||||
#endif
|
||||
|
||||
/* Last thread in block converts per-warp sizes to offsets, increments global size of
|
||||
* index array and gets offset to write to. */
|
||||
if (thread_index == blocksize - 1) {
|
||||
/* TODO: parallelize this. */
|
||||
int offset = 0;
|
||||
for (int i = 0; i < num_warps; i++) {
|
||||
int num_active = warp_offset[i];
|
||||
warp_offset[i] = offset;
|
||||
offset += num_active;
|
||||
}
|
||||
|
||||
const uint block_num_active = warp_offset[warp_index] + thread_offset + is_active;
|
||||
warp_offset[num_warps] = atomic_fetch_and_add_uint32(num_indices, block_num_active);
|
||||
}
|
||||
|
||||
#ifdef __KERNEL_ONEAPI__
|
||||
/* NOTE(@nsirgien): For us here only important local memory writing (warp_offset),
|
||||
* so faster local barriers can be used. */
|
||||
ccl_gpu_local_syncthreads();
|
||||
#else
|
||||
ccl_gpu_syncthreads();
|
||||
#endif
|
||||
|
||||
/* Write to index array. */
|
||||
if (is_active) {
|
||||
const uint block_offset = warp_offset[num_warps];
|
||||
indices[block_offset + warp_offset[warp_index] + thread_offset] = state_index;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __KERNEL_METAL__
|
||||
|
||||
# define gpu_parallel_active_index_array(num_states, indices, num_indices, is_active_op) \
|
||||
const uint is_active = (ccl_gpu_global_id_x() < num_states) ? \
|
||||
is_active_op(ccl_gpu_global_id_x()) : \
|
||||
0; \
|
||||
gpu_parallel_active_index_array_impl(num_states, \
|
||||
indices, \
|
||||
num_indices, \
|
||||
is_active, \
|
||||
metal_local_size, \
|
||||
metal_local_id, \
|
||||
metal_global_id, \
|
||||
simdgroup_size, \
|
||||
simd_lane_index, \
|
||||
simd_group_index, \
|
||||
num_simd_groups, \
|
||||
(threadgroup int *)threadgroup_array)
|
||||
#elif defined(__KERNEL_ONEAPI__)
|
||||
|
||||
# define gpu_parallel_active_index_array(num_states, indices, num_indices, is_active_op) \
|
||||
gpu_parallel_active_index_array_impl(num_states, indices, num_indices, is_active_op)
|
||||
|
||||
#else
|
||||
|
||||
# define gpu_parallel_active_index_array(num_states, indices, num_indices, is_active_op) \
|
||||
gpu_parallel_active_index_array_impl(num_states, indices, num_indices, is_active_op)
|
||||
|
||||
#endif
|
||||
|
||||
CCL_NAMESPACE_END
|
||||
@@ -0,0 +1,36 @@
|
||||
/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
CCL_NAMESPACE_BEGIN
|
||||
|
||||
/* Parallel prefix sum.
|
||||
*
|
||||
* TODO: actually make this work in parallel.
|
||||
*
|
||||
* This is used for an array the size of the number of shaders in the scene
|
||||
* which is not usually huge, so might not be a significant bottleneck. */
|
||||
|
||||
#include "util/atomic.h"
|
||||
|
||||
__device__ void gpu_parallel_prefix_sum(const int global_id,
|
||||
ccl_global int *counter,
|
||||
ccl_global int *prefix_sum,
|
||||
const int num_values)
|
||||
{
|
||||
if (global_id != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int offset = 0;
|
||||
for (int i = 0; i < num_values; i++) {
|
||||
const int new_offset = offset + counter[i];
|
||||
prefix_sum[i] = offset;
|
||||
counter[i] = 0;
|
||||
offset = new_offset;
|
||||
}
|
||||
}
|
||||
|
||||
CCL_NAMESPACE_END
|
||||
@@ -0,0 +1,165 @@
|
||||
/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
CCL_NAMESPACE_BEGIN
|
||||
|
||||
/* Given an array of states, build an array of indices for which the states
|
||||
* are active and sorted by a given key. The prefix sum of the number of active
|
||||
* states per key must have already been computed.
|
||||
*
|
||||
* TODO: there may be ways to optimize this to avoid this many atomic ops? */
|
||||
|
||||
#include "kernel/device/gpu/block_sizes.h"
|
||||
#include "util/atomic.h"
|
||||
|
||||
#if defined(__KERNEL_LOCAL_ATOMIC_SORT__)
|
||||
|
||||
ccl_device_inline void gpu_parallel_sort_bucket_pass(const uint num_states,
|
||||
const uint partition_size,
|
||||
const uint max_shaders,
|
||||
const uint queued_kernel,
|
||||
ccl_global ushort *d_queued_kernel,
|
||||
ccl_global uint *d_shader_sort_key,
|
||||
ccl_global int *partition_key_offsets,
|
||||
ccl_gpu_shared int *buckets,
|
||||
const ushort local_id,
|
||||
const ushort local_size,
|
||||
const uint grid_id)
|
||||
{
|
||||
/* Zero the bucket sizes. */
|
||||
for (uint i = local_id; i < max_shaders; i += local_size) {
|
||||
atomic_store_local(&buckets[i], 0);
|
||||
}
|
||||
|
||||
# ifdef __KERNEL_ONEAPI__
|
||||
/* NOTE(@nsirgien): For us here only local memory writing (buckets) is important,
|
||||
* so faster local barriers can be used. */
|
||||
ccl_gpu_local_syncthreads();
|
||||
# else
|
||||
ccl_gpu_syncthreads();
|
||||
# endif
|
||||
|
||||
/* Determine bucket sizes within the partitions. */
|
||||
|
||||
const uint partition_start = partition_size * uint(grid_id);
|
||||
const uint partition_end = min(num_states, partition_start + partition_size);
|
||||
|
||||
for (int state_index = partition_start + uint(local_id); state_index < partition_end;
|
||||
state_index += uint(local_size))
|
||||
{
|
||||
ushort kernel_index = d_queued_kernel[state_index];
|
||||
if (kernel_index == queued_kernel) {
|
||||
uint key = d_shader_sort_key[state_index] % max_shaders;
|
||||
atomic_fetch_and_add_uint32_shared(&buckets[key], 1);
|
||||
}
|
||||
}
|
||||
|
||||
# ifdef __KERNEL_ONEAPI__
|
||||
/* NOTE(@nsirgien): For us here only local memory writing (buckets) is important,
|
||||
* so faster local barriers can be used. */
|
||||
ccl_gpu_local_syncthreads();
|
||||
# else
|
||||
ccl_gpu_syncthreads();
|
||||
# endif
|
||||
|
||||
/* Calculate the partition's local offsets from the prefix sum of bucket sizes. */
|
||||
|
||||
if (local_id == 0) {
|
||||
int offset = 0;
|
||||
for (int i = 0; i < max_shaders; i++) {
|
||||
partition_key_offsets[i + uint(grid_id) * (max_shaders + 1)] = offset;
|
||||
offset = offset + atomic_load_local(&buckets[i]);
|
||||
}
|
||||
|
||||
/* Store the number of active states in this partition. */
|
||||
partition_key_offsets[max_shaders + uint(grid_id) * (max_shaders + 1)] = offset;
|
||||
}
|
||||
}
|
||||
|
||||
ccl_device_inline void gpu_parallel_sort_write_pass(const uint num_states,
|
||||
const uint partition_size,
|
||||
const uint max_shaders,
|
||||
const uint queued_kernel,
|
||||
const int num_states_limit,
|
||||
ccl_global int *indices,
|
||||
ccl_global ushort *d_queued_kernel,
|
||||
ccl_global uint *d_shader_sort_key,
|
||||
ccl_global int *partition_key_offsets,
|
||||
ccl_gpu_shared int *local_offset,
|
||||
const ushort local_id,
|
||||
const ushort local_size,
|
||||
const uint grid_id)
|
||||
{
|
||||
/* Calculate each partition's global offset from the prefix sum of the active state counts per
|
||||
* partition. */
|
||||
|
||||
int partition_offset = 0;
|
||||
for (uint i = 0; i < grid_id; i++) {
|
||||
partition_offset += partition_key_offsets[max_shaders + i * (max_shaders + 1)];
|
||||
}
|
||||
|
||||
ccl_global int *key_offsets = partition_key_offsets + grid_id * (max_shaders + 1);
|
||||
for (uint i = local_id; i < max_shaders; i += local_size) {
|
||||
atomic_store_local(&local_offset[i], key_offsets[i] + partition_offset);
|
||||
}
|
||||
|
||||
# ifdef __KERNEL_ONEAPI__
|
||||
/* NOTE(@nsirgien): For us here only local memory writing (local_offset) is important,
|
||||
* so faster local barriers can be used. */
|
||||
ccl_gpu_local_syncthreads();
|
||||
# else
|
||||
ccl_gpu_syncthreads();
|
||||
# endif
|
||||
|
||||
/* Write the sorted active indices. */
|
||||
|
||||
const uint partition_start = partition_size * uint(grid_id);
|
||||
const uint partition_end = min(num_states, partition_start + partition_size);
|
||||
|
||||
for (int state_index = partition_start + uint(local_id); state_index < partition_end;
|
||||
state_index += uint(local_size))
|
||||
{
|
||||
ushort kernel_index = d_queued_kernel[state_index];
|
||||
if (kernel_index == queued_kernel) {
|
||||
uint key = d_shader_sort_key[state_index] % max_shaders;
|
||||
int index = atomic_fetch_and_add_uint32_shared(&local_offset[key], 1);
|
||||
if (index < num_states_limit) {
|
||||
indices[index] = state_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* __KERNEL_LOCAL_ATOMIC_SORT__ */
|
||||
|
||||
template<typename GetKeyOp>
|
||||
__device__ void gpu_parallel_sorted_index_array(const uint state_index,
|
||||
const uint num_states,
|
||||
const int num_states_limit,
|
||||
ccl_global int *indices,
|
||||
ccl_global int *num_indices,
|
||||
ccl_global int *key_counter,
|
||||
ccl_global int *key_prefix_sum,
|
||||
GetKeyOp get_key_op)
|
||||
{
|
||||
const int key = (state_index < num_states) ? get_key_op(state_index) :
|
||||
GPU_PARALLEL_SORTED_INDEX_INACTIVE_KEY;
|
||||
|
||||
if (key != GPU_PARALLEL_SORTED_INDEX_INACTIVE_KEY) {
|
||||
const uint index = atomic_fetch_and_add_uint32(&key_prefix_sum[key], 1);
|
||||
if (index < num_states_limit) {
|
||||
/* Assign state index. */
|
||||
indices[index] = state_index;
|
||||
}
|
||||
else {
|
||||
/* Can't process this state now, increase the counter again so that
|
||||
* it will be handled in another iteration. */
|
||||
atomic_fetch_and_add_uint32(&key_counter[key], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CCL_NAMESPACE_END
|
||||
@@ -0,0 +1,43 @@
|
||||
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0 */
|
||||
|
||||
#pragma once
|
||||
|
||||
CCL_NAMESPACE_BEGIN
|
||||
|
||||
/*
|
||||
* Utility functions for work stealing
|
||||
*/
|
||||
|
||||
/* Map global work index to tile, pixel X/Y and sample. */
|
||||
ccl_device_inline void get_work_pixel(const ccl_global KernelWorkTile *tile,
|
||||
const uint global_work_index,
|
||||
ccl_private uint *x,
|
||||
ccl_private uint *y,
|
||||
ccl_private uint *sample)
|
||||
{
|
||||
uint sample_offset, pixel_offset;
|
||||
|
||||
if (kernel_data.integrator.scrambling_distance < 0.9f) {
|
||||
/* Keep threads for the same sample together. */
|
||||
const uint tile_pixels = tile->w * tile->h;
|
||||
sample_offset = global_work_index / tile_pixels;
|
||||
pixel_offset = global_work_index - sample_offset * tile_pixels;
|
||||
}
|
||||
else {
|
||||
/* Keeping threads for the same pixel together.
|
||||
* Appears to improve performance by a few % on CUDA and OptiX. */
|
||||
sample_offset = global_work_index % tile->num_samples;
|
||||
pixel_offset = global_work_index / tile->num_samples;
|
||||
}
|
||||
|
||||
const uint y_offset = pixel_offset / tile->w;
|
||||
const uint x_offset = pixel_offset - y_offset * tile->w;
|
||||
|
||||
*x = tile->x + x_offset;
|
||||
*y = tile->y + y_offset;
|
||||
*sample = tile->start_sample + sample_offset;
|
||||
}
|
||||
|
||||
CCL_NAMESPACE_END
|
||||
Reference in New Issue
Block a user