Add Chromium-only Blender WebEngine parity work

This commit is contained in:
mes123456
2026-08-12 04:47:48 -04:00
commit 9fd26010f6
18225 changed files with 11622124 additions and 0 deletions

View File

@@ -0,0 +1,70 @@
# SPDX-FileCopyrightText: 2011-2026 Blender Foundation
#
# SPDX-License-Identifier: Apache-2.0
set(INC
../../..
)
set(INC_SYS
)
set(SRC_KERNEL_DEVICE_CPU
globals.cpp
kernel.cpp
kernel_avx2.cpp
)
set(SRC_KERNEL_DEVICE_CPU_HEADERS
bvh.h
compat.h
image.h
globals.h
kernel.h
kernel_arch.h
kernel_arch_impl.h
)
set(LIB
PUBLIC cycles_util
)
# -----------------------------------------------------------------------------
# CPU module.
include_directories(${INC})
include_directories(SYSTEM ${INC_SYS})
if(DEFINED CYCLES_KERNEL_FLAGS)
set_source_files_properties(kernel.cpp PROPERTIES COMPILE_FLAGS "${CYCLES_KERNEL_FLAGS}")
endif()
if(CXX_HAS_AVX2 AND CXX_HAS_F16C)
set_source_files_properties(kernel_avx2.cpp PROPERTIES COMPILE_FLAGS "${CYCLES_AVX2_F16C_FLAGS}")
endif()
# Warnings to avoid using doubles in the kernel.
if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_C_COMPILER_ID MATCHES "Clang")
add_check_cxx_compiler_flags(
CMAKE_CXX_FLAGS
_has_cxxflag_float_conversion "-Werror=float-conversion"
_has_cxxflag_double_promotion "-Werror=double-promotion"
)
unset(_has_cxxflag_float_conversion)
unset(_has_cxxflag_double_promotion)
endif()
if(WITH_CYCLES_OSL)
list(APPEND LIB
PUBLIC cycles_kernel_osl
)
endif()
cycles_add_library(cycles_kernel_cpu "${LIB}"
${SRC_KERNEL_DEVICE_CPU}
${SRC_KERNEL_DEVICE_CPU_HEADERS}
)
cycles_set_solution_folder(cycles_kernel_cpu)
source_group("device\\cpu" FILES ${SRC_KERNEL_DEVICE_CPU} ${SRC_KERNEL_DEVICE_CPU_HEADERS})

View File

@@ -0,0 +1,660 @@
/* SPDX-FileCopyrightText: 2021-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* CPU Embree implementation of ray-scene intersection. */
#pragma once
#include <embree4/rtcore_geometry.h>
#include <embree4/rtcore_ray.h>
#include <embree4/rtcore_scene.h>
#ifdef __KERNEL_ONEAPI__
# include "kernel/device/oneapi/compat.h"
# include "kernel/device/oneapi/globals.h"
#else
# include "kernel/device/cpu/compat.h"
# include "kernel/device/cpu/globals.h"
#endif
#include "kernel/bvh/intersect_filter.h"
#include "kernel/bvh/types.h"
#include "kernel/bvh/util.h"
#include "kernel/geom/object.h"
#include "kernel/integrator/state.h"
#include "kernel/integrator/state_util.h"
#include "kernel/sample/lcg.h"
CCL_NAMESPACE_BEGIN
#ifdef __KERNEL_ONEAPI__
using numhit_t = uint16_t;
#else
using numhit_t = uint32_t;
#endif
/* Before Embree 4.4, the so-called Traversable functionality was exposed through Scene API.
* So, in order to simplify code between different versions, we are defining the traversable class
* and calls for older Embree versions as well. */
#if RTC_VERSION < 40400
# define RTCTraversable RTCScene
# define rtcGetGeometryUserDataFromTraversable rtcGetGeometryUserDataFromScene
# define rtcTraversableIntersect1 rtcIntersect1
# define rtcTraversableOccluded1 rtcOccluded1
#endif
#ifdef __KERNEL_ONEAPI__
# define CYCLES_EMBREE_USED_FEATURES \
(kernel_handler.get_specialization_constant<oneapi_embree_features>())
#else
# define CYCLES_EMBREE_USED_FEATURES \
(RTCFeatureFlags)(RTC_FEATURE_FLAG_TRIANGLE | RTC_FEATURE_FLAG_INSTANCE | \
RTC_FEATURE_FLAG_FILTER_FUNCTION_IN_ARGUMENTS | RTC_FEATURE_FLAG_POINT | \
RTC_FEATURE_FLAG_MOTION_BLUR | RTC_FEATURE_FLAG_ROUND_CATMULL_ROM_CURVE | \
RTC_FEATURE_FLAG_FLAT_CATMULL_ROM_CURVE | \
RTC_FEATURE_FLAG_ROUND_LINEAR_CURVE)
#endif
#define EMBREE_IS_HAIR(x) (x & 1)
/* Intersection context. */
struct CCLFirstHitContext : public RTCRayQueryContext {
KernelGlobals kg;
/* For avoiding self intersections */
const Ray *ray;
};
struct CCLShadowContext : public RTCRayQueryContext {
#if defined(__KERNEL_ONEAPI__)
ONEAPIKernelContext *oneapi_kernel_context;
#else
KernelGlobals kg;
#endif
BVHShadowAllPayload *payload;
};
struct CCLLocalContext : public RTCRayQueryContext {
KernelGlobals kg;
const Ray *ray;
numhit_t max_hits;
int local_object_id;
LocalIntersection *local_isect;
uint *lcg_state;
bool is_sss;
};
struct CCLVolumeContext : public RTCRayQueryContext {
KernelGlobals kg;
const Ray *ray;
#ifdef __VOLUME_RECORD_ALL__
numhit_t max_hits;
#endif
numhit_t num_hits;
Intersection *vol_isect;
};
/* Utilities. */
ccl_device_inline void kernel_embree_setup_ray(const Ray &ray,
RTCRay &rtc_ray,
const uint visibility)
{
rtc_ray.org_x = ray.P.x;
rtc_ray.org_y = ray.P.y;
rtc_ray.org_z = ray.P.z;
rtc_ray.dir_x = ray.D.x;
rtc_ray.dir_y = ray.D.y;
rtc_ray.dir_z = ray.D.z;
rtc_ray.tnear = ray.tmin;
rtc_ray.tfar = ray.tmax;
rtc_ray.time = ray.time;
rtc_ray.mask = visibility;
}
ccl_device_inline void kernel_embree_setup_rayhit(const Ray &ray,
RTCRayHit &rayhit,
const uint visibility)
{
kernel_embree_setup_ray(ray, rayhit.ray, visibility);
rayhit.hit.geomID = RTC_INVALID_GEOMETRY_ID;
rayhit.hit.instID[0] = RTC_INVALID_GEOMETRY_ID;
}
ccl_device_inline int kernel_embree_get_hit_object(const RTCHit *hit)
{
return (hit->instID[0] != RTC_INVALID_GEOMETRY_ID ? hit->instID[0] : hit->geomID) / 2;
}
ccl_device_inline bool kernel_embree_is_self_intersection(const KernelGlobals kg,
const RTCHit *hit,
const Ray *ray,
const intptr_t prim_offset)
{
const int object = kernel_embree_get_hit_object(hit);
int prim;
if ((ray->self.object == object) || (ray->self.light_object == object)) {
prim = hit->primID + prim_offset;
}
else {
return false;
}
const bool is_hair = hit->geomID & 1;
if (is_hair) {
prim = kernel_data_fetch(curve_segments, prim).prim;
}
return intersection_skip_self_shadow(ray->self, object, prim);
}
ccl_device_inline void kernel_embree_convert_hit(KernelGlobals kg,
const RTCRay *ray,
const RTCHit *hit,
Intersection *isect,
const intptr_t prim_offset)
{
isect->t = ray->tfar;
isect->prim = hit->primID + prim_offset;
isect->object = kernel_embree_get_hit_object(hit);
const bool is_hair = hit->geomID & 1;
if (is_hair) {
const KernelCurveSegment segment = kernel_data_fetch(curve_segments, isect->prim);
isect->type = segment.type;
isect->prim = segment.prim;
isect->u = hit->u;
isect->v = hit->v;
}
else {
isect->type = kernel_data_fetch(objects, isect->object).primitive_type;
isect->u = hit->u;
isect->v = hit->v;
}
}
ccl_device_inline void kernel_embree_convert_hit(KernelGlobals kg,
const RTCRay *ray,
const RTCHit *hit,
Intersection *isect)
{
intptr_t prim_offset;
if (hit->instID[0] != RTC_INVALID_GEOMETRY_ID) {
RTCTraversable inst_scene = (RTCTraversable)rtcGetGeometryUserDataFromTraversable(
kernel_data.device_bvh, hit->instID[0]);
prim_offset = intptr_t(rtcGetGeometryUserDataFromTraversable(inst_scene, hit->geomID));
}
else {
prim_offset = intptr_t(
rtcGetGeometryUserDataFromTraversable(kernel_data.device_bvh, hit->geomID));
}
kernel_embree_convert_hit(kg, ray, hit, isect, prim_offset);
}
ccl_device_inline void kernel_embree_convert_sss_hit(KernelGlobals kg,
const RTCRay *ray,
const RTCHit *hit,
Intersection *isect,
const int object,
const intptr_t prim_offset)
{
isect->u = hit->u;
isect->v = hit->v;
isect->t = ray->tfar;
isect->prim = hit->primID + prim_offset;
isect->object = object;
isect->type = kernel_data_fetch(objects, object).primitive_type;
}
/* Ray filter functions. */
/* This gets called by Embree at every valid ray/object intersection.
* Things like recording subsurface or shadow hits for later evaluation
* as well as filtering for volume objects happen here.
* Cycles' own BVH does that directly inside the traversal calls. */
ccl_device_forceinline void kernel_embree_filter_intersection_func_impl(
const RTCFilterFunctionNArguments *args)
{
/* Current implementation in Cycles assumes only single-ray intersection queries. */
assert(args->N == 1);
RTCHit *hit = (RTCHit *)args->hit;
CCLFirstHitContext *ctx = (CCLFirstHitContext *)(args->context);
#ifdef __KERNEL_ONEAPI__
KernelGlobalsGPU *kg = nullptr;
#else
const ThreadKernelGlobalsCPU *kg = ctx->kg;
#endif
const Ray *cray = ctx->ray;
if (kernel_embree_is_self_intersection(
kg, hit, cray, reinterpret_cast<intptr_t>(args->geometryUserPtr)))
{
*args->valid = 0;
return;
}
#ifdef __SHADOW_LINKING__
if (intersection_skip_shadow_link(kg, cray->self, kernel_embree_get_hit_object(hit))) {
*args->valid = 0;
return;
}
#endif
}
/* This gets called by Embree at every valid ray/object intersection.
* Things like recording subsurface or shadow hits for later evaluation
* as well as filtering for volume objects happen here.
* Cycles' own BVH does that directly inside the traversal calls.
*/
ccl_device_forceinline void kernel_embree_filter_occluded_shadow_all_func_impl(
const RTCFilterFunctionNArguments *args)
{
/* Current implementation in Cycles assumes only single-ray intersection queries. */
assert(args->N == 1);
const RTCRay *ray = (RTCRay *)args->ray;
const RTCHit *hit = (RTCHit *)args->hit;
CCLShadowContext *ctx = (CCLShadowContext *)(args->context);
BVHShadowAllPayload &payload = *ctx->payload;
#ifdef __KERNEL_ONEAPI__
KernelGlobalsGPU *kg = nullptr;
#else
const ThreadKernelGlobalsCPU *kg = ctx->kg;
#endif
Intersection isect;
kernel_embree_convert_hit(
kg, ray, hit, &isect, reinterpret_cast<intptr_t>(args->geometryUserPtr));
if (!bvh_shadow_all_anyhit_filter<ISECT_TEST_ALL & ~ISECT_TEST_VISIBILITY_FLAG>(
kg, payload.state, payload, payload.base.ray_self, 0, isect))
{
return;
}
*args->valid = 0;
}
ccl_device_forceinline void kernel_embree_filter_occluded_local_func_impl(
const RTCFilterFunctionNArguments *args)
{
/* Current implementation in Cycles assumes only single-ray intersection queries. */
assert(args->N == 1);
const RTCRay *ray = (RTCRay *)args->ray;
RTCHit *hit = (RTCHit *)args->hit;
CCLLocalContext *ctx = (CCLLocalContext *)(args->context);
#ifdef __KERNEL_ONEAPI__
KernelGlobalsGPU *kg = nullptr;
#else
const ThreadKernelGlobalsCPU *kg = ctx->kg;
#endif
const Ray *cray = ctx->ray;
/* Check if it's hitting the correct object. */
Intersection current_isect;
if (ctx->is_sss) {
kernel_embree_convert_sss_hit(kg,
ray,
hit,
&current_isect,
ctx->local_object_id,
reinterpret_cast<intptr_t>(args->geometryUserPtr));
}
else {
kernel_embree_convert_hit(
kg, ray, hit, &current_isect, reinterpret_cast<intptr_t>(args->geometryUserPtr));
if (ctx->local_object_id != current_isect.object) {
/* This tells Embree to continue tracing. */
*args->valid = 0;
return;
}
}
if (intersection_skip_self_local(cray->self, current_isect.prim)) {
*args->valid = 0;
return;
}
/* No intersection information requested, just return a hit. */
if (ctx->max_hits == 0) {
return;
}
/* Ignore curves. */
if (EMBREE_IS_HAIR(hit->geomID)) {
/* This tells Embree to continue tracing. */
*args->valid = 0;
return;
}
LocalIntersection *local_isect = ctx->local_isect;
int hit_idx = 0;
if (ctx->lcg_state) {
/* See triangle_intersect_subsurface() for the native equivalent. */
for (int i = min((int)ctx->max_hits, local_isect->num_hits) - 1; i >= 0; --i) {
if (local_isect->hits[i].t == ray->tfar) {
/* This tells Embree to continue tracing. */
*args->valid = 0;
return;
}
}
local_isect->num_hits++;
if (local_isect->num_hits <= ctx->max_hits) {
hit_idx = local_isect->num_hits - 1;
}
else {
/* reservoir sampling: if we are at the maximum number of
* hits, randomly replace element or skip it */
hit_idx = lcg_step_uint(ctx->lcg_state) % local_isect->num_hits;
if (hit_idx >= ctx->max_hits) {
/* This tells Embree to continue tracing. */
*args->valid = 0;
return;
}
}
}
else {
/* Record closest intersection only. */
if (local_isect->num_hits && current_isect.t > local_isect->hits[0].t) {
*args->valid = 0;
return;
}
local_isect->num_hits = 1;
}
/* record intersection */
local_isect->hits[hit_idx] = current_isect;
local_isect->Ng[hit_idx] = normalize(make_float3(hit->Ng_x, hit->Ng_y, hit->Ng_z));
/* This tells Embree to continue tracing. */
*args->valid = 0;
}
ccl_device_forceinline void kernel_embree_filter_occluded_volume_all_func_impl(
const RTCFilterFunctionNArguments *args)
{
/* Current implementation in Cycles assumes only single-ray intersection queries. */
assert(args->N == 1);
const RTCRay *ray = (RTCRay *)args->ray;
RTCHit *hit = (RTCHit *)args->hit;
CCLVolumeContext *ctx = (CCLVolumeContext *)(args->context);
#ifdef __KERNEL_ONEAPI__
KernelGlobalsGPU *kg = nullptr;
#else
const ThreadKernelGlobalsCPU *kg = ctx->kg;
#endif
const Ray *cray = ctx->ray;
#ifdef __VOLUME_RECORD_ALL__
/* Append the intersection to the end of the array. */
if (ctx->num_hits < ctx->max_hits) {
#endif
Intersection current_isect;
kernel_embree_convert_hit(
kg, ray, hit, &current_isect, reinterpret_cast<intptr_t>(args->geometryUserPtr));
if (bvh_volume_anyhit_triangle_filter<false>(
kg, current_isect.object, current_isect.prim, cray->self, 0))
{
*args->valid = 0;
return;
}
Intersection *isect = &ctx->vol_isect[ctx->num_hits];
++ctx->num_hits;
*isect = current_isect;
#ifdef __VOLUME_RECORD_ALL__
/* This tells Embree to continue tracing. */
*args->valid = 0;
}
#endif
}
#ifdef __KERNEL_ONEAPI__
/* Static wrappers so we can call the callbacks from out side the ONEAPIKernelContext class */
RTC_SYCL_INDIRECTLY_CALLABLE static void ccl_always_inline
kernel_embree_filter_intersection_func_static(const RTCFilterFunctionNArguments *args)
{
RTCHit *hit = (RTCHit *)args->hit;
CCLFirstHitContext *ctx = (CCLFirstHitContext *)(args->context);
ONEAPIKernelContext *context = static_cast<ONEAPIKernelContext *>(ctx->kg);
context->kernel_embree_filter_intersection_func_impl(args);
}
RTC_SYCL_INDIRECTLY_CALLABLE static void ccl_always_inline
kernel_embree_filter_occluded_shadow_all_func_static(const RTCFilterFunctionNArguments *args)
{
RTCHit *hit = (RTCHit *)args->hit;
CCLShadowContext *ctx = (CCLShadowContext *)(args->context);
ONEAPIKernelContext *context = ctx->oneapi_kernel_context;
context->kernel_embree_filter_occluded_shadow_all_func_impl(args);
}
RTC_SYCL_INDIRECTLY_CALLABLE static void ccl_always_inline
kernel_embree_filter_occluded_local_func_static(const RTCFilterFunctionNArguments *args)
{
RTCHit *hit = (RTCHit *)args->hit;
CCLLocalContext *ctx = (CCLLocalContext *)(args->context);
ONEAPIKernelContext *context = static_cast<ONEAPIKernelContext *>(ctx->kg);
context->kernel_embree_filter_occluded_local_func_impl(args);
}
RTC_SYCL_INDIRECTLY_CALLABLE static void ccl_always_inline
kernel_embree_filter_occluded_volume_all_func_static(const RTCFilterFunctionNArguments *args)
{
RTCHit *hit = (RTCHit *)args->hit;
CCLVolumeContext *ctx = (CCLVolumeContext *)(args->context);
ONEAPIKernelContext *context = static_cast<ONEAPIKernelContext *>(ctx->kg);
context->kernel_embree_filter_occluded_volume_all_func_impl(args);
}
# define kernel_embree_filter_intersection_func \
ONEAPIKernelContext::kernel_embree_filter_intersection_func_static
# define kernel_embree_filter_occluded_shadow_all_func \
ONEAPIKernelContext::kernel_embree_filter_occluded_shadow_all_func_static
# define kernel_embree_filter_occluded_local_func \
ONEAPIKernelContext::kernel_embree_filter_occluded_local_func_static
# define kernel_embree_filter_occluded_volume_all_func \
ONEAPIKernelContext::kernel_embree_filter_occluded_volume_all_func_static
#else
# define kernel_embree_filter_intersection_func kernel_embree_filter_intersection_func_impl
# define kernel_embree_filter_occluded_shadow_all_func \
kernel_embree_filter_occluded_shadow_all_func_impl
# define kernel_embree_filter_occluded_local_func kernel_embree_filter_occluded_local_func_impl
# define kernel_embree_filter_occluded_volume_all_func \
kernel_embree_filter_occluded_volume_all_func_impl
#endif
/* Scene intersection. */
ccl_device_intersect bool kernel_embree_intersect(KernelGlobals kg,
const ccl_private Ray *ray,
const uint visibility,
ccl_private Intersection *isect)
{
isect->t = ray->tmax;
CCLFirstHitContext ctx;
rtcInitRayQueryContext(&ctx);
#ifdef __KERNEL_ONEAPI__
/* NOTE(sirgienko): Cycles GPU back-ends passes nullptr to KernelGlobals and
* uses global device allocation (CUDA, Optix, HIP) or passes all needed data
* as a class context (Metal, oneAPI). So we need to pass this context here
* in order to have an access to it later in Embree filter functions on GPU. */
ctx.kg = (KernelGlobals)this;
#else
ctx.kg = kg;
#endif
RTCRayHit ray_hit;
ctx.ray = ray;
kernel_embree_setup_rayhit(*ray, ray_hit, visibility);
RTCIntersectArguments args;
rtcInitIntersectArguments(&args);
args.filter = reinterpret_cast<RTCFilterFunctionN>(kernel_embree_filter_intersection_func);
args.feature_mask = CYCLES_EMBREE_USED_FEATURES;
args.context = &ctx;
rtcTraversableIntersect1(kernel_data.device_bvh, &ray_hit, &args);
if (ray_hit.hit.geomID == RTC_INVALID_GEOMETRY_ID ||
ray_hit.hit.primID == RTC_INVALID_GEOMETRY_ID)
{
return false;
}
kernel_embree_convert_hit(kg, &ray_hit.ray, &ray_hit.hit, isect);
return true;
}
#ifdef __BVH_LOCAL__
ccl_device_intersect bool kernel_embree_intersect_local(KernelGlobals kg,
const ccl_private Ray *ray,
ccl_private LocalIntersection *local_isect,
const int local_object,
ccl_private uint *lcg_state,
const int max_hits)
{
const bool has_bvh = !(kernel_data_fetch(object_flag, local_object) &
SD_OBJECT_TRANSFORM_APPLIED);
CCLLocalContext ctx;
rtcInitRayQueryContext(&ctx);
# ifdef __KERNEL_ONEAPI__
/* NOTE(sirgienko): Cycles GPU back-ends passes nullptr to KernelGlobals and
* uses global device allocation (CUDA, Optix, HIP) or passes all needed data
* as a class context (Metal, oneAPI). So we need to pass this context here
* in order to have an access to it later in Embree filter functions on GPU. */
ctx.kg = (KernelGlobals)this;
# else
ctx.kg = kg;
# endif
ctx.is_sss = has_bvh;
ctx.lcg_state = lcg_state;
ctx.max_hits = max_hits;
ctx.ray = ray;
ctx.local_isect = local_isect;
if (local_isect) {
local_isect->num_hits = 0;
}
ctx.local_object_id = local_object;
RTCRay rtc_ray;
kernel_embree_setup_ray(*ray, rtc_ray, PATH_RAY_VISIBILITY_ALL);
RTCOccludedArguments args;
rtcInitOccludedArguments(&args);
args.filter = reinterpret_cast<RTCFilterFunctionN>(kernel_embree_filter_occluded_local_func);
args.feature_mask = CYCLES_EMBREE_USED_FEATURES;
args.context = &ctx;
/* If this object has its own BVH, use it. */
if (has_bvh) {
float3 P = ray->P;
float3 dir = ray->D;
float3 idir = ray->D;
# ifdef __OBJECT_MOTION__
bvh_instance_motion_push(kg, local_object, ray, &P, &dir, &idir);
# else
bvh_instance_push(kg, local_object, ray, &P, &dir, &idir);
# endif
rtc_ray.org_x = P.x;
rtc_ray.org_y = P.y;
rtc_ray.org_z = P.z;
rtc_ray.dir_x = dir.x;
rtc_ray.dir_y = dir.y;
rtc_ray.dir_z = dir.z;
rtc_ray.tnear = ray->tmin;
rtc_ray.tfar = ray->tmax;
RTCTraversable scene = (RTCTraversable)rtcGetGeometryUserDataFromTraversable(
kernel_data.device_bvh, local_object * 2);
kernel_assert(scene);
if (scene) {
rtcTraversableOccluded1(scene, &rtc_ray, &args);
}
}
else {
rtcTraversableOccluded1(kernel_data.device_bvh, &rtc_ray, &args);
}
/* rtcOccluded1 sets tfar to -inf if a hit was found. */
return (local_isect && local_isect->num_hits > 0) || (rtc_ray.tfar < 0);
}
#endif
#ifdef __TRANSPARENT_SHADOWS__
ccl_device_intersect void kernel_embree_intersect_shadow_all(KernelGlobals kg,
const ccl_private Ray *ray,
BVHShadowAllPayload &payload)
{
CCLShadowContext ctx;
rtcInitRayQueryContext(&ctx);
# if defined(__KERNEL_ONEAPI__)
ctx.oneapi_kernel_context = this;
# else
ctx.kg = kg;
# endif
ctx.payload = &payload;
RTCRay rtc_ray;
kernel_embree_setup_ray(*ray, rtc_ray, payload.base.ray_visibility);
RTCOccludedArguments args;
rtcInitOccludedArguments(&args);
args.filter = reinterpret_cast<RTCFilterFunctionN>(
kernel_embree_filter_occluded_shadow_all_func);
args.feature_mask = CYCLES_EMBREE_USED_FEATURES;
args.context = &ctx;
rtcTraversableOccluded1(kernel_data.device_bvh, &rtc_ray, &args);
}
#endif
#ifdef __VOLUME__
ccl_device_intersect uint kernel_embree_intersect_volume(KernelGlobals kg,
const ccl_private Ray *ray,
ccl_private Intersection *isect,
# ifdef __VOLUME_RECORD_ALL__
const uint max_hits,
# endif
const uint visibility)
{
CCLVolumeContext ctx;
rtcInitRayQueryContext(&ctx);
# ifdef __KERNEL_ONEAPI__
/* NOTE(sirgienko) Cycles GPU back-ends passes nullptr to KernelGlobals and
* uses global device allocation (CUDA, Optix, HIP) or passes all needed data
* as a class context (Metal, oneAPI). So we need to pass this context here
* in order to have an access to it later in Embree filter functions on GPU. */
ctx.kg = (KernelGlobals)this;
# else
ctx.kg = kg;
# endif
ctx.vol_isect = isect;
# ifdef __VOLUME_RECORD_ALL__
ctx.max_hits = numhit_t(max_hits);
# endif
ctx.num_hits = numhit_t(0);
ctx.ray = ray;
RTCRay rtc_ray;
kernel_embree_setup_ray(*ray, rtc_ray, visibility);
RTCOccludedArguments args;
rtcInitOccludedArguments(&args);
args.filter = reinterpret_cast<RTCFilterFunctionN>(
kernel_embree_filter_occluded_volume_all_func);
args.feature_mask = CYCLES_EMBREE_USED_FEATURES;
args.context = &ctx;
rtcTraversableOccluded1(kernel_data.device_bvh, &rtc_ray, &args);
return ctx.num_hits;
}
#endif
CCL_NAMESPACE_END

View File

@@ -0,0 +1,22 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
#pragma once
/* Release kernel has too much false-positive maybe-uninitialized warnings,
* which makes it possible to miss actual warnings.
*/
#if (defined(__GNUC__) && !defined(__clang__)) && defined(NDEBUG)
# pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
# pragma GCC diagnostic ignored "-Wuninitialized"
#endif
CCL_NAMESPACE_BEGIN
/* Assertions inside the kernel only work for the CPU device, so we wrap it in
* a macro which is empty for other devices */
#define kernel_assert(cond) assert(cond)
CCL_NAMESPACE_END

View File

@@ -0,0 +1,43 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
#include "kernel/device/cpu/globals.h"
#include "kernel/osl/globals.h"
#include "util/guiding.h" // IWYU pragma: keep
#include "util/profiling.h"
CCL_NAMESPACE_BEGIN
ThreadKernelGlobalsCPU::ThreadKernelGlobalsCPU(const KernelGlobalsCPU &kernel_globals,
OSLGlobals *osl_globals,
Profiler &cpu_profiler,
const int thread_index)
: KernelGlobalsCPU(kernel_globals),
#ifdef WITH_OSL
osl(osl_globals, thread_index),
#endif
cpu_profiler_(cpu_profiler)
{
#ifndef WITH_OSL
(void)thread_index;
(void)osl_globals;
#endif
#if defined(WITH_PATH_GUIDING)
opgl_path_segment_storage = make_unique<openpgl::cpp::PathSegmentStorage>();
#endif
}
void ThreadKernelGlobalsCPU::start_profiling()
{
cpu_profiler_.add_state(&profiler);
}
void ThreadKernelGlobalsCPU::stop_profiling()
{
cpu_profiler_.remove_state(&profiler);
}
CCL_NAMESPACE_END

View File

@@ -0,0 +1,113 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* Constant Globals */
#pragma once
#include "kernel/types.h"
#include "kernel/util/profiler.h"
#ifdef __OSL__
# include "kernel/osl/globals.h"
#endif
#include "util/guiding.h" // IWYU pragma: keep
#include "util/types_image.h" // IWYU pragma: keep
#include "util/unique_ptr.h"
CCL_NAMESPACE_BEGIN
struct OSLGlobals;
/* On the CPU, we pass along the struct KernelGlobals to nearly everywhere in
* the kernel, to access constant data. These are all stored as flat arrays.
* these are really just standard arrays. We can't use actually globals because
* multiple renders may be running inside the same process. */
/* Array for kernel data, with size to be able to assert on invalid data access. */
template<typename T> struct kernel_array {
const ccl_always_inline T &fetch(const int index) const
{
kernel_assert(index >= 0 && index < width);
return data[index];
}
ccl_always_inline void write(const int index, const T &value) const
{
data[index] = value;
}
T *data = nullptr;
int width = 0;
};
/* Constant globals shared between all threads. */
struct KernelGlobalsCPU {
#define KERNEL_DATA_ARRAY(type, name) kernel_array<const type> name;
#define KERNEL_DATA_ARRAY_WRITABLE(type, name) kernel_array<type> name;
#include "kernel/data_arrays.h"
KernelData data = {};
KernelImageLoadRequestedCPU image_load_requested_cpu;
ProfilingState profiler;
};
/* Per-thread global state.
*
* To avoid pointer indirection, the constant globals are copied to each thread.
*
* This may not be ideal for cache pressure. Alternative would be to pass an
* additional thread index to every function, and potentially to make the shared
* part an actual global variable. That would match the GPU more closely, but
* also require mutex locks for multiple Cycles instances. */
struct ThreadKernelGlobalsCPU : public KernelGlobalsCPU {
ThreadKernelGlobalsCPU(const KernelGlobalsCPU &kernel_globals,
OSLGlobals *osl_globals_memory,
Profiler &cpu_profiler,
const int thread_index);
ThreadKernelGlobalsCPU(ThreadKernelGlobalsCPU &other) = delete;
ThreadKernelGlobalsCPU(ThreadKernelGlobalsCPU &&other) noexcept = default;
ThreadKernelGlobalsCPU &operator=(const ThreadKernelGlobalsCPU &other) = delete;
ThreadKernelGlobalsCPU &operator=(ThreadKernelGlobalsCPU &&other) = delete;
void start_profiling();
void stop_profiling();
#ifdef __OSL__
OSLThreadData osl;
#endif
#if defined(__PATH_GUIDING__)
/* Pointers to shared global data structures. */
openpgl::cpp::SampleStorage *opgl_sample_data_storage = nullptr;
openpgl::cpp::Field *opgl_guiding_field = nullptr;
/* Local data structures owned by the thread. */
unique_ptr<openpgl::cpp::PathSegmentStorage> opgl_path_segment_storage;
unique_ptr<openpgl::cpp::SurfaceSamplingDistribution> opgl_surface_sampling_distribution;
unique_ptr<openpgl::cpp::VolumeSamplingDistribution> opgl_volume_sampling_distribution;
#endif
protected:
Profiler &cpu_profiler_;
};
using KernelGlobals = const ThreadKernelGlobalsCPU *;
/* Abstraction macros */
#define kernel_data_fetch(name, index) (kg->name.fetch(index))
#define kernel_data_write(name, index, value) (kg->name.write(index, value))
#define kernel_data_array(name) (kg->name.data)
#define kernel_data (kg->data)
#if defined(WITH_PATH_GUIDING)
# define guiding_guiding_field kg->opgl_guiding_field
# define guiding_ssd kg->opgl_surface_sampling_distribution
# define guiding_vsd kg->opgl_volume_sampling_distribution
#endif
CCL_NAMESPACE_END

View File

@@ -0,0 +1,418 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
#pragma once
#include "kernel/device/cpu/compat.h"
#include "kernel/device/cpu/globals.h"
#include "kernel/util/image_2d.h"
#include "util/defines.h"
#include "util/half.h"
#include "util/types_image.h"
CCL_NAMESPACE_BEGIN
/* Make template functions private so symbols don't conflict between kernels with different
* instruction sets. */
namespace {
#define SET_CUBIC_SPLINE_WEIGHTS(u, t) \
{ \
u[0] = (((-1.0f / 6.0f) * t + 0.5f) * t - 0.5f) * t + (1.0f / 6.0f); \
u[1] = ((0.5f * t - 1.0f) * t) * t + (2.0f / 3.0f); \
u[2] = ((-0.5f * t + 0.5f) * t + 0.5f) * t + (1.0f / 6.0f); \
u[3] = (1.0f / 6.0f) * t * t * t; \
} \
(void)0
ccl_device_inline float frac(const float x, int *ix)
{
int i = float_to_int(x) - ((x < 0.0f) ? 1 : 0);
*ix = i;
return x - (float)i;
}
template<typename TexT, typename OutT = float4> struct ImageInterpolator {
static ccl_always_inline OutT zero()
{
if constexpr (std::is_same_v<OutT, float4>) {
return zero_float4();
}
else {
return 0.0f;
}
}
static ccl_always_inline float4 read(const float4 r)
{
return r;
}
static ccl_always_inline float4 read(const uchar4 r)
{
const float f = 1.0f / 255.0f;
return make_float4(r.x * f, r.y * f, r.z * f, r.w * f);
}
static ccl_always_inline float read(const uchar r)
{
return r * (1.0f / 255.0f);
}
static ccl_always_inline float read(const float r)
{
return r;
}
static ccl_always_inline float4 read(half4 r)
{
return half4_to_float4_image(r);
}
static ccl_always_inline float read(half r)
{
return half_to_float_image(r);
}
static ccl_always_inline float read(const uint16_t r)
{
return r * (1.0f / 65535.0f);
}
static ccl_always_inline float4 read(ushort4 r)
{
const float f = 1.0f / 65535.0f;
return make_float4(r.x * f, r.y * f, r.z * f, r.w * f);
}
/* Read 2D Texture Data
* Does not check if data request is in bounds. */
static ccl_always_inline OutT
read(const TexT *data, const int x, int y, const int width, const int /*height*/)
{
return read(data[y * width + x]);
}
/* Read 2D Texture Data Clip
* Returns transparent black if data request is out of bounds. */
static ccl_always_inline OutT
read_clip(const TexT *data, const int x, int y, const int width, const int height)
{
if (x < 0 || x >= width || y < 0 || y >= height) {
return zero();
}
return read(data[y * width + x]);
}
static ccl_always_inline int wrap_periodic(int x, const int width)
{
x %= width;
if (x < 0) {
x += width;
}
return x;
}
static ccl_always_inline int wrap_clamp(const int x, const int width)
{
return clamp(x, 0, width - 1);
}
static ccl_always_inline int wrap_mirror(const int x, const int width)
{
const int m = abs(x + (x < 0)) % (2 * width);
if (m >= width) {
return 2 * width - m - 1;
}
return m;
}
/* ******** 2D interpolation ******** */
static ccl_always_inline OutT interp_closest(const KernelImageInfo &info, const float x, float y)
{
const int width = info.width;
const int height = info.height;
int ix, iy;
frac(x, &ix);
frac(y, &iy);
switch (info.extension) {
case EXTENSION_REPEAT:
ix = wrap_periodic(ix, width);
iy = wrap_periodic(iy, height);
break;
case EXTENSION_CLIP:
/* No samples are inside the clip region. */
if (ix < 0 || ix >= width || iy < 0 || iy >= height) {
return zero();
}
break;
case EXTENSION_EXTEND:
ix = wrap_clamp(ix, width);
iy = wrap_clamp(iy, height);
break;
case EXTENSION_MIRROR:
ix = wrap_mirror(ix, width);
iy = wrap_mirror(iy, height);
break;
default:
kernel_assert(0);
return zero();
}
const TexT *data = (const TexT *)info.data;
return read(data, ix, iy, width, height);
}
static ccl_always_inline OutT interp_linear(const KernelImageInfo &info, const float x, float y)
{
const int width = info.width;
const int height = info.height;
/* A -0.5 offset is used to center the linear samples around the sample point. */
int ix, iy;
int nix, niy;
const float tx = frac(x - 0.5f, &ix);
const float ty = frac(y - 0.5f, &iy);
const TexT *data = (const TexT *)info.data;
switch (info.extension) {
case EXTENSION_REPEAT:
ix = wrap_periodic(ix, width);
nix = wrap_periodic(ix + 1, width);
iy = wrap_periodic(iy, height);
niy = wrap_periodic(iy + 1, height);
break;
case EXTENSION_CLIP:
/* No linear samples are inside the clip region. */
if (ix < -1 || ix >= width || iy < -1 || iy >= height) {
return zero();
}
nix = ix + 1;
niy = iy + 1;
return (1.0f - ty) * (1.0f - tx) * read_clip(data, ix, iy, width, height) +
(1.0f - ty) * tx * read_clip(data, nix, iy, width, height) +
ty * (1.0f - tx) * read_clip(data, ix, niy, width, height) +
ty * tx * read_clip(data, nix, niy, width, height);
case EXTENSION_EXTEND:
nix = wrap_clamp(ix + 1, width);
ix = wrap_clamp(ix, width);
niy = wrap_clamp(iy + 1, height);
iy = wrap_clamp(iy, height);
break;
case EXTENSION_MIRROR:
nix = wrap_mirror(ix + 1, width);
ix = wrap_mirror(ix, width);
niy = wrap_mirror(iy + 1, height);
iy = wrap_mirror(iy, height);
break;
default:
kernel_assert(0);
return zero();
}
return (1.0f - ty) * (1.0f - tx) * read(data, ix, iy, width, height) +
(1.0f - ty) * tx * read(data, nix, iy, width, height) +
ty * (1.0f - tx) * read(data, ix, niy, width, height) +
ty * tx * read(data, nix, niy, width, height);
}
static ccl_always_inline OutT interp_cubic(const KernelImageInfo &info, const float x, float y)
{
const int width = info.width;
const int height = info.height;
/* A -0.5 offset is used to center the cubic samples around the sample point. */
int ix, iy;
const float tx = frac(x - 0.5f, &ix);
const float ty = frac(y - 0.5f, &iy);
int pix, piy;
int nix, niy;
int nnix, nniy;
switch (info.extension) {
case EXTENSION_REPEAT:
ix = wrap_periodic(ix, width);
pix = wrap_periodic(ix - 1, width);
nix = wrap_periodic(ix + 1, width);
nnix = wrap_periodic(ix + 2, width);
iy = wrap_periodic(iy, height);
piy = wrap_periodic(iy - 1, height);
niy = wrap_periodic(iy + 1, height);
nniy = wrap_periodic(iy + 2, height);
break;
case EXTENSION_CLIP:
/* No cubic samples are inside the clip region. */
if (ix < -2 || ix > width || iy < -2 || iy > height) {
return zero();
}
pix = ix - 1;
nix = ix + 1;
nnix = ix + 2;
piy = iy - 1;
niy = iy + 1;
nniy = iy + 2;
break;
case EXTENSION_EXTEND:
pix = wrap_clamp(ix - 1, width);
nix = wrap_clamp(ix + 1, width);
nnix = wrap_clamp(ix + 2, width);
ix = wrap_clamp(ix, width);
piy = wrap_clamp(iy - 1, height);
niy = wrap_clamp(iy + 1, height);
nniy = wrap_clamp(iy + 2, height);
iy = wrap_clamp(iy, height);
break;
case EXTENSION_MIRROR:
pix = wrap_mirror(ix - 1, width);
nix = wrap_mirror(ix + 1, width);
nnix = wrap_mirror(ix + 2, width);
ix = wrap_mirror(ix, width);
piy = wrap_mirror(iy - 1, height);
niy = wrap_mirror(iy + 1, height);
nniy = wrap_mirror(iy + 2, height);
iy = wrap_mirror(iy, height);
break;
default:
kernel_assert(0);
return zero();
}
const TexT *data = (const TexT *)info.data;
const int xc[4] = {pix, ix, nix, nnix};
const int yc[4] = {piy, iy, niy, nniy};
float u[4], v[4];
/* Some helper macros to keep code size reasonable.
* Lets the compiler inline all the matrix multiplications.
*/
#define DATA(x, y) (read_clip(data, xc[x], yc[y], width, height))
#define TERM(col) \
(v[col] * \
(u[0] * DATA(0, col) + u[1] * DATA(1, col) + u[2] * DATA(2, col) + u[3] * DATA(3, col)))
SET_CUBIC_SPLINE_WEIGHTS(u, tx);
SET_CUBIC_SPLINE_WEIGHTS(v, ty);
/* Actual interpolation. */
return TERM(0) + TERM(1) + TERM(2) + TERM(3);
#undef TERM
#undef DATA
}
static ccl_always_inline OutT interp(const KernelImageInfo &info, const float x, float y)
{
switch (info.interpolation) {
case INTERPOLATION_CLOSEST:
return interp_closest(info, x, y);
case INTERPOLATION_LINEAR:
return interp_linear(info, x, y);
default:
return interp_cubic(info, x, y);
}
}
};
#undef SET_CUBIC_SPLINE_WEIGHTS
ccl_device float4 kernel_image_interp(KernelGlobals kg,
ShaderData *sd,
const int image_texture_id,
dual2 uv)
{
if (image_texture_id == KERNEL_IMAGE_NONE) {
return IMAGE_MISSING_RGBA;
}
const ccl_global KernelImageTexture &tex = kernel_data_fetch(image_textures, image_texture_id);
const ccl_global KernelImageInfo *info;
float2 xy = zero_float2();
if (tex.tile_descriptor_offset != UINT_MAX) {
/* Wrapping. */
if (!kernel_image_tile_wrap(ExtensionType(tex.extension), uv.val)) {
return zero_float4();
}
/* Tile mapping */
const KernelTileDescriptor tile_descriptor = kernel_image_tile_map(
kg, sd, tex, image_texture_id, uv, xy);
if (!kernel_tile_descriptor_loaded(tile_descriptor)) {
return (tile_descriptor == KERNEL_TILE_LOAD_FAILED) ? IMAGE_MISSING_RGBA : tex.average_color;
}
info = &kernel_data_fetch(image_info, kernel_tile_descriptor_image_info_id(tile_descriptor));
}
else {
/* Full image sampling. */
if (tex.image_info_id == KERNEL_IMAGE_NONE) {
return IMAGE_MISSING_RGBA;
}
/* Convert to pixel space. */
info = &kernel_data_fetch(image_info, tex.image_info_id);
xy = make_float2(uv.val.x * info->width, uv.val.y * info->height);
}
if (UNLIKELY(!info->data)) {
return zero_float4();
}
switch (info->data_type) {
case IMAGE_DATA_TYPE_HALF: {
const float f = ImageInterpolator<half, float>::interp(*info, xy.x, xy.y);
return make_float4(f, f, f, 1.0f);
}
case IMAGE_DATA_TYPE_BYTE: {
const float f = ImageInterpolator<uchar, float>::interp(*info, xy.x, xy.y);
return make_float4(f, f, f, 1.0f);
}
case IMAGE_DATA_TYPE_USHORT: {
const float f = ImageInterpolator<uint16_t, float>::interp(*info, xy.x, xy.y);
return make_float4(f, f, f, 1.0f);
}
case IMAGE_DATA_TYPE_FLOAT: {
const float f = ImageInterpolator<float, float>::interp(*info, xy.x, xy.y);
return make_float4(f, f, f, 1.0f);
}
case IMAGE_DATA_TYPE_HALF4:
return ImageInterpolator<half4>::interp(*info, xy.x, xy.y);
case IMAGE_DATA_TYPE_BYTE4:
return ImageInterpolator<uchar4>::interp(*info, xy.x, xy.y);
case IMAGE_DATA_TYPE_USHORT4:
return ImageInterpolator<ushort4>::interp(*info, xy.x, xy.y);
case IMAGE_DATA_TYPE_FLOAT4:
return ImageInterpolator<float4>::interp(*info, xy.x, xy.y);
default:
assert(0);
return IMAGE_MISSING_RGBA;
}
}
ccl_device_forceinline float4 kernel_image_interp_with_udim(KernelGlobals kg,
ShaderData *sd,
const int udim_id,
dual2 uv)
{
const int image_texture_id = kernel_image_udim_map(kg, udim_id, uv.val);
if (image_texture_id == KERNEL_IMAGE_NONE) {
return IMAGE_MISSING_RGBA;
}
return kernel_image_interp(kg, sd, image_texture_id, uv);
}
} /* Namespace. */
CCL_NAMESPACE_END

View File

@@ -0,0 +1,86 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* CPU kernel entry points */
/* On x86-64, our minimum is SSE4.2, so avoid the extra kernel and compile this
* one with SSE4.2 intrinsics.
*/
#if defined(__x86_64__) || defined(_M_X64)
# define __KERNEL_SSE__
# define __KERNEL_SSE2__
# define __KERNEL_SSE3__
# define __KERNEL_SSSE3__
# define __KERNEL_SSE42__
#endif
/* When building kernel for native machine detect kernel features from the flags
* set by compiler.
*/
#ifdef WITH_KERNEL_NATIVE
# ifdef __SSE4_2__
# ifndef __KERNEL_SSE42__
# define __KERNEL_SSE42__
# endif
# endif
# ifdef __AVX__
# ifndef __KERNEL_SSE__
# define __KERNEL_SSE__
# endif
# define __KERNEL_AVX__
# endif
# ifdef __AVX2__
# ifndef __KERNEL_SSE__
# define __KERNEL_SSE__
# endif
# define __KERNEL_AVX2__
# endif
#endif
/* quiet unused define warnings */
#if defined(__KERNEL_SSE2__)
/* do nothing */
#endif
#include "kernel/device/cpu/globals.h"
#include "kernel/device/cpu/kernel.h"
#define KERNEL_ARCH cpu
#include "kernel/device/cpu/kernel_arch_impl.h"
CCL_NAMESPACE_BEGIN
/* Memory Copy */
void kernel_const_copy(KernelGlobalsCPU *kg, const char *name, void *host, size_t /*unused*/)
{
if (strcmp(name, "data") == 0) {
kg->data = *(KernelData *)host;
}
else {
assert(0);
}
}
void kernel_global_memory_copy(KernelGlobalsCPU *kg,
const char *name,
void *mem,
const size_t size)
{
if (false) {
}
#define KERNEL_DATA_ARRAY(type, tname) \
else if (strcmp(name, #tname) == 0) { \
kg->tname.data = (type *)mem; \
kg->tname.width = size; \
}
#include "kernel/data_arrays.h"
else {
assert(0);
}
}
CCL_NAMESPACE_END

View File

@@ -0,0 +1,41 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
#pragma once
/* CPU Kernel Interface */
#include "kernel/types.h"
#include "util/half.h"
CCL_NAMESPACE_BEGIN
#define KERNEL_NAME_JOIN(x, y, z) x##_##y##_##z
#define KERNEL_NAME_EVAL(arch, name) KERNEL_NAME_JOIN(kernel, arch, name)
#define KERNEL_FUNCTION_FULL_NAME(name) KERNEL_NAME_EVAL(KERNEL_ARCH, name)
struct IntegratorStateCPU;
struct KernelGlobalsCPU;
struct KernelData;
KernelGlobalsCPU *kernel_globals_create();
void kernel_globals_free(KernelGlobalsCPU *kg);
void *kernel_osl_memory(const KernelGlobalsCPU *kg);
bool kernel_osl_use(const KernelGlobalsCPU *kg);
void kernel_const_copy(KernelGlobalsCPU *kg, const char *name, void *host, const size_t size);
void kernel_global_memory_copy(KernelGlobalsCPU *kg,
const char *name,
void *mem,
const size_t size);
#define KERNEL_ARCH cpu
#include "kernel/device/cpu/kernel_arch.h"
#define KERNEL_ARCH cpu_avx2
#include "kernel/device/cpu/kernel_arch.h"
CCL_NAMESPACE_END

View File

@@ -0,0 +1,148 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* Templated common declaration part of all CPU kernels. */
/* --------------------------------------------------------------------
* Integrator.
*/
#define KERNEL_INTEGRATOR_FUNCTION(name) \
void KERNEL_FUNCTION_FULL_NAME(integrator_##name)( \
const ThreadKernelGlobalsCPU *ccl_restrict kg, IntegratorStateCPU *state)
#define KERNEL_INTEGRATOR_SHADE_FUNCTION(name) \
void KERNEL_FUNCTION_FULL_NAME(integrator_##name)( \
const ThreadKernelGlobalsCPU *ccl_restrict kg, \
IntegratorStateCPU *state, \
ccl_global float *render_buffer)
#define KERNEL_INTEGRATOR_INIT_FUNCTION(name) \
bool KERNEL_FUNCTION_FULL_NAME(integrator_##name)( \
const ThreadKernelGlobalsCPU *ccl_restrict kg, \
IntegratorStateCPU *state, \
KernelWorkTile *tile, \
ccl_global float *render_buffer)
KERNEL_INTEGRATOR_INIT_FUNCTION(init_from_camera);
KERNEL_INTEGRATOR_INIT_FUNCTION(init_from_bake);
KERNEL_INTEGRATOR_SHADE_FUNCTION(megakernel);
#undef KERNEL_INTEGRATOR_FUNCTION
#undef KERNEL_INTEGRATOR_INIT_FUNCTION
#undef KERNEL_INTEGRATOR_SHADE_FUNCTION
#define KERNEL_FILM_CONVERT_FUNCTION(name) \
void KERNEL_FUNCTION_FULL_NAME(film_convert_##name)(const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
float *pixel, \
const int width, \
const int buffer_stride, \
const int pixel_stride); \
void KERNEL_FUNCTION_FULL_NAME(film_convert_half_rgba_##name)( \
const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
half4 *pixel, \
const int width, \
const int buffer_stride);
KERNEL_FILM_CONVERT_FUNCTION(depth)
KERNEL_FILM_CONVERT_FUNCTION(mist)
KERNEL_FILM_CONVERT_FUNCTION(sample_count)
KERNEL_FILM_CONVERT_FUNCTION(volume_majorant)
KERNEL_FILM_CONVERT_FUNCTION(float)
KERNEL_FILM_CONVERT_FUNCTION(light_path)
KERNEL_FILM_CONVERT_FUNCTION(rgbe)
KERNEL_FILM_CONVERT_FUNCTION(float3)
KERNEL_FILM_CONVERT_FUNCTION(motion)
KERNEL_FILM_CONVERT_FUNCTION(cryptomatte)
KERNEL_FILM_CONVERT_FUNCTION(shadow_catcher)
KERNEL_FILM_CONVERT_FUNCTION(shadow_catcher_matte_with_shadow)
KERNEL_FILM_CONVERT_FUNCTION(combined)
KERNEL_FILM_CONVERT_FUNCTION(float4)
#undef KERNEL_FILM_CONVERT_FUNCTION
/* --------------------------------------------------------------------
* Shader evaluation.
*/
void KERNEL_FUNCTION_FULL_NAME(shader_eval_background)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset);
void KERNEL_FUNCTION_FULL_NAME(shader_eval_displace)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset);
void KERNEL_FUNCTION_FULL_NAME(shader_eval_curve_shadow_transparency)(
const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset);
void KERNEL_FUNCTION_FULL_NAME(shader_eval_volume_density)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset);
/* --------------------------------------------------------------------
* Adaptive sampling.
*/
bool KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_convergence_check)(
const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int y,
const float threshold,
const int reset,
const int offset,
int stride);
void KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_filter_x)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int y,
const int start_x,
const int width,
const int offset,
int stride);
void KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_filter_y)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int start_y,
const int height,
const int offset,
int stride);
/* --------------------------------------------------------------------
* Cryptomatte.
*/
void KERNEL_FUNCTION_FULL_NAME(cryptomatte_postprocess)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
int pixel_index);
/* --------------------------------------------------------------------
* Volume Scattering Probability Guiding.
*/
void KERNEL_FUNCTION_FULL_NAME(volume_guiding_filter_x)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int y,
const int center_x,
const int min_x,
const int max_x,
const int offset,
int stride);
void KERNEL_FUNCTION_FULL_NAME(volume_guiding_filter_y)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int center_y,
const int height,
const int offset,
int stride);
#undef KERNEL_ARCH

View File

@@ -0,0 +1,401 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* Templated common implementation part of all CPU kernels.
*
* The idea is that particular `.cpp` files sets needed optimization flags and
* simply includes this file without worry of copying actual implementation over.
*/
#pragma once
// clang-format off
#include "kernel/device/cpu/compat.h"
#ifndef KERNEL_STUB
# include "kernel/globals.h"
# include "kernel/device/cpu/image.h"
# include "kernel/integrator/state.h"
# include "kernel/integrator/state_flow.h"
# include "kernel/integrator/state_util.h"
# include "kernel/integrator/init_from_camera.h"
# include "kernel/integrator/init_from_bake.h"
# include "kernel/integrator/megakernel.h"
# include "kernel/film/adaptive_sampling.h"
# include "kernel/film/cryptomatte_passes.h"
# include "kernel/film/read.h"
# include "kernel/film/volume_guiding_denoise.h"
# include "kernel/bake/bake.h"
#else
# define STUB_ASSERT(arch, name) \
assert(!(#name " kernel stub for architecture " #arch " was called!"))
#endif /* KERNEL_STUB */
// clang-format on
CCL_NAMESPACE_BEGIN
/* --------------------------------------------------------------------
* Integrator.
*/
#ifdef KERNEL_STUB
# define KERNEL_INVOKE(name, ...) (STUB_ASSERT(KERNEL_ARCH, name), 0)
#else
# define KERNEL_INVOKE(name, ...) integrator_##name(__VA_ARGS__)
#endif
/* TODO: Either use something like get_work_pixel(), or simplify tile which is passed here, so
* that it does not contain unused fields. */
#define DEFINE_INTEGRATOR_INIT_KERNEL(name) \
bool KERNEL_FUNCTION_FULL_NAME(integrator_##name)(const ThreadKernelGlobalsCPU *kg, \
IntegratorStateCPU *state, \
KernelWorkTile *tile, \
ccl_global float *render_buffer) \
{ \
(void)kg; \
(void)state; \
(void)tile; \
(void)render_buffer; \
return KERNEL_INVOKE( \
name, kg, state, tile, render_buffer, tile->x, tile->y, tile->start_sample); \
}
#define DEFINE_INTEGRATOR_SHADE_KERNEL(name) \
void KERNEL_FUNCTION_FULL_NAME(integrator_##name)(const ThreadKernelGlobalsCPU *kg, \
IntegratorStateCPU *state, \
ccl_global float *render_buffer) \
{ \
(void)kg; \
(void)state; \
(void)render_buffer; \
KERNEL_INVOKE(name, kg, state, render_buffer); \
}
DEFINE_INTEGRATOR_INIT_KERNEL(init_from_camera)
DEFINE_INTEGRATOR_INIT_KERNEL(init_from_bake)
DEFINE_INTEGRATOR_SHADE_KERNEL(megakernel)
/* --------------------------------------------------------------------
* Shader evaluation.
*/
void KERNEL_FUNCTION_FULL_NAME(shader_eval_displace)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, shader_eval_displace);
(void)kg;
(void)input;
(void)output;
(void)offset;
#else
uint cache_miss_unused = false;
kernel_displace_evaluate(kg, input, output, &cache_miss_unused, offset);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(shader_eval_background)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, shader_eval_background);
(void)kg;
(void)input;
(void)output;
(void)offset;
#else
uint cache_miss_unused = false;
kernel_background_evaluate(kg, input, output, &cache_miss_unused, offset);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(shader_eval_curve_shadow_transparency)(
const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, shader_eval_curve_shadow_transparency);
(void)kg;
(void)input;
(void)output;
(void)offset;
#else
uint cache_miss_unused = false;
kernel_curve_shadow_transparency_evaluate(kg, input, output, &cache_miss_unused, offset);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(shader_eval_volume_density)(const ThreadKernelGlobalsCPU *kg,
const KernelShaderEvalInput *input,
float *output,
const int offset)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, shader_eval_volume_density);
(void)kg;
(void)input;
(void)output;
(void)offset;
#else
uint cache_miss_unused = false;
kernel_volume_density_evaluate(kg, input, output, &cache_miss_unused, offset);
#endif
}
/* --------------------------------------------------------------------
* Adaptive sampling.
*/
bool KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_convergence_check)(
const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int y,
const float threshold,
const int reset,
const int offset,
const int stride)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, adaptive_sampling_convergence_check);
(void)kg;
(void)render_buffer;
(void)x;
(void)y;
(void)threshold;
(void)reset;
(void)offset;
(void)stride;
return false;
#else
return film_adaptive_sampling_convergence_check(
kg, render_buffer, x, y, threshold, reset, offset, stride);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_filter_x)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int y,
const int start_x,
const int width,
const int offset,
const int stride)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, adaptive_sampling_filter_x);
(void)kg;
(void)render_buffer;
(void)y;
(void)start_x;
(void)width;
(void)offset;
(void)stride;
#else
film_adaptive_sampling_filter_x(kg, render_buffer, y, start_x, width, offset, stride);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(adaptive_sampling_filter_y)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int start_y,
const int height,
const int offset,
const int stride)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, adaptive_sampling_filter_y);
(void)kg;
(void)render_buffer;
(void)x;
(void)start_y;
(void)height;
(void)offset;
(void)stride;
#else
film_adaptive_sampling_filter_y(kg, render_buffer, x, start_y, height, offset, stride);
#endif
}
/* --------------------------------------------------------------------
* Cryptomatte.
*/
void KERNEL_FUNCTION_FULL_NAME(cryptomatte_postprocess)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int pixel_index)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, cryptomatte_postprocess);
(void)kg;
(void)render_buffer;
(void)pixel_index;
#else
film_cryptomatte_post(kg, render_buffer, pixel_index);
#endif
}
/* --------------------------------------------------------------------
* Volume Scattering Probability Guiding.
*/
void KERNEL_FUNCTION_FULL_NAME(volume_guiding_filter_x)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int y,
const int center_x,
const int min_x,
const int max_x,
const int offset,
const int stride)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, volume_guiding_filter_x);
(void)kg;
(void)render_buffer;
(void)y;
(void)center_x;
(void)min_x;
(void)max_x;
(void)offset;
(void)stride;
#else
volume_guiding_filter_x(kg, render_buffer, y, center_x, min_x, max_x, offset, stride);
#endif
}
void KERNEL_FUNCTION_FULL_NAME(volume_guiding_filter_y)(const ThreadKernelGlobalsCPU *kg,
ccl_global float *render_buffer,
const int x,
const int min_y,
const int max_y,
const int offset,
const int stride)
{
#ifdef KERNEL_STUB
STUB_ASSERT(KERNEL_ARCH, volume_guiding_filter_y);
(void)kg;
(void)render_buffer;
(void)x;
(void)min_y;
(void)max_y;
(void)offset;
(void)stride;
#else
volume_guiding_filter_y(kg, render_buffer, x, min_y, max_y, offset, stride);
#endif
}
/* --------------------------------------------------------------------
* Film Convert.
*/
#ifdef KERNEL_STUB
# define KERNEL_FILM_CONVERT_FUNCTION(name, is_float) \
void KERNEL_FUNCTION_FULL_NAME(film_convert_##name)(const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
float *pixel, \
const int width, \
const int buffer_stride, \
const int pixel_stride) \
{ \
STUB_ASSERT(KERNEL_ARCH, film_convert_##name); \
(void)kfilm_convert; \
(void)buffer; \
(void)pixel; \
(void)width; \
(void)buffer_stride; \
(void)pixel_stride; \
} \
void KERNEL_FUNCTION_FULL_NAME(film_convert_half_rgba_##name)( \
const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
half4 *pixel, \
const int width, \
const int buffer_stride) \
{ \
STUB_ASSERT(KERNEL_ARCH, film_convert_##name); \
(void)kfilm_convert; \
(void)buffer; \
(void)pixel; \
(void)width; \
(void)buffer_stride; \
}
#else
# define KERNEL_FILM_CONVERT_FUNCTION(name, is_float) \
void KERNEL_FUNCTION_FULL_NAME(film_convert_##name)(const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
float *pixel, \
const int width, \
const int buffer_stride, \
const int pixel_stride) \
{ \
for (int i = 0; i < width; i++, buffer += buffer_stride, pixel += pixel_stride) { \
film_get_pass_pixel_##name(kfilm_convert, buffer, pixel); \
} \
} \
void KERNEL_FUNCTION_FULL_NAME(film_convert_half_rgba_##name)( \
const KernelFilmConvert *kfilm_convert, \
const float *buffer, \
half4 *pixel, \
const int width, \
const int buffer_stride) \
{ \
for (int i = 0; i < width; i++, buffer += buffer_stride, pixel++) { \
float pixel_rgba[4] = {0.0f, 0.0f, 0.0f, 1.0f}; \
film_get_pass_pixel_##name(kfilm_convert, buffer, pixel_rgba); \
if (is_float) { \
pixel_rgba[1] = pixel_rgba[0]; \
pixel_rgba[2] = pixel_rgba[0]; \
} \
film_apply_pass_pixel_overlays_rgba(kfilm_convert, buffer, pixel_rgba); \
*pixel = float4_to_half4_display( \
make_float4(pixel_rgba[0], pixel_rgba[1], pixel_rgba[2], pixel_rgba[3])); \
} \
}
#endif
KERNEL_FILM_CONVERT_FUNCTION(depth, true)
KERNEL_FILM_CONVERT_FUNCTION(mist, true)
KERNEL_FILM_CONVERT_FUNCTION(sample_count, true)
KERNEL_FILM_CONVERT_FUNCTION(volume_majorant, true)
KERNEL_FILM_CONVERT_FUNCTION(float, true)
KERNEL_FILM_CONVERT_FUNCTION(light_path, false)
KERNEL_FILM_CONVERT_FUNCTION(rgbe, false)
KERNEL_FILM_CONVERT_FUNCTION(float3, false)
KERNEL_FILM_CONVERT_FUNCTION(motion, false)
KERNEL_FILM_CONVERT_FUNCTION(cryptomatte, false)
KERNEL_FILM_CONVERT_FUNCTION(shadow_catcher, false)
KERNEL_FILM_CONVERT_FUNCTION(shadow_catcher_matte_with_shadow, false)
KERNEL_FILM_CONVERT_FUNCTION(combined, false)
KERNEL_FILM_CONVERT_FUNCTION(float4, false)
#undef KERNEL_FILM_CONVERT_FUNCTION
#undef KERNEL_INVOKE
#undef DEFINE_INTEGRATOR_SHADE_KERNEL
#undef DEFINE_INTEGRATOR_INIT_KERNEL
#undef KERNEL_STUB
#undef STUB_ASSERT
#undef KERNEL_ARCH
CCL_NAMESPACE_END

View File

@@ -0,0 +1,29 @@
/* SPDX-FileCopyrightText: 2011-2022 Blender Foundation
*
* SPDX-License-Identifier: Apache-2.0 */
/* Optimized CPU kernel entry points. This file is compiled with AVX2
* optimization flags and nearly all functions inlined, while kernel.cpp
* is compiled without for other CPU's. */
#include "util/optimization.h"
#ifndef WITH_CYCLES_OPTIMIZED_KERNEL_AVX2
# define KERNEL_STUB
#else
/* SSE optimization disabled for now on 32 bit, see bug #36316. */
# if !(defined(__GNUC__) && (defined(i386) || defined(_M_IX86)))
# define __KERNEL_SSE__
# define __KERNEL_SSE2__
# define __KERNEL_SSE3__
# define __KERNEL_SSSE3__
# define __KERNEL_SSE42__
# define __KERNEL_AVX__
# define __KERNEL_AVX2__
# endif
#endif /* WITH_CYCLES_OPTIMIZED_KERNEL_AVX2 */
#include "kernel/device/cpu/globals.h"
#include "kernel/device/cpu/kernel.h"
#define KERNEL_ARCH cpu_avx2
#include "kernel/device/cpu/kernel_arch_impl.h"