diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/bundle.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/bundle.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/bundle.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/bundle.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/conv-dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/conv-dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/conv-dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/conv-dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/depthart.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/depthart.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/depthart.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/depthart.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/disparity-range.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/disparity-range.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/disparity-range.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/disparity-range.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatch-helpers.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/dispatch-helpers.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatch-helpers.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/dispatch-helpers.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/execution-plan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/execution-plan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/execution-plan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/execution-plan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/gpu-resources.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/gpu-resources.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/gpu-resources.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/gpu-resources.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/half-weights.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/half-weights.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/half-weights.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/half-weights.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/channel-views.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/channel-views.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/channel-views.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/channel-views.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/cross-scan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/cross-scan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/cross-scan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/cross-scan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/depthwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/depthwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/depthwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/depthwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/elementwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/elementwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/elementwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/elementwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/helpers.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/helpers.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/helpers.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/helpers.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/layouts.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/layouts.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/layouts.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/layouts.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/native-f16-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/native-f16-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/native-f16-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/native-f16-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/normalization.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/normalization.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/normalization.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/normalization.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/packed-f16-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/packed-f16-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/packed-f16-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/packed-f16-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pointwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pointwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pointwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pointwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pooling.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pooling.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pooling.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pooling.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/resize.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/resize.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/resize.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/resize.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/scan-project.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/scan-project.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/scan-project.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/scan-project.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/selective-scan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/selective-scan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/selective-scan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/selective-scan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-specialized.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-specialized.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-specialized.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-specialized.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/types.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/types.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/types.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/types.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-f4.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-f4.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-f4.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-f4.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-gemm.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-gemm.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-gemm.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-gemm.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/preprocess.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/preprocess.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/preprocess.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/preprocess.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/tensor-arena.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/tensor-arena.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/tensor-arena.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/tensor-arena.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/types.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/types.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/types.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/types.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/winograd-dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/winograd-dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-weight.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/winograd-weight.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-weight.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/winograd-weight.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/camera-session.ts b/apps/typegpu-docs/src/examples/common/depthart/camera-session.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/camera-session.ts rename to apps/typegpu-docs/src/examples/common/depthart/camera-session.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/chooser.ts b/apps/typegpu-docs/src/examples/common/depthart/chooser.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/chooser.ts rename to apps/typegpu-docs/src/examples/common/depthart/chooser.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/model-store.ts b/apps/typegpu-docs/src/examples/common/depthart/model-store.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/model-store.ts rename to apps/typegpu-docs/src/examples/common/depthart/model-store.ts diff --git a/apps/typegpu-docs/src/examples/common/depthart/surface.ts b/apps/typegpu-docs/src/examples/common/depthart/surface.ts new file mode 100644 index 0000000000..da1f28949d --- /dev/null +++ b/apps/typegpu-docs/src/examples/common/depthart/surface.ts @@ -0,0 +1,167 @@ +import { d, std, tgpu } from 'typegpu'; + +export const DEPTH_WORKGROUP_SIZE = 64; +export const SURFACE_WORKGROUP_SIZE = 8; +const RING_OFFSETS = [-1, 0, 1] as const; + +const RANGE_BLEND = 0.12; +const TEMPORAL_ALPHA = 0.32; +const MOTION_ALPHA = 0.8; +const MOTION_LOW = 0.02; +const MOTION_HIGH = 0.09; + +const GRADIENT_RADIUS = 7; +const GRADIENT_BACK = -GRADIENT_RADIUS; +const GRADIENT_LIMIT = 0.009; +const GRADIENT_NOISE = 0.0003; +const GRADIENT_NOISE_ENERGY = GRADIENT_NOISE ** 2; +const OCCLUSION_RADII = [3, 9] as const; +const OCCLUSION_TAPS = OCCLUSION_RADII.length * (RING_OFFSETS.length ** 2 - 1); +const OCCLUSION_SCALE = 0.07; +const OCCLUSION_RANGE = 0.25; +const OCCLUSION_FLOOR = 0.012; + +export const DepthParams = d.struct({ + outputSize: d.vec2u, + reset: d.u32, +}); + +export const rangeStabilityLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + frameRange: { storage: d.vec2f, access: 'readonly' }, + stableRange: { storage: d.vec2f, access: 'mutable' }, +}); + +export const depthPrepareLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + disparity: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, + stableRange: { storage: d.vec2f, access: 'readonly' }, + history: { storage: d.arrayOf(d.f32), access: 'mutable' }, +}); + +export const surfaceLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + depth: { storage: d.arrayOf(d.f32), access: 'readonly' }, + surface: { storageTexture: d.textureStorage2d('rgba16float', 'write-only') }, +}); + +export const stabilizeRangeKernel = tgpu.computeFn({ workgroupSize: [1] })(() => { + 'use gpu'; + const low = rangeStabilityLayout.$.frameRange.x; + const high = std.max(rangeStabilityLayout.$.frameRange.y, low + 0.001); + if (rangeStabilityLayout.$.params.reset !== 0) { + rangeStabilityLayout.$.stableRange = d.vec2f(low, high); + return; + } + + const previousLow = rangeStabilityLayout.$.stableRange.x; + const previousHigh = rangeStabilityLayout.$.stableRange.y; + rangeStabilityLayout.$.stableRange = d.vec2f( + std.mix(previousLow, low, RANGE_BLEND), + std.mix(previousHigh, high, RANGE_BLEND), + ); +}); + +export const depthPrepareKernel = tgpu.computeFn({ + in: { gid: d.builtin.globalInvocationId }, + workgroupSize: [DEPTH_WORKGROUP_SIZE], +})(({ gid }) => { + 'use gpu'; + const width = depthPrepareLayout.$.params.outputSize.x; + const index = gid.x; + if (index >= width * depthPrepareLayout.$.params.outputSize.y) { + return; + } + + const low = depthPrepareLayout.$.stableRange.x; + const span = std.max(depthPrepareLayout.$.stableRange.y - low, 0.001); + const disparity = depthPrepareLayout.$.disparity[index].x; + let normalized = d.f32(0); + if (disparity === disparity) { + normalized = std.saturate((disparity - low) / span); + } + + let filtered = d.f32(normalized); + if (depthPrepareLayout.$.params.reset === 0) { + const previous = depthPrepareLayout.$.history[index]; + const motion = std.smoothstep(MOTION_LOW, MOTION_HIGH, std.abs(normalized - previous)); + filtered = std.mix(previous, normalized, std.mix(TEMPORAL_ALPHA, MOTION_ALPHA, motion)); + } + + depthPrepareLayout.$.history[index] = filtered; +}); + +function texelIndex(coord: d.v2i, size: d.v2i): number { + 'use gpu'; + const clamped = std.clamp(coord, d.vec2i(0), size - 1); + return d.u32(clamped.y) * d.u32(size.x) + d.u32(clamped.x); +} + +function depthTexelAt(coord: d.v2i, size: d.v2i): number { + 'use gpu'; + return surfaceLayout.$.depth[texelIndex(coord, size)]; +} + +function gentlerDelta(backward: number, forward: number): number { + 'use gpu'; + const back = std.abs(backward); + const front = std.abs(forward); + return (backward * front + forward * back) / std.max(back + front, 0.000000001); +} + +function surfaceSlope(gradient: d.v2f): d.v2f { + 'use gpu'; + const steepness = std.max(std.length(gradient), 0.000000001); + const shrunk = std.sqrt(std.max(steepness * steepness - GRADIENT_NOISE_ENERGY, 0)); + const ceiling = GRADIENT_LIMIT * std.tanh(shrunk / GRADIENT_LIMIT); + return gradient * (ceiling / steepness); +} + +/** Compile out occlusion taps for consumers that only need slopes and depth. */ +export const surfaceOcclusionSlot = tgpu.slot(true); + +/** Derives the surface slope and a height-field occlusion term from the depth field */ +export const surfaceKernel = tgpu.computeFn({ + in: { gid: d.builtin.globalInvocationId }, + workgroupSize: [SURFACE_WORKGROUP_SIZE, SURFACE_WORKGROUP_SIZE], +})(({ gid }) => { + 'use gpu'; + const size = d.vec2i(surfaceLayout.$.params.outputSize); + const coord = d.vec2i(gid.xy); + if (coord.x >= size.x || coord.y >= size.y) { + return; + } + + const center = depthTexelAt(coord, size); + const left = depthTexelAt(coord + d.vec2i(GRADIENT_BACK, 0), size); + const right = depthTexelAt(coord + d.vec2i(GRADIENT_RADIUS, 0), size); + const up = depthTexelAt(coord + d.vec2i(0, GRADIENT_BACK), size); + const down = depthTexelAt(coord + d.vec2i(0, GRADIENT_RADIUS), size); + const gradient = surfaceSlope( + d.vec2f(gentlerDelta(center - left, right - center), gentlerDelta(center - up, down - center)) / + d.f32(GRADIENT_RADIUS), + ); + + let occlusion = d.f32(0); + if (surfaceOcclusionSlot.$) { + for (const radius of tgpu.unroll(OCCLUSION_RADII)) { + for (const stepY of tgpu.unroll(RING_OFFSETS)) { + for (const stepX of tgpu.unroll(RING_OFFSETS)) { + if (stepX !== 0 || stepY !== 0) { + const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); + const difference = neighbor - center; + const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); + const cleared = std.max(difference - OCCLUSION_FLOOR, 0); + occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; + } + } + } + } + } + + std.textureStore( + surfaceLayout.$.surface, + d.vec2u(gid.xy), + d.vec4f(gradient, 1 - std.saturate(occlusion / d.f32(OCCLUSION_TAPS)), center), + ); +}); diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts index 8b4f56a507..546350a08a 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts @@ -1,12 +1,12 @@ import { d, tgpu } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; import { defineControls } from '../../common/defineControls.ts'; -import { DepthCameraSession } from './camera-session.ts'; -import { SourceChoice, SourceChooser } from './chooser.ts'; -import { parseDepthBundle } from './inference/bundle.ts'; -import { DepthInferencePlan } from './inference/depthart.ts'; +import { DepthCameraSession } from '../../common/depthart/camera-session.ts'; +import { SourceChoice, SourceChooser } from '../../common/depthart/chooser.ts'; +import { parseDepthBundle } from '../../common/depthart-inference/bundle.ts'; +import { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { setupLightInput } from './light-input.ts'; -import { fetchModel, modelLabel, type ModelSize } from './model-store.ts'; +import { fetchModel, modelLabel, type ModelSize } from '../../common/depthart/model-store.ts'; import { DepthRelightingRenderer, defaultRelightingSettings } from './renderer.ts'; import { RelightMode } from './shaders.ts'; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts index 2d041ac095..b78ad66cc3 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts @@ -1,3 +1,14 @@ +import { + DEPTH_WORKGROUP_SIZE, + DepthParams, + SURFACE_WORKGROUP_SIZE, + depthPrepareKernel, + depthPrepareLayout, + rangeStabilityLayout, + stabilizeRangeKernel, + surfaceKernel, + surfaceLayout, +} from '../../common/depthart/surface.ts'; import { common, d } from 'typegpu'; import type { SampledFlag, @@ -11,25 +22,16 @@ import type { TgpuTexture, UniformFlag, } from 'typegpu'; -import type { DepthCameraFrame } from './camera-session.ts'; -import { DepthDisparityRangeEstimator } from './inference/disparity-range.ts'; -import type { DepthInferencePlan } from './inference/depthart.ts'; +import type { DepthCameraFrame } from '../../common/depthart/camera-session.ts'; +import { DepthDisparityRangeEstimator } from '../../common/depthart-inference/disparity-range.ts'; +import type { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { - DEPTH_WORKGROUP_SIZE, - DepthParams, RelightMode, RelightParams, SURFACE_FAR_Z, - SURFACE_WORKGROUP_SIZE, - depthPrepareKernel, - depthPrepareLayout, - rangeStabilityLayout, relightFragment, relightFrameLayout, relightLayout, - stabilizeRangeKernel, - surfaceKernel, - surfaceLayout, } from './shaders.ts'; const MAX_CANVAS_SIDE = 1024; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts index 5fa1f1fb7a..abc0438b4f 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts @@ -1,8 +1,5 @@ import { d, std, tgpu } from 'typegpu'; -export const DEPTH_WORKGROUP_SIZE = 64; -export const SURFACE_WORKGROUP_SIZE = 8; - const RING_OFFSETS = [-1, 0, 1] as const; export const RelightMode = { @@ -12,23 +9,6 @@ export const RelightMode = { NORMALS: 3, } as const; -const RANGE_BLEND = 0.12; -const TEMPORAL_ALPHA = 0.32; -const MOTION_ALPHA = 0.8; -const MOTION_LOW = 0.02; -const MOTION_HIGH = 0.09; - -const GRADIENT_RADIUS = 7; -const GRADIENT_BACK = -GRADIENT_RADIUS; -const GRADIENT_LIMIT = 0.009; -const GRADIENT_NOISE = 0.0003; -const GRADIENT_NOISE_ENERGY = GRADIENT_NOISE ** 2; -const OCCLUSION_RADII = [3, 9] as const; -const OCCLUSION_TAPS = OCCLUSION_RADII.length * (RING_OFFSETS.length ** 2 - 1); -const OCCLUSION_SCALE = 0.07; -const OCCLUSION_RANGE = 0.25; -const OCCLUSION_FLOOR = 0.012; - const NEAR_Z = 0; /** Depth of the furthest surface the relit scene can hold */ export const SURFACE_FAR_Z = -0.7; @@ -76,11 +56,6 @@ const SHADOW_GAIN = 2.5; /** How far above the light plane an occluder may rise before it stops casting */ const SHADOW_FRONT_FADE = 0.2; -export const DepthParams = d.struct({ - outputSize: d.vec2u, - reset: d.u32, -}); - export const RelightParams = d.struct({ uvTransform: d.mat2x2f, lightColor: d.vec4f, @@ -97,25 +72,6 @@ export const RelightParams = d.struct({ mode: d.u32, }); -export const rangeStabilityLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - frameRange: { storage: d.vec2f, access: 'readonly' }, - stableRange: { storage: d.vec2f, access: 'mutable' }, -}); - -export const depthPrepareLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - disparity: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, - stableRange: { storage: d.vec2f, access: 'readonly' }, - history: { storage: d.arrayOf(d.f32), access: 'mutable' }, -}); - -export const surfaceLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - depth: { storage: d.arrayOf(d.f32), access: 'readonly' }, - surface: { storageTexture: d.textureStorage2d('rgba16float', 'write-only') }, -}); - export const relightLayout = tgpu.bindGroupLayout({ params: { uniform: RelightParams }, surface: { texture: d.texture2d() }, @@ -126,122 +82,6 @@ export const relightFrameLayout = tgpu.bindGroupLayout({ frame: { externalTexture: d.textureExternal() }, }); -export const stabilizeRangeKernel = tgpu.computeFn({ workgroupSize: [1] })(() => { - 'use gpu'; - const low = rangeStabilityLayout.$.frameRange.x; - const high = std.max(rangeStabilityLayout.$.frameRange.y, low + 0.001); - if (rangeStabilityLayout.$.params.reset !== 0) { - rangeStabilityLayout.$.stableRange = d.vec2f(low, high); - return; - } - - const previousLow = rangeStabilityLayout.$.stableRange.x; - const previousHigh = rangeStabilityLayout.$.stableRange.y; - rangeStabilityLayout.$.stableRange = d.vec2f( - std.mix(previousLow, low, RANGE_BLEND), - std.mix(previousHigh, high, RANGE_BLEND), - ); -}); - -export const depthPrepareKernel = tgpu.computeFn({ - in: { gid: d.builtin.globalInvocationId }, - workgroupSize: [DEPTH_WORKGROUP_SIZE], -})(({ gid }) => { - 'use gpu'; - const width = depthPrepareLayout.$.params.outputSize.x; - const index = gid.x; - if (index >= width * depthPrepareLayout.$.params.outputSize.y) { - return; - } - - const low = depthPrepareLayout.$.stableRange.x; - const span = std.max(depthPrepareLayout.$.stableRange.y - low, 0.001); - const disparity = depthPrepareLayout.$.disparity[index].x; - let normalized = d.f32(0); - if (disparity === disparity) { - normalized = std.saturate((disparity - low) / span); - } - - let filtered = d.f32(normalized); - if (depthPrepareLayout.$.params.reset === 0) { - const previous = depthPrepareLayout.$.history[index]; - const motion = std.smoothstep(MOTION_LOW, MOTION_HIGH, std.abs(normalized - previous)); - filtered = std.mix(previous, normalized, std.mix(TEMPORAL_ALPHA, MOTION_ALPHA, motion)); - } - - depthPrepareLayout.$.history[index] = filtered; -}); - -function texelIndex(coord: d.v2i, size: d.v2i): number { - 'use gpu'; - const clamped = std.clamp(coord, d.vec2i(0), size - 1); - return d.u32(clamped.y) * d.u32(size.x) + d.u32(clamped.x); -} - -function depthTexelAt(coord: d.v2i, size: d.v2i): number { - 'use gpu'; - return surfaceLayout.$.depth[texelIndex(coord, size)]; -} - -function gentlerDelta(backward: number, forward: number): number { - 'use gpu'; - const back = std.abs(backward); - const front = std.abs(forward); - return (backward * front + forward * back) / std.max(back + front, 0.000000001); -} - -function surfaceSlope(gradient: d.v2f): d.v2f { - 'use gpu'; - const steepness = std.max(std.length(gradient), 0.000000001); - const shrunk = std.sqrt(std.max(steepness * steepness - GRADIENT_NOISE_ENERGY, 0)); - const ceiling = GRADIENT_LIMIT * std.tanh(shrunk / GRADIENT_LIMIT); - return gradient * (ceiling / steepness); -} - -/** Derives the surface slope and a height-field occlusion term from the depth field */ -export const surfaceKernel = tgpu.computeFn({ - in: { gid: d.builtin.globalInvocationId }, - workgroupSize: [SURFACE_WORKGROUP_SIZE, SURFACE_WORKGROUP_SIZE], -})(({ gid }) => { - 'use gpu'; - const size = d.vec2i(surfaceLayout.$.params.outputSize); - const coord = d.vec2i(gid.xy); - if (coord.x >= size.x || coord.y >= size.y) { - return; - } - - const center = depthTexelAt(coord, size); - const left = depthTexelAt(coord + d.vec2i(GRADIENT_BACK, 0), size); - const right = depthTexelAt(coord + d.vec2i(GRADIENT_RADIUS, 0), size); - const up = depthTexelAt(coord + d.vec2i(0, GRADIENT_BACK), size); - const down = depthTexelAt(coord + d.vec2i(0, GRADIENT_RADIUS), size); - const gradient = surfaceSlope( - d.vec2f(gentlerDelta(center - left, right - center), gentlerDelta(center - up, down - center)) / - d.f32(GRADIENT_RADIUS), - ); - - let occlusion = d.f32(0); - for (const radius of tgpu.unroll(OCCLUSION_RADII)) { - for (const stepY of tgpu.unroll(RING_OFFSETS)) { - for (const stepX of tgpu.unroll(RING_OFFSETS)) { - if (stepX !== 0 || stepY !== 0) { - const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); - const difference = neighbor - center; - const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); - const cleared = std.max(difference - OCCLUSION_FLOOR, 0); - occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; - } - } - } - } - - std.textureStore( - surfaceLayout.$.surface, - d.vec2u(gid.xy), - d.vec4f(gradient, 1 - std.saturate(occlusion / d.f32(OCCLUSION_TAPS)), center), - ); -}); - function surfaceZ(depth: number): number { 'use gpu'; return std.mix(d.f32(SURFACE_FAR_Z), d.f32(NEAR_Z), depth); diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html new file mode 100644 index 0000000000..8c3238c8fd --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html @@ -0,0 +1,200 @@ + + + +
+ +

Preparing WebGPU…

+
+ + + + + diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts new file mode 100644 index 0000000000..4037238fa5 --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts @@ -0,0 +1,344 @@ +import { d, tgpu } from 'typegpu'; +import type { TgpuRoot } from 'typegpu'; +import { defineControls } from '../../common/defineControls.ts'; +import { DepthCameraSession } from '../../common/depthart/camera-session.ts'; +import { SourceChoice, SourceChooser } from '../../common/depthart/chooser.ts'; +import { parseDepthBundle } from '../../common/depthart-inference/bundle.ts'; +import { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; +import { fetchModel, modelLabel, type ModelSize } from '../../common/depthart/model-store.ts'; +import { DepthPaintingRenderer, defaultPaintingSettings } from './renderer.ts'; +import { PaintMode } from './shaders.ts'; + +/** Ordered to match PaintMode, so a view's index is the mode it selects */ +const VIEW_MODES = ['painting', 'camera', 'depth', 'normals', 'detail'] as const; +const FACING_MODES = ['front', 'back'] as const; +const CAMERA_FRAME_RATE = 60; +const DEMO_IMAGE_URL = '/TypeGPU/assets/depthart/demo.jpg'; + +const canvas = document.querySelector('canvas') as HTMLCanvasElement; +const video = document.querySelector('video') as HTMLVideoElement; +const status = document.querySelector('.status') as HTMLDivElement; +const statusMessage = document.querySelector('.status-message') as HTMLParagraphElement; +const listenerController = new AbortController(); + +let root: TgpuRoot | undefined; +let plan: DepthInferencePlan | undefined; +let renderer: DepthPaintingRenderer | undefined; +let chooser: SourceChooser | undefined; +let disposed = false; +let deviceLost = false; +let currentBundle: string | undefined; +let demoImage: ImageBitmap | undefined; +let staticLoopGeneration = 0; +let depthDirty = true; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function setStatus(tone: 'busy' | 'error', message: string): void { + status.dataset.tone = tone; + status.hidden = false; + statusMessage.textContent = message; +} + +function clearTransientStatus(): void { + if (status.dataset.tone === 'busy') { + status.hidden = true; + } +} + +const camera = new DepthCameraSession( + video, + { + onFrame: (frame) => { + const activeRenderer = renderer; + if (!activeRenderer || disposed || deviceLost) { + return; + } + activeRenderer.render(frame); + clearTransientStatus(); + }, + onError: (error) => { + if (!disposed && !deviceLost) { + setStatus('error', `Camera stopped: ${errorMessage(error)}`); + } + }, + onEnded: () => { + if (!disposed && !deviceLost) { + setStatus('error', 'The camera stream ended.'); + } + }, + }, + { frameRate: CAMERA_FRAME_RATE, facingMode: 'user' }, +); + +function stopStaticLoop(): void { + staticLoopGeneration += 1; +} + +function startStaticLoop(bitmap: ImageBitmap): void { + const generation = ++staticLoopGeneration; + depthDirty = true; + const step = (): void => { + if (generation !== staticLoopGeneration || disposed || deviceLost) { + return; + } + const activeRenderer = renderer; + if (activeRenderer) { + const source = new VideoFrame(bitmap, { timestamp: performance.now() * 1000 }); + try { + activeRenderer.render( + { source, uvTransform: d.mat2x2f.identity(), swapAxes: false }, + { skipDepth: !depthDirty }, + ); + depthDirty = false; + clearTransientStatus(); + } catch (error) { + if (generation === staticLoopGeneration && !disposed && !deviceLost) { + setStatus('error', `Rendering stopped: ${errorMessage(error)}`); + } + return; + } finally { + source.close(); + } + } + if (generation === staticLoopGeneration) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); +} + +async function setFacing(facing: (typeof FACING_MODES)[number]): Promise { + camera.facingMode = facing === 'front' ? 'user' : 'environment'; + if (chooser?.source !== SourceChoice.CAMERA) { + return; + } + renderer?.update({ mirror: facing === 'front' }); + if (!camera.active) { + return; + } + camera.stop(); + try { + await camera.start(); + renderer?.resetHistory(); + depthDirty = true; + } catch (error) { + setStatus('error', `Could not switch camera: ${errorMessage(error)}`); + } +} + +async function loadDemoImage(): Promise { + if (!demoImage) { + const response = await fetch(DEMO_IMAGE_URL, { signal: listenerController.signal }); + if (!response.ok) { + throw new Error(`Demo photo download failed (${response.status}).`); + } + demoImage = await createImageBitmap(await response.blob()); + } + return demoImage; +} + +async function startSource(source: SourceChoice, uploadedImage?: ImageBitmap): Promise { + if (source === SourceChoice.CAMERA) { + setStatus('busy', 'Waiting for the camera…'); + renderer?.update({ mirror: camera.facingMode === 'user' }); + await camera.start(); + renderer?.resetHistory(); + depthDirty = true; + return; + } + setStatus('busy', 'Preparing the photo…'); + const bitmap = + source === SourceChoice.UPLOAD && uploadedImage ? uploadedImage : await loadDemoImage(); + renderer?.update({ mirror: false }); + renderer?.resetHistory(); + startStaticLoop(bitmap); +} + +async function attachBundle(bytes: ArrayBuffer): Promise { + const activeRoot = root; + if (!activeRoot || disposed || deviceLost) { + return; + } + const bundle = parseDepthBundle(bytes); + setStatus('busy', `Compiling ${bundle.model} pipelines…`); + const nextPlan = new DepthInferencePlan(activeRoot, bundle); + try { + await nextPlan.initAsync(); + if (disposed || deviceLost) { + nextPlan.destroy(); + return; + } + if (!renderer) { + const nextRenderer = new DepthPaintingRenderer(activeRoot, canvas); + await nextRenderer.initAsync(); + renderer = nextRenderer; + } + } catch (error) { + nextPlan.destroy(); + throw error; + } + renderer.attach(nextPlan); + plan?.destroy(); + plan = nextPlan; + renderer.resetHistory(); + depthDirty = true; +} + +async function loadModel(size: ModelSize): Promise { + const variant = chooser?.variant(size); + if (!variant) { + setStatus('error', `The ${size} model is unavailable on this device.`); + return false; + } + const label = modelLabel(size, variant); + try { + setStatus('busy', `Downloading ${label}…`); + await attachBundle(await fetchModel(variant, listenerController.signal)); + currentBundle = variant.bundle; + return true; + } catch (error) { + if (!disposed) { + setStatus('error', `Could not load ${label}: ${errorMessage(error)}`); + } + return false; + } +} + +function showChooser(errorText?: string): void { + stopStaticLoop(); + camera.stop(); + status.hidden = true; + chooser?.show(errorText); +} + +async function start(): Promise { + if (!chooser) { + return; + } + chooser.hide(); + if (chooser.variant(chooser.model)?.bundle !== currentBundle || !plan) { + const loaded = await loadModel(chooser.model); + if (!loaded || disposed || deviceLost) { + if (!disposed && !deviceLost) { + showChooser(statusMessage.textContent ?? 'Could not load the model.'); + } + return; + } + } + try { + await startSource(chooser.source, chooser.uploadedImage); + } catch (error) { + if (!disposed && !deviceLost) { + showChooser(`Could not start: ${errorMessage(error)}`); + } + } +} + +async function initialize(): Promise { + setStatus('busy', 'Initializing WebGPU…'); + try { + const nextRoot = await tgpu.init({ + device: { optionalFeatures: ['shader-f16'] }, + }); + if (disposed) { + nextRoot.destroy(); + return; + } + root = nextRoot; + chooser = new SourceChooser( + nextRoot.device.features.has('shader-f16'), + start, + listenerController.signal, + ); + void nextRoot.device.lost.then((info) => { + if (disposed) { + return; + } + deviceLost = true; + stopStaticLoop(); + camera.stop(); + setStatus('error', `GPU device lost: ${info.message || info.reason}`); + }); + showChooser(); + } catch (error) { + if (!disposed) { + setStatus('error', `Could not start: ${errorMessage(error)}`); + } + } +} + +void initialize(); + +// #region Example controls and cleanup + +export const controls = defineControls({ + 'switch model / source': { + onButtonClick: showChooser, + }, + 'stroke size': { + initial: defaultPaintingSettings.spacing, + min: 6, + max: 32, + step: 1, + onSliderChange: (value: number) => renderer?.update({ spacing: value }), + }, + 'detail preservation': { + initial: defaultPaintingSettings.detail, + min: 0, + max: 3, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ detail: value }), + }, + 'brush texture': { + initial: defaultPaintingSettings.texture, + min: 0, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ texture: value }), + }, + 'paint opacity': { + initial: defaultPaintingSettings.opacity, + min: 0.3, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ opacity: value }), + }, + 'normal influence': { + initial: defaultPaintingSettings.normalInfluence, + min: 0, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ normalInfluence: value }), + }, + view: { + initial: VIEW_MODES[PaintMode.PAINTING], + options: VIEW_MODES, + onSelectChange: (value: (typeof VIEW_MODES)[number]) => + renderer?.update({ mode: VIEW_MODES.indexOf(value) }), + }, + camera: { + initial: defaultPaintingSettings.mirror ? FACING_MODES[0] : FACING_MODES[1], + options: FACING_MODES, + onSelectChange: (value: (typeof FACING_MODES)[number]) => void setFacing(value), + }, +}); + +export function onCleanup(): void { + if (disposed) { + return; + } + disposed = true; + stopStaticLoop(); + listenerController.abort(); + chooser?.destroy(); + demoImage?.close(); + camera.destroy(); + renderer?.destroy(); + plan?.destroy(); + root?.destroy(); +} + +// #endregion diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json new file mode 100644 index 0000000000..0c0d810a3a --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json @@ -0,0 +1,7 @@ +{ + "title": "Monocular Painting", + "description": "Paints camera feeds and photos with layered, textured brush strokes guided by inferred surface normals and image detail.", + "category": "image-processing", + "tags": ["ai", "camera", "depth", "painting"], + "coolFactor": 11 +} diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts new file mode 100644 index 0000000000..7d0ea86abf --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -0,0 +1,525 @@ +import { randf } from '@typegpu/noise'; +import { + DEPTH_WORKGROUP_SIZE, + DepthParams, + SURFACE_WORKGROUP_SIZE, + depthPrepareKernel, + depthPrepareLayout, + rangeStabilityLayout, + stabilizeRangeKernel, + surfaceKernel, + surfaceOcclusionSlot, + surfaceLayout, +} from '../../common/depthart/surface.ts'; +import { common, d, std } from 'typegpu'; +import type { + SampledFlag, + RenderFlag, + StorageFlag, + TgpuBindGroup, + TgpuBuffer, + TgpuComputePipeline, + TgpuRenderPipeline, + TgpuRoot, + TgpuSampler, + TgpuTexture, + UniformFlag, +} from 'typegpu'; +import type { DepthCameraFrame } from '../../common/depthart/camera-session.ts'; +import { DepthDisparityRangeEstimator } from '../../common/depthart-inference/disparity-range.ts'; +import type { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; +import { + PaintMode, + underpaintLayout, + copyFrameFragment, + downsampleFragment, + Stroke, + STROKES_PER_LAYER, + prepareStrokes, + prepareStrokeCells, + StrokeCell, + cellWriteLayout, + strokeWriteLayout, + strokeReadLayout, + PaintParams, + paintFragment, + paintFrameLayout, + paintLayout, +} from './shaders.ts'; + +const MAX_CANVAS_SIDE = 1024; +const MAX_PIXEL_RATIO = 2; + +type SurfaceTexture = TgpuTexture<{ + size: readonly [number, number]; + format: 'rgba16float'; +}> & + StorageFlag & + SampledFlag; + +interface PaintAttachment { + readonly depthWorkgroups: number; + readonly fieldWorkgroups: readonly [number, number]; + readonly disparity: TgpuBuffer> & StorageFlag; + readonly history: TgpuBuffer> & StorageFlag; + readonly surface: SurfaceTexture; + readonly depthBindGroup: TgpuBindGroup; + readonly surfaceBindGroup: TgpuBindGroup; + readonly paintBindGroup: TgpuBindGroup; +} + +interface PaintingState { + readonly mirror: boolean; + readonly spacing: number; + readonly detail: number; + readonly texture: number; + readonly opacity: number; + readonly normalInfluence: number; + readonly mode: number; +} + +type PaintingSettings = Partial; + +export const defaultPaintingSettings: PaintingState = { + mirror: true, + spacing: 7, + detail: 1.5, + texture: 1, + opacity: 0.45, + normalInfluence: 1, + mode: PaintMode.PAINTING, +}; + +export class DepthPaintingRenderer { + readonly #root: TgpuRoot; + readonly #canvas: HTMLCanvasElement; + readonly #context: GPUCanvasContext; + readonly #rangeEstimator: DepthDisparityRangeEstimator; + readonly #frameRange: TgpuBuffer & StorageFlag; + readonly #stableRange: TgpuBuffer & StorageFlag; + readonly #depthParams: TgpuBuffer & UniformFlag; + readonly #paintParams: TgpuBuffer & UniformFlag; + readonly #sampler: TgpuSampler; + readonly #grainSampler: TgpuSampler; + #cellCount = 1; + readonly #brushGrain: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + StorageFlag; + readonly #rangeBindGroup: TgpuBindGroup; + readonly #stabilizePipeline: TgpuComputePipeline; + readonly #depthPipeline: TgpuComputePipeline; + readonly #surfacePipeline: TgpuComputePipeline; + readonly #paintPipeline: TgpuRenderPipeline; + readonly #strokes: TgpuBuffer> & StorageFlag; + readonly #strokeWriteGroup: TgpuBindGroup; + #strokeReadGroups: TgpuBindGroup[]; + readonly #paintHistory: (TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + RenderFlag)[]; + readonly #historyViews: GPUTextureView[]; + readonly #presentGroups: TgpuBindGroup[]; + readonly #presentPipeline: TgpuRenderPipeline; + #historyIndex = 0; + #paintDirty = true; + #lastPaintTime = performance.now(); + #revealStep = 1 / 6; + #cells: TgpuBuffer> & StorageFlag; + #cellGroup: TgpuBindGroup; + readonly #cellPipeline: TgpuComputePipeline; + readonly #strokePipeline: TgpuComputePipeline; + readonly #underpainting: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + RenderFlag; + readonly #underpaintGroup: TgpuBindGroup; + readonly #mipGroups: TgpuBindGroup[]; + readonly #mipViews: ReturnType[]; + readonly #copyPipeline: TgpuRenderPipeline; + readonly #mipPipeline: TgpuRenderPipeline; + #plan: DepthInferencePlan | undefined; + #attachment: PaintAttachment | undefined; + #uvTransform = d.mat2x2f.identity(); + #swapAxes = false; + #firstFrame = true; + #settings: PaintingState = defaultPaintingSettings; + + constructor(root: TgpuRoot, canvas: HTMLCanvasElement) { + this.#root = root; + this.#canvas = canvas; + this.#context = root.configureContext({ canvas, alphaMode: 'opaque' }); + this.#rangeEstimator = new DepthDisparityRangeEstimator(root); + this.#frameRange = root.createBuffer(d.vec2f, d.vec2f(0, 1)).$usage('storage'); + this.#stableRange = root.createBuffer(d.vec2f, d.vec2f(0, 1)).$usage('storage'); + this.#depthParams = root + .createBuffer(DepthParams, { outputSize: d.vec2u(1), reset: 1 }) + .$usage('uniform'); + this.#paintParams = root.createBuffer(PaintParams).$usage('uniform'); + this.#sampler = root.createSampler({ + magFilter: 'linear', + minFilter: 'linear', + mipmapFilter: 'linear', + }); + this.#rangeBindGroup = root.createBindGroup(rangeStabilityLayout, { + params: this.#depthParams, + frameRange: this.#frameRange, + stableRange: this.#stableRange, + }); + this.#stabilizePipeline = root.createComputePipeline({ compute: stabilizeRangeKernel }); + this.#depthPipeline = root.createComputePipeline({ compute: depthPrepareKernel }); + this.#surfacePipeline = root + .with(surfaceOcclusionSlot, false) + .createComputePipeline({ compute: surfaceKernel }); + this.#paintPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER * 2)).$usage('storage'); + this.#cells = root.createBuffer(d.arrayOf(StrokeCell, this.#cellCount)).$usage('storage'); + this.#cellGroup = root.createBindGroup(cellWriteLayout, { cells: this.#cells }); + this.#cellPipeline = root.createComputePipeline({ compute: prepareStrokeCells }); + this.#strokeWriteGroup = root.createBindGroup(strokeWriteLayout, { strokes: this.#strokes }); + this.#paintHistory = Array.from({ length: 2 }, () => + root + .createTexture({ + size: [1024, 1024], + format: 'rgba8unorm', + }) + .$usage('sampled', 'render'), + ); + this.#historyViews = this.#paintHistory.map((texture) => root.unwrap(texture).createView()); + // Stable, independent noise channels sampled at different scales in brush space. + // Build once; no per-frame random changes that could make the paint sparkle. + this.#brushGrain = root + .createTexture({ size: [128, 128], format: 'rgba8unorm' }) + .$usage('sampled', 'storage'); + const grainOutput = this.#brushGrain.createView(d.textureStorage2d('rgba8unorm', 'write-only')); + root + .createGuardedComputePipeline((x: number, y: number) => { + 'use gpu'; + randf.seed2(d.vec2f(x, y).add(0.731).div(128)); + std.textureStore( + grainOutput.$, + d.vec2u(x, y), + d.vec4f(randf.sample(), randf.sample(), randf.sample(), randf.sample()), + ); + }) + .dispatchThreads(128, 128); + const grainSampler = root.createSampler({ + magFilter: 'linear', + minFilter: 'linear', + addressModeU: 'repeat', + addressModeV: 'repeat', + }); + this.#grainSampler = grainSampler; + this.#strokeReadGroups = this.#paintHistory.map((texture) => + root.createBindGroup(strokeReadLayout, { + strokes: this.#strokes, + cells: this.#cells, + history: texture.createView(), + grain: this.#brushGrain.createView(), + grainSampler, + }), + ); + this.#presentGroups = this.#paintHistory.map((texture) => + root.createBindGroup(underpaintLayout, { + image: texture.createView(), + sampler: this.#sampler, + }), + ); + this.#presentPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: downsampleFragment, + targets: { format: navigator.gpu.getPreferredCanvasFormat() }, + }); + this.#strokePipeline = root.createComputePipeline({ compute: prepareStrokes }); + this.#underpainting = root + .createTexture({ size: [1024, 1024], format: 'rgba8unorm', mipLevelCount: 11 }) + .$usage('sampled', 'render'); + this.#underpaintGroup = root.createBindGroup(underpaintLayout, { + image: this.#underpainting.createView(), + sampler: this.#sampler, + }); + this.#mipViews = Array.from({ length: 11 }, (_, level) => + root.unwrap(this.#underpainting).createView({ baseMipLevel: level, mipLevelCount: 1 }), + ); + this.#mipGroups = Array.from({ length: 10 }, (_, level) => + root.createBindGroup(underpaintLayout, { + image: this.#underpainting.createView(d.texture2d(), { + baseMipLevel: level, + mipLevelCount: 1, + }), + sampler: this.#sampler, + }), + ); + this.#copyPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: copyFrameFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#mipPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: downsampleFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#writePaintParams(); + } + + async initAsync(): Promise { + await Promise.all([ + this.#rangeEstimator.initAsync(), + this.#stabilizePipeline.initAsync(), + this.#depthPipeline.initAsync(), + this.#surfacePipeline.initAsync(), + this.#paintPipeline.initAsync(), + this.#strokePipeline.initAsync(), + this.#cellPipeline.initAsync(), + this.#copyPipeline.initAsync(), + this.#mipPipeline.initAsync(), + this.#presentPipeline.initAsync(), + ]); + } + + attach(plan: DepthInferencePlan): void { + this.detach(); + const [width, height] = plan.outputSize; + const pixelCount = width * height; + const disparity = this.#root + .createBuffer(d.arrayOf(d.vec4f, pixelCount), plan.outputBuffer) + .$usage('storage'); + const history = this.#root.createBuffer(d.arrayOf(d.f32, pixelCount)).$usage('storage'); + const surface: SurfaceTexture = this.#root + .createTexture({ size: [width, height], format: 'rgba16float' }) + .$usage('storage', 'sampled'); + + this.#plan = plan; + this.#attachment = { + depthWorkgroups: Math.ceil(pixelCount / DEPTH_WORKGROUP_SIZE), + fieldWorkgroups: [ + Math.ceil(width / SURFACE_WORKGROUP_SIZE), + Math.ceil(height / SURFACE_WORKGROUP_SIZE), + ], + disparity, + history, + surface, + depthBindGroup: this.#root.createBindGroup(depthPrepareLayout, { + params: this.#depthParams, + disparity, + stableRange: this.#stableRange, + history, + }), + surfaceBindGroup: this.#root.createBindGroup(surfaceLayout, { + params: this.#depthParams, + depth: history, + surface: surface.createView(d.textureStorage2d('rgba16float', 'write-only')), + }), + paintBindGroup: this.#root.createBindGroup(paintLayout, { + params: this.#paintParams, + surface: surface.createView(), + sampler: this.#sampler, + }), + }; + this.#rangeEstimator.attach(disparity, this.#frameRange, pixelCount); + this.#depthParams.write({ outputSize: d.vec2u(width, height), reset: 1 }); + this.#firstFrame = true; + this.#paintDirty = true; + } + + detach(): void { + this.#rangeEstimator.detach(); + const attachment = this.#attachment; + if (attachment) { + attachment.disparity.destroy(); + attachment.history.destroy(); + attachment.surface.destroy(); + } + this.#attachment = undefined; + this.#plan = undefined; + } + + update(settings: PaintingSettings): void { + this.#paintDirty = true; + this.#settings = { + ...this.#settings, + ...settings, + }; + } + + resetHistory(): void { + this.#firstFrame = true; + this.#paintDirty = true; + } + + render(frame: DepthCameraFrame, options?: { skipDepth?: boolean }): void { + const plan = this.#plan; + const attachment = this.#attachment; + if (!plan || !attachment) { + throw new Error('No depth inference plan is attached to the painting renderer.'); + } + const updateDepth = !options?.skipDepth || this.#firstFrame; + + const now = performance.now(); + // A 100ms stroke; cap long gaps so returning to the tab does not pop marks in. + this.#revealStep = Math.min(Math.max(now - this.#lastPaintTime, 1), 50) / 100; + this.#lastPaintTime = now; + this.#syncCanvasSize(); + this.#syncStrokeCells(); + this.#uvTransform = frame.uvTransform; + this.#swapAxes = frame.swapAxes; + this.#writePaintParams(); + if (updateDepth) { + this.#depthParams.patch({ reset: this.#firstFrame ? 1 : 0 }); + } + + const encoder = this.#root['~unstable'].createCommandEncoder(); + const externalFrame = this.#root.device.importExternalTexture({ source: frame.source }); + if (updateDepth) { + const pass = encoder.beginComputePass(); + plan.encodeFrame(pass, externalFrame, { + uvTransform: frame.uvTransform, + mirrorX: this.#settings.mirror, + swapAxes: frame.swapAxes, + }); + this.#rangeEstimator.encode(pass); + this.#stabilizePipeline.with(pass).with(this.#rangeBindGroup).dispatchWorkgroups(1); + this.#depthPipeline + .with(pass) + .with(attachment.depthBindGroup) + .dispatchWorkgroups(attachment.depthWorkgroups); + const [fieldX, fieldY] = attachment.fieldWorkgroups; + this.#surfacePipeline + .with(pass) + .with(attachment.surfaceBindGroup) + .dispatchWorkgroups(fieldX, fieldY); + pass.end(); + } + + const frameGroup = this.#root.createBindGroup(paintFrameLayout, { frame: externalFrame }); + const copyPass = encoder.beginRenderPass({ colorAttachments: { view: this.#mipViews[0] } }); + this.#copyPipeline.with(copyPass).with(attachment.paintBindGroup).with(frameGroup).draw(3); + copyPass.end(); + for (let level = 1; level < this.#mipViews.length; level++) { + const mipPass = encoder.beginRenderPass({ + colorAttachments: { view: this.#mipViews[level] }, + }); + this.#mipPipeline + .with(mipPass) + .with(this.#mipGroups[level - 1]) + .draw(3); + mipPass.end(); + } + const strokePass = encoder.beginComputePass(); + this.#strokePipeline + .with(strokePass) + .with(attachment.paintBindGroup) + .with(frameGroup) + .with(this.#strokeWriteGroup) + .with(this.#underpaintGroup) + .dispatchWorkgroups( + Math.ceil(this.#canvas.width / this.#settings.spacing / 8), + Math.ceil(this.#canvas.height / this.#settings.spacing / 8), + 2, + ); + this.#cellPipeline + .with(strokePass) + .with(attachment.paintBindGroup) + .with(this.#strokeWriteGroup) + .with(this.#cellGroup) + .dispatchWorkgroups( + Math.ceil(this.#canvas.width / this.#settings.spacing / 8), + Math.ceil(this.#canvas.height / this.#settings.spacing / 8), + ); + strokePass.end(); + + const nextHistory = 1 - this.#historyIndex; + // Explicit ping-pong history survives presentation; canvas swapchain contents do not. + const pass = encoder.beginRenderPass({ + colorAttachments: { view: this.#historyViews[nextHistory] }, + }); + this.#paintPipeline + .with(pass) + .with(attachment.paintBindGroup) + .with(frameGroup) + .with(this.#strokeReadGroups[this.#historyIndex]) + .with(this.#underpaintGroup) + .draw(3); + pass.end(); + const present = encoder.beginRenderPass({ colorAttachments: { view: this.#context } }); + this.#presentPipeline.with(present).with(this.#presentGroups[nextHistory]).draw(3); + present.end(); + encoder.submit(); + this.#historyIndex = nextHistory; + this.#paintDirty = false; + this.#firstFrame = false; + } + + destroy(): void { + this.detach(); + this.#rangeEstimator.destroy(); + this.#frameRange.destroy(); + this.#stableRange.destroy(); + this.#depthParams.destroy(); + this.#paintParams.destroy(); + this.#strokes.destroy(); + this.#cells.destroy(); + this.#underpainting.destroy(); + this.#brushGrain.destroy(); + for (const texture of this.#paintHistory) { + texture.destroy(); + } + this.#context.unconfigure(); + } + + #syncCanvasSize(): void { + const displayWidth = this.#canvas.clientWidth; + if (displayWidth <= 0) { + return; + } + const ratio = Math.min(globalThis.devicePixelRatio || 1, MAX_PIXEL_RATIO); + const side = Math.min(MAX_CANVAS_SIDE, Math.max(1, Math.round(displayWidth * ratio))); + if (this.#canvas.width !== side || this.#canvas.height !== side) { + this.#paintDirty = true; + this.#canvas.width = side; + this.#canvas.height = side; + } + } + + #syncStrokeCells(): void { + const count = + Math.ceil(this.#canvas.width / this.#settings.spacing) * + Math.ceil(this.#canvas.height / this.#settings.spacing); + if (count === this.#cellCount) { + return; + } + this.#cells.destroy(); + this.#cellCount = count; + this.#cells = this.#root.createBuffer(d.arrayOf(StrokeCell, count)).$usage('storage'); + this.#cellGroup = this.#root.createBindGroup(cellWriteLayout, { cells: this.#cells }); + this.#strokeReadGroups = this.#paintHistory.map((texture) => + this.#root.createBindGroup(strokeReadLayout, { + strokes: this.#strokes, + cells: this.#cells, + history: texture.createView(), + grain: this.#brushGrain.createView(), + grainSampler: this.#grainSampler, + }), + ); + this.#paintDirty = true; + } + + #writePaintParams(): void { + this.#paintParams.write({ + uvTransform: this.#uvTransform, + canvasSize: d.vec2f(this.#canvas.width, this.#canvas.height), + spacing: this.#settings.spacing, + detail: this.#settings.detail, + texture: this.#settings.texture, + opacity: this.#settings.opacity, + normalInfluence: this.#settings.normalInfluence, + swapAxes: this.#swapAxes ? 1 : 0, + mirror: this.#settings.mirror ? 1 : 0, + mode: this.#settings.mode, + resetPaint: this.#paintDirty ? 1 : 0, + revealStep: this.#revealStep, + }); + } +} diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts new file mode 100644 index 0000000000..2b8f79ef56 --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -0,0 +1,569 @@ +import { randf } from '@typegpu/noise'; +import { tgpu, d, std } from 'typegpu'; + +export const PaintMode = { PAINTING: 0, CAMERA: 1, DEPTH: 2, NORMALS: 3, DETAIL: 4 }; +export const STROKES_PER_LAYER = 32768; +export const Stroke = d.struct({ + center: d.vec2f, + axis: d.vec2f, + halfSize: d.vec2f, + grain: d.f32, + detail: d.f32, + depth: d.f32, + opacity: d.f32, + color: d.vec4f, + memory: d.vec4f, + footprintMemory: d.arrayOf(d.vec4f, 4), + dirty: d.u32, + progress: d.f32, +}); +export const PaintParams = d.struct({ + uvTransform: d.mat2x2f, + canvasSize: d.vec2f, + spacing: d.f32, + detail: d.f32, + texture: d.f32, + opacity: d.f32, + normalInfluence: d.f32, + swapAxes: d.u32, + mirror: d.u32, + mode: d.u32, + resetPaint: d.u32, + revealStep: d.f32, +}); +export const paintLayout = tgpu.bindGroupLayout({ + params: { uniform: PaintParams }, + surface: { texture: d.texture2d() }, + sampler: { sampler: 'filtering' }, +}); +export const paintFrameLayout = tgpu.bindGroupLayout({ + frame: { externalTexture: d.textureExternal() }, +}); +export const strokeWriteLayout = tgpu.bindGroupLayout({ + strokes: { storage: d.arrayOf(Stroke), access: 'mutable' }, +}); +export const StrokeCell = d.struct({ + count: d.u32, + changing: d.u32, + indices: d.arrayOf(d.u32, 450), +}); +export const cellWriteLayout = tgpu.bindGroupLayout({ + cells: { storage: d.arrayOf(StrokeCell), access: 'mutable' }, +}); +export const strokeReadLayout = tgpu.bindGroupLayout({ + cells: { storage: d.arrayOf(StrokeCell), access: 'readonly' }, + grain: { texture: d.texture2d() }, + grainSampler: { sampler: 'filtering' }, + history: { texture: d.texture2d() }, + strokes: { storage: d.arrayOf(Stroke), access: 'readonly' }, +}); + +function cameraUvAt(uv: d.v2f): d.v2f { + 'use gpu'; + let size = d.vec2f(std.textureDimensions(paintFrameLayout.$.frame)); + if (paintLayout.$.params.swapAxes !== 0) { + size = d.vec2f(size.yx); + } + let framed = d.vec2f(uv); + if (paintLayout.$.params.mirror !== 0) { + framed = d.vec2f(1 - uv.x, uv.y); + } + const side = std.min(size.x, size.y); + const pixel = size.sub(side).mul(0.5).add(framed.mul(side)).sub(0.5); + const sourceUv = std.clamp(pixel, d.vec2f(0), size.sub(1)).add(0.5).div(size); + return std.mul(paintLayout.$.params.uvTransform, sourceUv.sub(0.5)).add(0.5); +} + +function colorAt(uv: d.v2f): d.v3f { + 'use gpu'; + return std.textureSampleBaseClampToEdge( + paintFrameLayout.$.frame, + paintLayout.$.sampler, + cameraUvAt(uv), + ).rgb; +} + +// The source is copied into mip zero before bilinear downsampling each successive level. +export const underpaintLayout = tgpu.bindGroupLayout({ + image: { texture: d.texture2d() }, + sampler: { sampler: 'filtering' }, +}); +export const copyFrameFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f })(({ + uv, +}) => { + 'use gpu'; + return d.vec4f(colorAt(uv), 1); +}); +export const downsampleFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f })(({ + uv, +}) => { + 'use gpu'; + return std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, uv, 0); +}); + +function grainAt(position: d.v2f): d.v4f { + 'use gpu'; + return std.textureSampleLevel( + strokeReadLayout.$.grain, + strokeReadLayout.$.grainSampler, + position.div(128), + 0, + ); +} + +/** Edge distance, irregular paint height, pigment variation, and pigment coverage. */ +function brushField(local: d.v2f, halfSize: d.v2f, seed: number): d.v4f { + 'use gpu'; + const p = local.div(halfSize); + const origin = d.vec2f(seed * 97, seed * 61); + const pressure = grainAt(p.mul(d.vec2f(2.4, 1.6)).add(origin)); + const bend = (pressure.x - 0.5) * 0.24 * std.max(1 - p.x * p.x, d.f32(0)); + const across = p.y - bend; + // Unequal bristle bundles, with fine grain breaking the tracks into deposits. + // These random fields have no periodic wave or evenly spaced comb pattern. + const bundles = grainAt( + d.vec2f(p.x * 2.2 + pressure.y * 0.6, across * halfSize.y * 0.65).add(origin), + ); + const tooth = grainAt(local.mul(0.85).add(origin.mul(1.7))); + const endTexture = std.smoothstep(0.35, 0.95, std.abs(p.x)); + const texture = paintLayout.$.params.texture; + const width = 0.84 + texture * (pressure.z - 0.5) * 0.24; + const strandLength = 1 - texture * endTexture * (0.04 + 0.24 * bundles.x); + const q = std.abs(d.vec2f(p.x / strandLength, across / width)); + const outline = std.pow(q.x * q.x * q.x + q.y * q.y * q.y, 1 / 3); + const distance = (outline - 1) * halfSize.y + texture * (tooth.x - 0.5) * 0.65; + const edge = std.smoothstep(d.f32(0), d.f32(1.8), -distance); + const dry = std.smoothstep(0.24, 0.64, bundles.y * 0.75 + tooth.y * 0.25); + const deposit = 1 - texture * endTexture * (1 - dry) * 0.9; + // Low, uneven deposits instead of an extruded stamp with a uniform shiny rim. + const height = edge * deposit * (0.16 + texture * pressure.w * 0.22); + const pigment = + texture * + ((pressure.y - 0.5) * 0.12 + (tooth.z - 0.5) * 0.035 + endTexture * (bundles.z - 0.5) * 0.07); + return d.vec4f(distance, height, pigment, deposit); +} + +function layerSpacing(layer: number): number { + 'use gpu'; + return paintLayout.$.params.spacing * std.select(d.f32(2), d.f32(1), layer === 1); +} + +/** Normalized disparity is 0 at the far plane and 1 at the near plane. */ +export function depthBlurAmount(depth: number): number { + 'use gpu'; + // Keep the near half in focus; ramp only over the far half. + return std.saturate(1 - 2 * depth); +} + +export function depthStrokeScale(depth: number): number { + 'use gpu'; + // UI depth runs near-to-far; the model's disparity runs in the opposite direction. + const near = std.saturate(2 * depth - 1); + const far = depthBlurAmount(depth); + const remaining = 1 - far; + // Grow gently toward the camera, and quickly toward the larger background marks. + return 1 + 1.4 * near * near + 2.4 * (1 - remaining * remaining * remaining); +} + +export function strokeComesBefore( + depth: number, + index: number, + otherDepth: number, + otherIndex: number, +): boolean { + 'use gpu'; + return depth > otherDepth || (depth === otherDepth && index > otherIndex); +} + +/** Luma dominates motion; each chroma axis has one quarter of its weight. */ +export function cameraChange(current: d.v3f, previous: d.v3f): number { + 'use gpu'; + const delta = current.sub(previous); + const luma = std.dot(delta, d.vec3f(0.2126, 0.7152, 0.0722)); + const cb = (delta.z - luma) / 1.8556; + const cr = (delta.x - luma) / 1.5748; + // Keep neutral brightness changes on the existing threshold/opacity scale. + return std.length(d.vec3f(luma, cb * 0.25, cr * 0.25)) * 1.7320508; +} + +/** Preserve gentle repainting near the motion threshold; cover large changes fully. */ +export function changeOpacity(change: number, baseOpacity: number): number { + 'use gpu'; + if (baseOpacity <= 0) { + return 0; + } + return std.mix(baseOpacity, d.f32(1), std.smoothstep(0.035, 0.2, change)); +} + +/** Compute expensive image/normal queries once per stroke, rather than once per pixel. */ +export const prepareStrokes = tgpu.computeFn({ + workgroupSize: [8, 8, 1], + in: { gid: d.builtin.globalInvocationId }, +})(({ gid }) => { + 'use gpu'; + const spacing = layerSpacing(gid.z); + const grid = d.vec2u(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + if (gid.x >= grid.x || gid.y >= grid.y) { + return; + } + // Seed only by the cell and layer: no frame time and no pixel-dependent jitter. + randf.seed3(d.vec3f(d.vec2f(gid.xy).mul(0.013), d.f32(gid.z) + 0.37)); + const jitter = d.vec2f(randf.sample(), randf.sample()).sub(0.5).mul(0.8); + const center = d.vec2f(gid.xy).add(0.5).add(jitter).mul(spacing); + const uv = center.div(paintLayout.$.params.canvasSize); + const index = gid.z * STROKES_PER_LAYER + gid.y * grid.x + gid.x; + // Mip 4 retains local motion that vanished in the old 16x16 average. + const memory = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 4, + ); + // Finish a mark before sampling a new one, so continuous movement cannot + // repeatedly restart the fade or change its color/orientation halfway through. + if (paintLayout.$.params.resetPaint === 0 && strokeWriteLayout.$.strokes[index].progress < 1) { + const previous = strokeWriteLayout.$.strokes[index].progress; + strokeWriteLayout.$.strokes[index].progress = std.min( + previous + paintLayout.$.params.revealStep, + d.f32(1), + ); + strokeWriteLayout.$.strokes[index].dirty = 1; + return; + } + const footprintMemory = d.arrayOf(d.vec4f, 4)(); + let change = cameraChange(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb); + for (const probe of std.range(4)) { + let offset = d.vec2f(0.7, 0); + if (probe === 1) { + offset = d.vec2f(-0.7, 0); + } + if (probe === 2) { + offset = d.vec2f(0, 0.7); + } + if (probe === 3) { + offset = d.vec2f(0, -0.7); + } + const sample = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv.add(offset.mul(spacing).div(paintLayout.$.params.canvasSize)), + 4, + ); + footprintMemory[probe] = d.vec4f(sample); + change = std.max( + change, + cameraChange(sample.rgb, strokeWriteLayout.$.strokes[index].footprintMemory[probe].rgb), + ); + } + const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); + // A surface leaving the foreground must release its old depth occlusion too. + change = std.max(change, std.abs(surface.w - strokeWriteLayout.$.strokes[index].depth) * 0.5); + if (paintLayout.$.params.resetPaint === 0 && change < 0.035) { + strokeWriteLayout.$.strokes[index].dirty = 0; + return; + } + const color = colorAt(uv); + const offset = d.vec2f(spacing * 0.35).div(paintLayout.$.params.canvasSize); + const left = colorAt(uv.sub(d.vec2f(offset.x, 0))); + const right = colorAt(uv.add(d.vec2f(offset.x, 0))); + const up = colorAt(uv.sub(d.vec2f(0, offset.y))); + const down = colorAt(uv.add(d.vec2f(0, offset.y))); + // Local high-pass energy also sees chromatic detail and symmetric thin features. + const mean = left.add(right).add(up).add(down).mul(0.25); + const energy = + std.length(color.sub(mean)) + + 0.5 * std.max(std.length(right.sub(left)), std.length(down.sub(up))); + const detail = std.saturate(energy * paintLayout.$.params.detail * 4); + const luminance = d.vec3f(0.2126, 0.7152, 0.0722); + const gradient = d.vec2f(std.dot(right.sub(left), luminance), std.dot(down.sub(up), luminance)); + // Surface.xy stores depth slopes. The projected normal's perpendicular follows contours. + const normal = std.normalize(d.vec3f(surface.xy.mul(-80), 1)); + let angle = d.f32(-0.65); + if (std.length(gradient) > 0.005) { + angle = std.atan2(gradient.x, -gradient.y); + } + const fallback = d.vec2f(std.cos(angle), std.sin(angle)); + let tangent = d.vec2f(fallback); + if (std.length(normal.xy) > 0.01) { + tangent = std.normalize(d.vec2f(-normal.y, normal.x)); + // Axes are unoriented: align their signs before interpolating to avoid cancellation. + if (std.dot(tangent, fallback) < 0) { + tangent = tangent.mul(-1); + } + } + const axis = std.normalize(std.mix(fallback, tangent, paintLayout.$.params.normalInfluence)); + // The model returns normalized disparity: larger values are closer. + const depth = std.saturate(surface.w); + const distanceScale = depthStrokeScale(depth); + const blur = depthBlurAmount(depth); + const size = std + .mix(d.vec2f(1.55, 0.85), d.vec2f(0.9, 0.52), detail) + .mul(spacing * distanceScale); + const softened = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + blur * 3.5, + ).rgb; + let opacity = paintLayout.$.params.opacity; + if (paintLayout.$.params.resetPaint === 0) { + opacity = changeOpacity(change, opacity); + } + strokeWriteLayout.$.strokes[index] = Stroke({ + center, + axis, + memory, + footprintMemory, + dirty: 1, + // Independent 5–15 ms delay from the moment this mark is needed. + progress: std.min(paintLayout.$.params.revealStep - (0.05 + randf.sample() * 0.1), d.f32(1)), + halfSize: size, + grain: randf.sample(), + detail, + depth, + opacity, + color: d.vec4f(std.mix(std.mix(color, mean, (1 - detail) * 0.35), softened, blur * 0.7), 1), + }); +}); + +/** Soft brush front traveling from local -X to +X, exactly empty/full at 0/1. */ +export function revealAt(along: number, progress: number): number { + 'use gpu'; + const position = std.saturate(along * 0.5 + 0.5); + const remaining = 1 - std.saturate(progress); + const eased = 1 - remaining * remaining * remaining; + return std.smoothstep(position - 0.12, position + 0.12, eased * 1.24 - 0.12); +} + +/** Share one conservative sorted candidate list across all pixels in a grid cell. */ +export const prepareStrokeCells = tgpu.computeFn({ + workgroupSize: [8, 8], + in: { gid: d.builtin.globalInvocationId }, +})(({ gid }) => { + 'use gpu'; + const gridSize = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + if (gid.x >= gridSize.x || gid.y >= gridSize.y) { + return; + } + const cellIndex = gid.y * gridSize.x + gid.x; + const pixel = d.vec2f(gid.xy).add(0.5).mul(paintLayout.$.params.spacing); + // Check only dirty flags before loading brush bounds or building a candidate list. + // Keep the full old neighborhood: a changed stroke may have shrunk or moved. + let changing = false; + for (const layer of std.range(2)) { + const spacing = layerSpacing(layer); + const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + const cell = d.vec2i(std.floor(pixel.div(spacing))); + for (let y = d.i32(-7); y < 8 && !changing; y++) { + for (let x = d.i32(-7); x < 8; x++) { + const neighbor = cell.add(d.vec2i(x, y)); + if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { + continue; + } + const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); + if (strokeWriteLayout.$.strokes[index].dirty !== 0) { + changing = true; + break; + } + } + } + if (changing) { + break; + } + } + cellWriteLayout.$.cells[cellIndex].changing = std.select(d.u32(0), d.u32(1), changing); + if (!changing) { + return; + } + const padding = paintLayout.$.params.spacing * Math.SQRT1_2; + // Gather both grids into one depth order. Equal depths retain fine-over-coarse + // and row-major ordering. Near-to-far compositing also shields existing near paint + // from distant marks that start their animation on a later frame. + const candidates = d.arrayOf(d.vec2f, 450)(); + let count = d.u32(0); + for (const layer of std.range(2)) { + const spacing = layerSpacing(layer); + const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + const cell = d.vec2i(std.floor(pixel.div(spacing))); + // Cover 3.4x strokes, including jitter, bent edges and the small shadow. + for (const y of std.range(-7, 8)) { + for (const x of std.range(-7, 8)) { + const neighbor = cell.add(d.vec2i(x, y)); + if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { + continue; + } + const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); + const stroke = strokeWriteLayout.$.strokes[index]; + if ( + stroke.progress <= 0 || + std.length(pixel.sub(stroke.center)) > spacing * 6.5 + padding + ) { + continue; + } + // Reject outside the rotated brush bounds before sorting. The padding + // includes the bent edge, antialiasing, grain displacement and tiny shadow. + const delta = pixel.sub(stroke.center); + const along = std.abs(std.dot(delta, stroke.axis)); + const across = std.abs(std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x))); + if ( + along > stroke.halfSize.x + 2 + padding || + across > stroke.halfSize.y * 1.15 + 2 + padding + ) { + continue; + } + candidates[count] = d.vec2f(d.f32(index), stroke.depth); + count++; + } + } + } + for (let position = d.u32(1); position < count; position++) { + const item = d.vec2f(candidates[position]); + const index = item.x; + const depth = item.y; + let insert = position; + while (insert > 0) { + const other = d.vec2f(candidates[insert - 1]); + const otherIndex = other.x; + const otherDepth = other.y; + if (strokeComesBefore(otherDepth, otherIndex, depth, index)) { + break; + } + candidates[insert] = d.vec2f(other); + insert--; + } + candidates[insert] = d.vec2f(item); + } + cellWriteLayout.$.cells[cellIndex].count = count; + for (let i = d.u32(0); i < count; i++) { + cellWriteLayout.$.cells[cellIndex].indices[i] = d.u32(candidates[i].x); + } +}); + +export const paintFragment = tgpu.fragmentFn({ + in: { uv: d.vec2f }, + out: d.vec4f, +})(({ uv }) => { + 'use gpu'; + const camera = colorAt(uv); + const mode = paintLayout.$.params.mode; + if (mode === PaintMode.CAMERA) { + return d.vec4f(camera, 1); + } + const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); + if (mode === PaintMode.DEPTH) { + return d.vec4f(d.vec3f(surface.w), 1); + } + if (mode === PaintMode.NORMALS) { + return d.vec4f( + std + .normalize(d.vec3f(surface.xy.mul(-80), 1)) + .mul(0.5) + .add(0.5), + 1, + ); + } + const pixel = uv.mul(paintLayout.$.params.canvasSize); + if (mode === PaintMode.DETAIL) { + const grid = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + const cell = std.min(d.vec2u(pixel.div(paintLayout.$.params.spacing)), grid.sub(1)); + const detail = strokeReadLayout.$.strokes[STROKES_PER_LAYER + cell.y * grid.x + cell.x].detail; + return d.vec4f(d.vec3f(detail), 1); + } + // Only the low-frequency image reaches the gaps between strokes (about 8x8 texels). + let painted = std.textureSampleLevel( + strokeReadLayout.$.history, + paintLayout.$.sampler, + uv, + 0, + ).rgb; + if (paintLayout.$.params.resetPaint !== 0) { + painted = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 7.3, + ).rgb; + } + const cellGrid = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + const cell = std.min(d.vec2u(pixel.div(paintLayout.$.params.spacing)), cellGrid.sub(1)); + const cellIndex = cell.y * cellGrid.x + cell.x; + if (strokeReadLayout.$.cells[cellIndex].changing === 0) { + return d.vec4f(painted, 1); + } + const count = strokeReadLayout.$.cells[cellIndex].count; + // Rebuild only invalidated pixels from remembered current strokes. Blending new + // paint over flattened history retained obsolete foreground paint indefinitely. + painted = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 7.3, + ).rgb; + let visibility = d.f32(1); + let added = d.vec3f(0); + for (let candidate = d.u32(0); candidate < count; candidate++) { + const stroke = + strokeReadLayout.$.strokes[strokeReadLayout.$.cells[cellIndex].indices[candidate]]; + const delta = pixel.sub(stroke.center); + const local = d.vec2f( + std.dot(delta, stroke.axis), + std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ); + if ( + std.abs(local.x) > stroke.halfSize.x + 2 || + std.abs(local.y) > stroke.halfSize.y * 1.15 + 2 + ) { + continue; + } + const field = brushField(local, stroke.halfSize, stroke.grain); + const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); + // A light above the canvas casts a short shadow downward onto the already + // composited more distant paint. Applying it before this stroke prevents self-shadowing. + const shadowDelta = d.vec2f(0, 0.7); + const shadowLocal = local.sub( + d.vec2f( + std.dot(shadowDelta, stroke.axis), + std.dot(shadowDelta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ), + ); + const shadowField = brushField(shadowLocal, stroke.halfSize, stroke.grain); + const shadow = + (1 - std.smoothstep(-0.5, 1.2, shadowField.x)) * + (1 - coverage) * + shadowField.w * + 0.04 * + paintLayout.$.params.opacity; + const shadowAfter = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.progress); + visibility *= 1 - shadow * shadowAfter; + if (coverage <= 0) { + continue; + } + // The per-stroke height field supplies its own bevel and bristle normals. + const dx = brushField(local.add(d.vec2f(0.5, 0)), stroke.halfSize, stroke.grain).y - field.y; + const dy = brushField(local.add(d.vec2f(0, 0.5)), stroke.halfSize, stroke.grain).y - field.y; + const slope = stroke.axis.mul(dx).add(d.vec2f(-stroke.axis.y, stroke.axis.x).mul(dy)).mul(2); + const normal = std.normalize(d.vec3f(slope.mul(-1), 1)); + const light = std.normalize(d.vec3f(-0.2, -0.85, 1)); + const halfLight = std.normalize(light.add(d.vec3f(0, 0, 1))); + const diffuse = std.max(std.dot(normal, light), d.f32(0)); + const specular = std.pow(std.max(std.dot(normal, halfLight), d.f32(0)), d.f32(26)); + const pigment = stroke.color.rgb + .mul((0.95 + 0.07 * diffuse) * (1 + field.z)) + .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.008)); + const after = revealAt(local.x / stroke.halfSize.x, stroke.progress); + const opacity = coverage * field.w * stroke.opacity * after; + added += pigment * opacity * visibility; + visibility *= 1 - opacity; + if (visibility < 0.001) { + break; + } + } + painted = painted * visibility + added; + return d.vec4f(std.saturate(painted), 1); +}); diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts new file mode 100644 index 0000000000..440a8f1c23 --- /dev/null +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -0,0 +1,86 @@ +import { describe, expect } from 'vitest'; +import { it } from 'typegpu-testing-utility'; +import { common, d } from 'typegpu'; +import { + prepareStrokes, + prepareStrokeCells, + changeOpacity, + cameraChange, + depthStrokeScale, + strokeComesBefore, + revealAt, + copyFrameFragment, + downsampleFragment, + paintFragment, +} from '../../src/examples/image-processing/monocular-painting/shaders.ts'; + +describe('monocular painting shaders', () => { + it('weights luma above chroma while preserving neutral-change sensitivity', () => { + const gray = d.vec3f(0.5); + const brightness = cameraChange(d.vec3f(0.6), gray); + // A 0.1 Cb shift with unchanged Rec.709 luma. + const chroma = cameraChange(gray.add(d.vec3f(0, (-0.018556 * 0.722) / 0.7152, 0.18556)), gray); + expect(cameraChange(gray, gray)).toBe(0); + expect(brightness).toBeCloseTo(Math.sqrt(3) * 0.1); + expect(chroma).toBeCloseTo(brightness * 0.25); + expect(cameraChange(gray, d.vec3f(0.6))).toBeCloseTo(brightness); + }); + it('raises repaint opacity smoothly for stronger camera changes', () => { + expect(changeOpacity(0.035, 0.45)).toBeCloseTo(0.45); + expect(changeOpacity(0.1175, 0.45)).toBeCloseTo(0.725); + expect(changeOpacity(0.2, 0.45)).toBe(1); + expect(changeOpacity(1, 0.45)).toBe(1); + expect(changeOpacity(0.2, 0)).toBe(0); + }); + it('orders nearer strokes first across grids and enlarges distant strokes', () => { + expect(strokeComesBefore(0.9, 0, 0.2, 32768)).toBe(true); + expect(strokeComesBefore(0.2, 32768, 0.9, 0)).toBe(false); + expect(strokeComesBefore(0.5, 32768, 0.5, 0)).toBe(true); + expect(depthStrokeScale(1)).toBeCloseTo(2.4); + expect(depthStrokeScale(0.75)).toBeCloseTo(1.35); + expect(depthStrokeScale(0.5)).toBe(1); + expect(depthStrokeScale(0.25)).toBeCloseTo(3.1); + expect(depthStrokeScale(0)).toBe(3.4); + expect(depthStrokeScale(0.25)).toBeGreaterThan(depthStrokeScale(0.75)); + }); + it('reveals in stroke direction with exact empty and full endpoints', () => { + expect(revealAt(-0.5, 0.2)).toBeGreaterThan(revealAt(0.5, 0.2)); + expect(revealAt(0.5, 0.5)).toBe(1); + for (const position of [-1, -0.5, 0, 0.5, 1]) { + expect(revealAt(position, -0.1)).toBe(0); + expect(revealAt(position, 0)).toBe(0); + expect(revealAt(position, 1)).toBe(1); + } + }); + it('resolves frame-copy and bilinear mip pipelines', async ({ root, device }) => { + for (const fragment of [copyFrameFragment, downsampleFragment]) { + await root + .createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment, + targets: { format: 'rgba8unorm' }, + }) + .initAsync(); + } + expect(device.mock.createShaderModule).toHaveBeenCalledTimes(2); + }); + it('resolves stroke preparation and painting pipelines', async ({ root, device }) => { + await root + .createComputePipeline({ compute: prepareStrokes }) + .$name('prepare strokes') + .initAsync(); + await root.createComputePipeline({ compute: prepareStrokeCells }).initAsync(); + await root + .createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }) + .$name('paint strokes') + .initAsync(); + expect(device.mock.createShaderModule).toHaveBeenCalledTimes(3); + expect(device.mock.createShaderModule).toHaveBeenCalledWith( + expect.objectContaining({ code: expect.stringContaining('texture_external') }), + ); + }); +}); diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts new file mode 100644 index 0000000000..20f6906462 --- /dev/null +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -0,0 +1,183 @@ +import { tgpu, d, common } from 'typegpu'; +import { + PaintParams, + Stroke, + STROKES_PER_LAYER, + prepareStrokes, + prepareStrokeCells, + StrokeCell, + cellWriteLayout, + paintFragment, + strokeReadLayout, + paintLayout, + paintFrameLayout, + strokeWriteLayout, + underpaintLayout, +} from '../src/examples/image-processing/monocular-painting/shaders.ts'; +import { expect, test } from 'vitest'; + +test('detects local changes and removes a departed foreground mark without disturbing idle paint', async () => { + const root = await tgpu.init(); + try { + const errors: string[] = []; + root.device.addEventListener('uncapturederror', (event) => errors.push(event.error.message)); + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 1024; + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = '#666666'; + ctx.fillRect(0, 0, 1024, 1024); + const image = root + .createTexture({ size: [1024, 1024], format: 'rgba8unorm', mipLevelCount: 11 }) + .$usage('sampled', 'render'); + const surface = root.createTexture({ size: [1, 1], format: 'rgba8unorm' }).$usage('sampled'); + const sampler = root.createSampler({ + minFilter: 'linear', + magFilter: 'linear', + mipmapFilter: 'linear', + }); + const params = root + .createBuffer(PaintParams, { + uvTransform: d.mat2x2f.identity(), + canvasSize: d.vec2f(256), + spacing: 32, + detail: 1.4, + texture: 1, + opacity: 1, + normalInfluence: 0.9, + swapAxes: 0, + mirror: 0, + mode: 0, + resetPaint: 1, + revealStep: 10, + }) + .$usage('uniform'); + const strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER + 64)).$usage('storage'); + const group = root.createBindGroup(paintLayout, { + params, + surface: surface.createView(), + sampler, + }); + const writeGroup = root.createBindGroup(strokeWriteLayout, { strokes }); + const mipGroup = root.createBindGroup(underpaintLayout, { image: image.createView(), sampler }); + const pipeline = root.createComputePipeline({ compute: prepareStrokes }); + const history = root + .createTexture({ size: [256, 256], format: 'rgba8unorm' }) + .$usage('sampled'); + const target = root.createTexture({ size: [256, 256], format: 'rgba8unorm' }).$usage('render'); + const grain = root.createTexture({ size: [128, 128], format: 'rgba8unorm' }).$usage('sampled'); + grain.write(new Uint8Array(128 * 128 * 4).fill(128)); + const paint = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }); + const cells = root.createBuffer(d.arrayOf(StrokeCell, 64)).$usage('storage'); + const cellPipeline = root.createComputePipeline({ compute: prepareStrokeCells }); + const cellGroup = root.createBindGroup(cellWriteLayout, { cells }); + const readGroup = root.createBindGroup(strokeReadLayout, { + strokes, + cells, + history, + grain, + grainSampler: sampler, + }); + let lastPixels = new Uint8Array(); + async function frame() { + image.write(canvas); + image.generateMipmaps(); + const source = new VideoFrame(canvas, { timestamp: 0 }); + const external = root.device.importExternalTexture({ source }); + pipeline + .with(group) + .with(writeGroup) + .with(mipGroup) + .with(root.createBindGroup(paintFrameLayout, { frame: external })) + .dispatchWorkgroups(1, 1, 2); + cellPipeline.with(group).with(writeGroup).with(cellGroup).dispatchWorkgroups(1, 1); + paint + .with(group) + .with(mipGroup) + .with(readGroup) + .with(root.createBindGroup(paintFrameLayout, { frame: external })) + .withColorAttachment({ view: target }) + .draw(3); + const buffer = root.device.createBuffer({ + size: 256 * 256 * 4, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); + const encoder = root.device.createCommandEncoder(); + encoder.copyTextureToTexture( + { texture: root.unwrap(target) }, + { texture: root.unwrap(history) }, + [256, 256], + ); + encoder.copyTextureToBuffer( + { texture: root.unwrap(target) }, + { buffer, bytesPerRow: 1024 }, + [256, 256], + ); + root.device.queue.submit([encoder.finish()]); + await buffer.mapAsync(GPUMapMode.READ); + lastPixels = new Uint8Array(buffer.getMappedRange()).slice(); + buffer.unmap(); + buffer.destroy(); + const all = await strokes.read(); + source.close(); + return [...all.slice(0, 16), ...all.slice(STROKES_PER_LAYER, STROKES_PER_LAYER + 64)]; + } + const first = await frame(); + const originalPixels = lastPixels.slice(); + params.patch({ resetPaint: 0 }); + const quiet = await frame(); + ctx.fillStyle = '#cc4433'; + ctx.fillRect(0, 0, 512, 1024); + const moved = await frame(); + const settled = await frame(); + const results = { + first: first.filter((s) => s.dirty).length, + quiet: quiet.filter((s) => s.dirty).length, + moved: moved.filter((s) => s.dirty).length, + farRight: moved.filter((s) => s.center.x > 208 && s.dirty).length, + settled: settled.filter((s) => s.dirty).length, + errors, + }; + ctx.fillStyle = '#666666'; + ctx.fillRect(0, 0, 1024, 1024); + await frame(); + // The departed object must leave no residual color, even in dry brush gaps. + expect(lastPixels).toEqual(originalPixels); + const restored = lastPixels.slice(); + await frame(); + expect(lastPixels).toEqual(restored); + // Animation keeps a stationary stroke active until it finishes, then stops. + params.patch({ resetPaint: 1, revealStep: 0.25 }); + const start = await frame(); + params.patch({ resetPaint: 0 }); + const middle = await frame(); + await frame(); + const end = await frame(); + expect(start.every((s) => s.progress >= 0.1 && s.progress <= 0.2)).toBe(true); + expect(new Set(start.map((s) => s.progress)).size).toBeGreaterThan(1); + expect(middle.every((s, i) => Math.abs(s.progress - start[i].progress - 0.25) < 0.00001)).toBe( + true, + ); + expect(end.every((s) => s.progress >= 0.85 && s.progress <= 0.95)).toBe(true); + const complete = await frame(); + const idle = await frame(); + expect(complete.every((s) => s.progress === 1 && s.dirty === 1)).toBe(true); + expect(idle.every((s) => s.dirty === 0)).toBe(true); + // At 200 FPS the first frame is still inside every stroke's delay. + params.patch({ resetPaint: 1, revealStep: 0.05 }); + const delayed = await frame(); + expect(delayed.every((s) => s.progress <= 0)).toBe(true); + expect(results.first).toBe(80); + expect(results.quiet).toBe(0); + expect(results.moved).toBeGreaterThan(0); + expect(results.moved).toBeLessThan(80); + expect(results.farRight).toBe(0); + expect(results.settled).toBe(0); + expect(errors).toEqual([]); + } finally { + root.destroy(); + } +}); diff --git a/apps/typegpu-docs/vitest.config.mts b/apps/typegpu-docs/vitest.config.mts index 631620a9d3..a9ce78eec4 100644 --- a/apps/typegpu-docs/vitest.config.mts +++ b/apps/typegpu-docs/vitest.config.mts @@ -26,6 +26,7 @@ export default defineConfig({ test: { projects: [ { + extends: true, test: { name: 'browser', include: ['**/*.{test,spec}.browser.ts'], @@ -36,6 +37,7 @@ export default defineConfig({ }, }, { + extends: true, test: { name: 'individual-example-tests', root: './tests/individual-example-tests',