From 959ec14711648b071cb7383bb010ffbef1ea2278 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Fri, 21 Aug 2026 17:18:44 +0200 Subject: [PATCH 1/8] docs: Move DepthART inference to 'common' --- .../inference => common/depthart-inference}/bundle.ts | 0 .../depthart-inference}/conv-dispatches.ts | 0 .../inference => common/depthart-inference}/depthart.ts | 0 .../depthart-inference}/disparity-range.ts | 0 .../depthart-inference}/dispatch-helpers.ts | 0 .../inference => common/depthart-inference}/dispatches.ts | 0 .../inference => common/depthart-inference}/execution-plan.ts | 0 .../inference => common/depthart-inference}/gpu-resources.ts | 0 .../inference => common/depthart-inference}/half-weights.ts | 0 .../depthart-inference}/kernels/channel-views.ts | 0 .../depthart-inference}/kernels/cross-scan.ts | 0 .../depthart-inference}/kernels/depthwise.ts | 0 .../depthart-inference}/kernels/elementwise.ts | 0 .../depthart-inference}/kernels/helpers.ts | 0 .../depthart-inference}/kernels/layouts.ts | 0 .../depthart-inference}/kernels/native-f16-conv.ts | 0 .../depthart-inference}/kernels/normalization.ts | 0 .../depthart-inference}/kernels/packed-f16-conv.ts | 0 .../depthart-inference}/kernels/pointwise.ts | 0 .../depthart-inference}/kernels/pooling.ts | 0 .../inference => common/depthart-inference}/kernels/resize.ts | 0 .../depthart-inference}/kernels/scan-project.ts | 0 .../depthart-inference}/kernels/selective-scan.ts | 0 .../depthart-inference}/kernels/spatial-conv.ts | 0 .../depthart-inference}/kernels/spatial-specialized.ts | 0 .../inference => common/depthart-inference}/kernels/types.ts | 0 .../depthart-inference}/kernels/winograd-f4.ts | 0 .../depthart-inference}/kernels/winograd-gemm.ts | 0 .../inference => common/depthart-inference}/preprocess.ts | 0 .../inference => common/depthart-inference}/tensor-arena.ts | 0 .../inference => common/depthart-inference}/types.ts | 0 .../depthart-inference}/winograd-dispatches.ts | 0 .../depthart-inference}/winograd-weight.ts | 0 .../image-processing/monocular-light-injection/index.ts | 4 ++-- .../image-processing/monocular-light-injection/renderer.ts | 4 ++-- 35 files changed, 4 insertions(+), 4 deletions(-) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/bundle.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/conv-dispatches.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/depthart.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/disparity-range.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/dispatch-helpers.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/dispatches.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/execution-plan.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/gpu-resources.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/half-weights.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/channel-views.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/cross-scan.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/depthwise.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/elementwise.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/helpers.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/layouts.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/native-f16-conv.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/normalization.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/packed-f16-conv.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/pointwise.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/pooling.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/resize.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/scan-project.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/selective-scan.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/spatial-conv.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/spatial-specialized.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/types.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/winograd-f4.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/kernels/winograd-gemm.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/preprocess.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/tensor-arena.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/types.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/winograd-dispatches.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection/inference => common/depthart-inference}/winograd-weight.ts (100%) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/bundle.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/bundle.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/bundle.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/bundle.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/conv-dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/conv-dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/conv-dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/conv-dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/depthart.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/depthart.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/depthart.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/depthart.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/disparity-range.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/disparity-range.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/disparity-range.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/disparity-range.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatch-helpers.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/dispatch-helpers.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatch-helpers.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/dispatch-helpers.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/execution-plan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/execution-plan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/execution-plan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/execution-plan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/gpu-resources.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/gpu-resources.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/gpu-resources.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/gpu-resources.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/half-weights.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/half-weights.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/half-weights.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/half-weights.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/channel-views.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/channel-views.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/channel-views.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/channel-views.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/cross-scan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/cross-scan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/cross-scan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/cross-scan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/depthwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/depthwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/depthwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/depthwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/elementwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/elementwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/elementwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/elementwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/helpers.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/helpers.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/helpers.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/helpers.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/layouts.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/layouts.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/layouts.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/layouts.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/native-f16-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/native-f16-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/native-f16-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/native-f16-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/normalization.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/normalization.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/normalization.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/normalization.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/packed-f16-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/packed-f16-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/packed-f16-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/packed-f16-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pointwise.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pointwise.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pointwise.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pointwise.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pooling.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pooling.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/pooling.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/pooling.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/resize.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/resize.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/resize.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/resize.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/scan-project.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/scan-project.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/scan-project.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/scan-project.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/selective-scan.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/selective-scan.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/selective-scan.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/selective-scan.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-conv.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-conv.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-conv.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-conv.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-specialized.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-specialized.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/spatial-specialized.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/spatial-specialized.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/types.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/types.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/types.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/types.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-f4.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-f4.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-f4.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-f4.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-gemm.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-gemm.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/kernels/winograd-gemm.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/kernels/winograd-gemm.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/preprocess.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/preprocess.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/preprocess.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/preprocess.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/tensor-arena.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/tensor-arena.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/tensor-arena.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/tensor-arena.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/types.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/types.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/types.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/types.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-dispatches.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/winograd-dispatches.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-dispatches.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/winograd-dispatches.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-weight.ts b/apps/typegpu-docs/src/examples/common/depthart-inference/winograd-weight.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/inference/winograd-weight.ts rename to apps/typegpu-docs/src/examples/common/depthart-inference/winograd-weight.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts index 8b4f56a507..b2f93c6189 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts @@ -3,8 +3,8 @@ import type { TgpuRoot } from 'typegpu'; import { defineControls } from '../../common/defineControls.ts'; import { DepthCameraSession } from './camera-session.ts'; import { SourceChoice, SourceChooser } from './chooser.ts'; -import { parseDepthBundle } from './inference/bundle.ts'; -import { DepthInferencePlan } from './inference/depthart.ts'; +import { parseDepthBundle } from '../../common/depthart-inference/bundle.ts'; +import { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { setupLightInput } from './light-input.ts'; import { fetchModel, modelLabel, type ModelSize } from './model-store.ts'; import { DepthRelightingRenderer, defaultRelightingSettings } from './renderer.ts'; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts index 2d041ac095..30856f502b 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts @@ -12,8 +12,8 @@ import type { UniformFlag, } from 'typegpu'; import type { DepthCameraFrame } from './camera-session.ts'; -import { DepthDisparityRangeEstimator } from './inference/disparity-range.ts'; -import type { DepthInferencePlan } from './inference/depthart.ts'; +import { DepthDisparityRangeEstimator } from '../../common/depthart-inference/disparity-range.ts'; +import type { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { DEPTH_WORKGROUP_SIZE, DepthParams, From 7e0dee95578e0417b8ae83e74c161c616c102e37 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Mon, 14 Sep 2026 17:45:10 +0200 Subject: [PATCH 2/8] docs: Depth paint example --- .../src/components/CodeEditor.tsx | 6 +- .../components/stackblitz/openInStackBlitz.ts | 3 +- .../src/examples/exampleContent.ts | 35 +- .../monocular-painting/index.html | 200 ++++++++ .../monocular-painting/index.ts | 348 ++++++++++++++ .../monocular-painting/meta.json | 7 + .../monocular-painting/renderer.ts | 437 ++++++++++++++++++ .../monocular-painting/shaders.ts | 312 +++++++++++++ .../monocular-painting.test.ts | 42 ++ .../tests/monocular-painting.test.browser.ts | 97 ++++ 10 files changed, 1483 insertions(+), 4 deletions(-) create mode 100644 apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html create mode 100644 apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts create mode 100644 apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json create mode 100644 apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts create mode 100644 apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts create mode 100644 apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts create mode 100644 apps/typegpu-docs/tests/monocular-painting.test.browser.ts diff --git a/apps/typegpu-docs/src/components/CodeEditor.tsx b/apps/typegpu-docs/src/components/CodeEditor.tsx index a7416f69c2..7708846d3d 100644 --- a/apps/typegpu-docs/src/components/CodeEditor.tsx +++ b/apps/typegpu-docs/src/components/CodeEditor.tsx @@ -1,4 +1,5 @@ import { useAtomValue } from 'jotai'; +import { normalize } from 'pathe'; import { useEffect, useState } from 'react'; import Editor, { type Monaco, type OnMount } from '@monaco-editor/react'; import catppuccinMocha from '@shikijs/themes/catppuccin-mocha'; @@ -138,8 +139,9 @@ export default function CodeEditor(props: Props) { }, []); // Monaco needs relative paths to work correctly and '../../common/file.ts' will not do - const path = - 'common' in file ? `common/${file.path}` : `${file.exampleKey.replace('--', '/')}/${file.path}`; + const path = normalize( + 'common' in file ? `common/${file.path}` : `${file.exampleKey.replace('--', '/')}/${file.path}`, + ); return (
diff --git a/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts b/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts index 3d4f68ab12..0f6cef82f8 100644 --- a/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts +++ b/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts @@ -1,4 +1,5 @@ import StackBlitzSDK from '@stackblitz/sdk'; +import { normalize } from 'pathe'; import typegpuColorPackageJson from '@typegpu/color/package.json' with { type: 'json' }; import typegpuGlPackageJson from '@typegpu/gl/package.json' with { type: 'json' }; import typegpuNoisePackageJson from '@typegpu/noise/package.json' with { type: 'json' }; @@ -21,7 +22,7 @@ export async function openInStackBlitz( const tsFiles: Record = {}; for (const file of exampleSource.tsFiles) { - tsFiles[`src/${file.path}`] = file.tsnotoverContent ?? file.content; + tsFiles[normalize(`src/${file.path}`)] = file.tsnotoverContent ?? file.content; } for (const file of common) { tsFiles[`src/common/${file.path}`] = file.tsnotoverContent ?? file.content; diff --git a/apps/typegpu-docs/src/examples/exampleContent.ts b/apps/typegpu-docs/src/examples/exampleContent.ts index 79d0e4010a..4d79cd065a 100644 --- a/apps/typegpu-docs/src/examples/exampleContent.ts +++ b/apps/typegpu-docs/src/examples/exampleContent.ts @@ -128,6 +128,39 @@ async function resolveExampleSrcFile(pending: PendingExampleSrcFile): Promise { + const exampleDir = key.replace('--', '/'); + const seen = new Set(files.map((file) => pathe.join(exampleDir, file.path))); + const result = [...files]; + for (const file of result) { + const directory = pathe.dirname(pathe.join(exampleDir, file.path)); + for (const match of file.content.matchAll( + /(?:from\s*|import\s*\(?\s*)['"](\.[^'"]+\.tsx?)['"]/g, + )) { + const path = pathe.normalize(pathe.join(directory, match[1])); + const getContent = tsContentLazy[`./${path}`]; + if (seen.has(path) || !getContent || path.startsWith('common/')) { + continue; + } + seen.add(path); + result.push( + await resolveExampleSrcFile({ + exampleKey: key, + path: pathe.relative(exampleDir, path), + getContent, + getTsnotoverContent: + tsnotoverContentLazy[replaceExt(`./${path}`, `.tsnotover${pathe.extname(path)}`)], + }), + ); + } + } + return result; +} + export const examples = R.pipe( metaFiles, R.mapValues((value, key) => { @@ -145,7 +178,7 @@ export const examples = R.pipe( const tsFilePromises = exampleTsFiles[key].map(resolveExampleSrcFile); const htmlFilePromise = resolveExampleSrcFile(htmlFiles[key]?.[0]); - const tsFiles = await Promise.all(tsFilePromises); + const tsFiles = await resolveSharedSources(key, await Promise.all(tsFilePromises)); const htmlFile = await htmlFilePromise; return { diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html new file mode 100644 index 0000000000..8c3238c8fd --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.html @@ -0,0 +1,200 @@ + + + +
+
+

Preparing WebGPU…

+
+ + + + + diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts new file mode 100644 index 0000000000..04223786df --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts @@ -0,0 +1,348 @@ +import { d, tgpu } from 'typegpu'; +import type { TgpuRoot } from 'typegpu'; +import { defineControls } from '../../common/defineControls.ts'; +import { DepthCameraSession } from '../monocular-light-injection/camera-session.ts'; +import { SourceChoice, SourceChooser } from '../monocular-light-injection/chooser.ts'; +import { parseDepthBundle } from '../monocular-light-injection/inference/bundle.ts'; +import { DepthInferencePlan } from '../monocular-light-injection/inference/depthart.ts'; +import { + fetchModel, + modelLabel, + type ModelSize, +} from '../monocular-light-injection/model-store.ts'; +import { DepthPaintingRenderer, defaultPaintingSettings } from './renderer.ts'; +import { PaintMode } from './shaders.ts'; + +/** Ordered to match PaintMode, so a view's index is the mode it selects */ +const VIEW_MODES = ['painting', 'camera', 'depth', 'normals', 'detail'] as const; +const FACING_MODES = ['front', 'back'] as const; +const CAMERA_FRAME_RATE = 60; +const DEMO_IMAGE_URL = '/TypeGPU/assets/depthart/demo.jpg'; + +const canvas = document.querySelector('canvas') as HTMLCanvasElement; +const video = document.querySelector('video') as HTMLVideoElement; +const status = document.querySelector('.status') as HTMLDivElement; +const statusMessage = document.querySelector('.status-message') as HTMLParagraphElement; +const listenerController = new AbortController(); + +let root: TgpuRoot | undefined; +let plan: DepthInferencePlan | undefined; +let renderer: DepthPaintingRenderer | undefined; +let chooser: SourceChooser | undefined; +let disposed = false; +let deviceLost = false; +let currentBundle: string | undefined; +let demoImage: ImageBitmap | undefined; +let staticLoopGeneration = 0; +let depthDirty = true; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function setStatus(tone: 'busy' | 'error', message: string): void { + status.dataset.tone = tone; + status.hidden = false; + statusMessage.textContent = message; +} + +function clearTransientStatus(): void { + if (status.dataset.tone === 'busy') { + status.hidden = true; + } +} + +const camera = new DepthCameraSession( + video, + { + onFrame: (frame) => { + const activeRenderer = renderer; + if (!activeRenderer || disposed || deviceLost) { + return; + } + activeRenderer.render(frame); + clearTransientStatus(); + }, + onError: (error) => { + if (!disposed && !deviceLost) { + setStatus('error', `Camera stopped: ${errorMessage(error)}`); + } + }, + onEnded: () => { + if (!disposed && !deviceLost) { + setStatus('error', 'The camera stream ended.'); + } + }, + }, + { frameRate: CAMERA_FRAME_RATE, facingMode: 'user' }, +); + +function stopStaticLoop(): void { + staticLoopGeneration += 1; +} + +function startStaticLoop(bitmap: ImageBitmap): void { + const generation = ++staticLoopGeneration; + depthDirty = true; + const step = (): void => { + if (generation !== staticLoopGeneration || disposed || deviceLost) { + return; + } + const activeRenderer = renderer; + if (activeRenderer) { + const source = new VideoFrame(bitmap, { timestamp: performance.now() * 1000 }); + try { + activeRenderer.render( + { source, uvTransform: d.mat2x2f.identity(), swapAxes: false }, + { skipDepth: !depthDirty }, + ); + depthDirty = false; + clearTransientStatus(); + } catch (error) { + if (generation === staticLoopGeneration && !disposed && !deviceLost) { + setStatus('error', `Rendering stopped: ${errorMessage(error)}`); + } + return; + } finally { + source.close(); + } + } + if (generation === staticLoopGeneration) { + requestAnimationFrame(step); + } + }; + requestAnimationFrame(step); +} + +async function setFacing(facing: (typeof FACING_MODES)[number]): Promise { + camera.facingMode = facing === 'front' ? 'user' : 'environment'; + if (chooser?.source !== SourceChoice.CAMERA) { + return; + } + renderer?.update({ mirror: facing === 'front' }); + if (!camera.active) { + return; + } + camera.stop(); + try { + await camera.start(); + renderer?.resetHistory(); + depthDirty = true; + } catch (error) { + setStatus('error', `Could not switch camera: ${errorMessage(error)}`); + } +} + +async function loadDemoImage(): Promise { + if (!demoImage) { + const response = await fetch(DEMO_IMAGE_URL, { signal: listenerController.signal }); + if (!response.ok) { + throw new Error(`Demo photo download failed (${response.status}).`); + } + demoImage = await createImageBitmap(await response.blob()); + } + return demoImage; +} + +async function startSource(source: SourceChoice, uploadedImage?: ImageBitmap): Promise { + if (source === SourceChoice.CAMERA) { + setStatus('busy', 'Waiting for the camera…'); + renderer?.update({ mirror: camera.facingMode === 'user' }); + await camera.start(); + renderer?.resetHistory(); + depthDirty = true; + return; + } + setStatus('busy', 'Preparing the photo…'); + const bitmap = + source === SourceChoice.UPLOAD && uploadedImage ? uploadedImage : await loadDemoImage(); + renderer?.update({ mirror: false }); + renderer?.resetHistory(); + startStaticLoop(bitmap); +} + +async function attachBundle(bytes: ArrayBuffer): Promise { + const activeRoot = root; + if (!activeRoot || disposed || deviceLost) { + return; + } + const bundle = parseDepthBundle(bytes); + setStatus('busy', `Compiling ${bundle.model} pipelines…`); + const nextPlan = new DepthInferencePlan(activeRoot, bundle); + try { + await nextPlan.initAsync(); + if (disposed || deviceLost) { + nextPlan.destroy(); + return; + } + if (!renderer) { + const nextRenderer = new DepthPaintingRenderer(activeRoot, canvas); + await nextRenderer.initAsync(); + renderer = nextRenderer; + } + } catch (error) { + nextPlan.destroy(); + throw error; + } + renderer.attach(nextPlan); + plan?.destroy(); + plan = nextPlan; + renderer.resetHistory(); + depthDirty = true; +} + +async function loadModel(size: ModelSize): Promise { + const variant = chooser?.variant(size); + if (!variant) { + setStatus('error', `The ${size} model is unavailable on this device.`); + return false; + } + const label = modelLabel(size, variant); + try { + setStatus('busy', `Downloading ${label}…`); + await attachBundle(await fetchModel(variant, listenerController.signal)); + currentBundle = variant.bundle; + return true; + } catch (error) { + if (!disposed) { + setStatus('error', `Could not load ${label}: ${errorMessage(error)}`); + } + return false; + } +} + +function showChooser(errorText?: string): void { + stopStaticLoop(); + camera.stop(); + status.hidden = true; + chooser?.show(errorText); +} + +async function start(): Promise { + if (!chooser) { + return; + } + chooser.hide(); + if (chooser.variant(chooser.model)?.bundle !== currentBundle || !plan) { + const loaded = await loadModel(chooser.model); + if (!loaded || disposed || deviceLost) { + if (!disposed && !deviceLost) { + showChooser(statusMessage.textContent ?? 'Could not load the model.'); + } + return; + } + } + try { + await startSource(chooser.source, chooser.uploadedImage); + } catch (error) { + if (!disposed && !deviceLost) { + showChooser(`Could not start: ${errorMessage(error)}`); + } + } +} + +async function initialize(): Promise { + setStatus('busy', 'Initializing WebGPU…'); + try { + const nextRoot = await tgpu.init({ + device: { optionalFeatures: ['shader-f16'] }, + }); + if (disposed) { + nextRoot.destroy(); + return; + } + root = nextRoot; + chooser = new SourceChooser( + nextRoot.device.features.has('shader-f16'), + start, + listenerController.signal, + ); + void nextRoot.device.lost.then((info) => { + if (disposed) { + return; + } + deviceLost = true; + stopStaticLoop(); + camera.stop(); + setStatus('error', `GPU device lost: ${info.message || info.reason}`); + }); + showChooser(); + } catch (error) { + if (!disposed) { + setStatus('error', `Could not start: ${errorMessage(error)}`); + } + } +} + +void initialize(); + +// #region Example controls and cleanup + +export const controls = defineControls({ + 'switch model / source': { + onButtonClick: showChooser, + }, + 'stroke size': { + initial: defaultPaintingSettings.spacing, + min: 6, + max: 32, + step: 1, + onSliderChange: (value: number) => renderer?.update({ spacing: value }), + }, + 'detail preservation': { + initial: defaultPaintingSettings.detail, + min: 0, + max: 3, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ detail: value }), + }, + 'brush texture': { + initial: defaultPaintingSettings.texture, + min: 0, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ texture: value }), + }, + 'paint opacity': { + initial: defaultPaintingSettings.opacity, + min: 0.3, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ opacity: value }), + }, + 'normal influence': { + initial: defaultPaintingSettings.normalInfluence, + min: 0, + max: 1, + step: 0.05, + onSliderChange: (value: number) => renderer?.update({ normalInfluence: value }), + }, + view: { + initial: VIEW_MODES[PaintMode.PAINTING], + options: VIEW_MODES, + onSelectChange: (value: (typeof VIEW_MODES)[number]) => + renderer?.update({ mode: VIEW_MODES.indexOf(value) }), + }, + camera: { + initial: defaultPaintingSettings.mirror ? FACING_MODES[0] : FACING_MODES[1], + options: FACING_MODES, + onSelectChange: (value: (typeof FACING_MODES)[number]) => void setFacing(value), + }, +}); + +export function onCleanup(): void { + if (disposed) { + return; + } + disposed = true; + stopStaticLoop(); + listenerController.abort(); + chooser?.destroy(); + demoImage?.close(); + camera.destroy(); + renderer?.destroy(); + plan?.destroy(); + root?.destroy(); +} + +// #endregion diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json new file mode 100644 index 0000000000..0c0d810a3a --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/meta.json @@ -0,0 +1,7 @@ +{ + "title": "Monocular Painting", + "description": "Paints camera feeds and photos with layered, textured brush strokes guided by inferred surface normals and image detail.", + "category": "image-processing", + "tags": ["ai", "camera", "depth", "painting"], + "coolFactor": 11 +} diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts new file mode 100644 index 0000000000..8d40beca74 --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -0,0 +1,437 @@ +import { + DEPTH_WORKGROUP_SIZE, + DepthParams, + SURFACE_WORKGROUP_SIZE, + depthPrepareKernel, + depthPrepareLayout, + rangeStabilityLayout, + stabilizeRangeKernel, + surfaceKernel, + surfaceLayout, +} from '../monocular-light-injection/shaders.ts'; +import { common, d } from 'typegpu'; +import type { + SampledFlag, + RenderFlag, + StorageFlag, + TgpuBindGroup, + TgpuBuffer, + TgpuComputePipeline, + TgpuRenderPipeline, + TgpuRoot, + TgpuSampler, + TgpuTexture, + UniformFlag, +} from 'typegpu'; +import type { DepthCameraFrame } from '../monocular-light-injection/camera-session.ts'; +import { DepthDisparityRangeEstimator } from '../monocular-light-injection/inference/disparity-range.ts'; +import type { DepthInferencePlan } from '../monocular-light-injection/inference/depthart.ts'; +import { + PaintMode, + underpaintLayout, + copyFrameFragment, + downsampleFragment, + Stroke, + STROKES_PER_LAYER, + prepareStrokes, + strokeWriteLayout, + strokeReadLayout, + PaintParams, + paintFragment, + paintFrameLayout, + paintLayout, +} from './shaders.ts'; + +const MAX_CANVAS_SIDE = 1024; +const MAX_PIXEL_RATIO = 2; + +type SurfaceTexture = TgpuTexture<{ + size: readonly [number, number]; + format: 'rgba16float'; +}> & + StorageFlag & + SampledFlag; + +interface PaintAttachment { + readonly depthWorkgroups: number; + readonly fieldWorkgroups: readonly [number, number]; + readonly disparity: TgpuBuffer> & StorageFlag; + readonly history: TgpuBuffer> & StorageFlag; + readonly surface: SurfaceTexture; + readonly depthBindGroup: TgpuBindGroup; + readonly surfaceBindGroup: TgpuBindGroup; + readonly paintBindGroup: TgpuBindGroup; +} + +interface PaintingState { + readonly mirror: boolean; + readonly spacing: number; + readonly detail: number; + readonly texture: number; + readonly opacity: number; + readonly normalInfluence: number; + readonly mode: number; +} + +type PaintingSettings = Partial; + +export const defaultPaintingSettings: PaintingState = { + mirror: true, + spacing: 16, + detail: 1.5, + texture: 1, + opacity: 0.45, + normalInfluence: 1, + mode: PaintMode.PAINTING, +}; + +export class DepthPaintingRenderer { + readonly #root: TgpuRoot; + readonly #canvas: HTMLCanvasElement; + readonly #context: GPUCanvasContext; + readonly #rangeEstimator: DepthDisparityRangeEstimator; + readonly #frameRange: TgpuBuffer & StorageFlag; + readonly #stableRange: TgpuBuffer & StorageFlag; + readonly #depthParams: TgpuBuffer & UniformFlag; + readonly #paintParams: TgpuBuffer & UniformFlag; + readonly #sampler: TgpuSampler; + readonly #rangeBindGroup: TgpuBindGroup; + readonly #stabilizePipeline: TgpuComputePipeline; + readonly #depthPipeline: TgpuComputePipeline; + readonly #surfacePipeline: TgpuComputePipeline; + readonly #paintPipeline: TgpuRenderPipeline; + readonly #strokes: TgpuBuffer> & StorageFlag; + readonly #strokeWriteGroup: TgpuBindGroup; + readonly #strokeReadGroups: TgpuBindGroup[]; + readonly #paintHistory: (TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + RenderFlag)[]; + readonly #historyViews: GPUTextureView[]; + readonly #presentGroups: TgpuBindGroup[]; + readonly #presentPipeline: TgpuRenderPipeline; + #historyIndex = 0; + #paintDirty = true; + readonly #strokePipeline: TgpuComputePipeline; + readonly #underpainting: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + RenderFlag; + readonly #underpaintGroup: TgpuBindGroup; + readonly #mipGroups: TgpuBindGroup[]; + readonly #mipViews: ReturnType[]; + readonly #copyPipeline: TgpuRenderPipeline; + readonly #mipPipeline: TgpuRenderPipeline; + #plan: DepthInferencePlan | undefined; + #attachment: PaintAttachment | undefined; + #uvTransform = d.mat2x2f.identity(); + #swapAxes = false; + #firstFrame = true; + #settings: PaintingState = defaultPaintingSettings; + + constructor(root: TgpuRoot, canvas: HTMLCanvasElement) { + this.#root = root; + this.#canvas = canvas; + this.#context = root.configureContext({ canvas, alphaMode: 'opaque' }); + this.#rangeEstimator = new DepthDisparityRangeEstimator(root); + this.#frameRange = root.createBuffer(d.vec2f, d.vec2f(0, 1)).$usage('storage'); + this.#stableRange = root.createBuffer(d.vec2f, d.vec2f(0, 1)).$usage('storage'); + this.#depthParams = root + .createBuffer(DepthParams, { outputSize: d.vec2u(1), reset: 1 }) + .$usage('uniform'); + this.#paintParams = root.createBuffer(PaintParams).$usage('uniform'); + this.#sampler = root.createSampler({ + magFilter: 'linear', + minFilter: 'linear', + mipmapFilter: 'linear', + }); + this.#rangeBindGroup = root.createBindGroup(rangeStabilityLayout, { + params: this.#depthParams, + frameRange: this.#frameRange, + stableRange: this.#stableRange, + }); + this.#stabilizePipeline = root.createComputePipeline({ compute: stabilizeRangeKernel }); + this.#depthPipeline = root.createComputePipeline({ compute: depthPrepareKernel }); + this.#surfacePipeline = root.createComputePipeline({ compute: surfaceKernel }); + this.#paintPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER * 2)).$usage('storage'); + this.#strokeWriteGroup = root.createBindGroup(strokeWriteLayout, { strokes: this.#strokes }); + this.#paintHistory = Array.from({ length: 2 }, () => + root + .createTexture({ + size: [1024, 1024], + format: 'rgba8unorm', + }) + .$usage('sampled', 'render'), + ); + this.#historyViews = this.#paintHistory.map((texture) => root.unwrap(texture).createView()); + this.#strokeReadGroups = this.#paintHistory.map((texture) => + root.createBindGroup(strokeReadLayout, { + strokes: this.#strokes, + history: texture.createView(), + }), + ); + this.#presentGroups = this.#paintHistory.map((texture) => + root.createBindGroup(underpaintLayout, { + image: texture.createView(), + sampler: this.#sampler, + }), + ); + this.#presentPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: downsampleFragment, + targets: { format: navigator.gpu.getPreferredCanvasFormat() }, + }); + this.#strokePipeline = root.createComputePipeline({ compute: prepareStrokes }); + this.#underpainting = root + .createTexture({ size: [1024, 1024], format: 'rgba8unorm', mipLevelCount: 11 }) + .$usage('sampled', 'render'); + this.#underpaintGroup = root.createBindGroup(underpaintLayout, { + image: this.#underpainting.createView(), + sampler: this.#sampler, + }); + this.#mipViews = Array.from({ length: 11 }, (_, level) => + root.unwrap(this.#underpainting).createView({ baseMipLevel: level, mipLevelCount: 1 }), + ); + this.#mipGroups = Array.from({ length: 10 }, (_, level) => + root.createBindGroup(underpaintLayout, { + image: this.#underpainting.createView(d.texture2d(), { + baseMipLevel: level, + mipLevelCount: 1, + }), + sampler: this.#sampler, + }), + ); + this.#copyPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: copyFrameFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#mipPipeline = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: downsampleFragment, + targets: { format: 'rgba8unorm' }, + }); + this.#writePaintParams(); + } + + async initAsync(): Promise { + await Promise.all([ + this.#rangeEstimator.initAsync(), + this.#stabilizePipeline.initAsync(), + this.#depthPipeline.initAsync(), + this.#surfacePipeline.initAsync(), + this.#paintPipeline.initAsync(), + this.#strokePipeline.initAsync(), + this.#copyPipeline.initAsync(), + this.#mipPipeline.initAsync(), + this.#presentPipeline.initAsync(), + ]); + } + + attach(plan: DepthInferencePlan): void { + this.detach(); + const [width, height] = plan.outputSize; + const pixelCount = width * height; + const disparity = this.#root + .createBuffer(d.arrayOf(d.vec4f, pixelCount), plan.outputBuffer) + .$usage('storage'); + const history = this.#root.createBuffer(d.arrayOf(d.f32, pixelCount)).$usage('storage'); + const surface: SurfaceTexture = this.#root + .createTexture({ size: [width, height], format: 'rgba16float' }) + .$usage('storage', 'sampled'); + + this.#plan = plan; + this.#attachment = { + depthWorkgroups: Math.ceil(pixelCount / DEPTH_WORKGROUP_SIZE), + fieldWorkgroups: [ + Math.ceil(width / SURFACE_WORKGROUP_SIZE), + Math.ceil(height / SURFACE_WORKGROUP_SIZE), + ], + disparity, + history, + surface, + depthBindGroup: this.#root.createBindGroup(depthPrepareLayout, { + params: this.#depthParams, + disparity, + stableRange: this.#stableRange, + history, + }), + surfaceBindGroup: this.#root.createBindGroup(surfaceLayout, { + params: this.#depthParams, + depth: history, + surface: surface.createView(d.textureStorage2d('rgba16float', 'write-only')), + }), + paintBindGroup: this.#root.createBindGroup(paintLayout, { + params: this.#paintParams, + surface: surface.createView(), + sampler: this.#sampler, + }), + }; + this.#rangeEstimator.attach(disparity, this.#frameRange, pixelCount); + this.#depthParams.write({ outputSize: d.vec2u(width, height), reset: 1 }); + this.#firstFrame = true; + this.#paintDirty = true; + } + + detach(): void { + this.#rangeEstimator.detach(); + const attachment = this.#attachment; + if (attachment) { + attachment.disparity.destroy(); + attachment.history.destroy(); + attachment.surface.destroy(); + } + this.#attachment = undefined; + this.#plan = undefined; + } + + update(settings: PaintingSettings): void { + this.#paintDirty = true; + this.#settings = { + ...this.#settings, + ...settings, + }; + } + + resetHistory(): void { + this.#firstFrame = true; + this.#paintDirty = true; + } + + render(frame: DepthCameraFrame, options?: { skipDepth?: boolean }): void { + const plan = this.#plan; + const attachment = this.#attachment; + if (!plan || !attachment) { + throw new Error('No depth inference plan is attached to the painting renderer.'); + } + const updateDepth = !options?.skipDepth || this.#firstFrame; + + this.#syncCanvasSize(); + this.#uvTransform = frame.uvTransform; + this.#swapAxes = frame.swapAxes; + this.#writePaintParams(); + if (updateDepth) { + this.#depthParams.patch({ reset: this.#firstFrame ? 1 : 0 }); + } + + const encoder = this.#root['~unstable'].createCommandEncoder(); + const externalFrame = this.#root.device.importExternalTexture({ source: frame.source }); + if (updateDepth) { + const pass = encoder.beginComputePass(); + plan.encodeFrame(pass, externalFrame, { + uvTransform: frame.uvTransform, + mirrorX: this.#settings.mirror, + swapAxes: frame.swapAxes, + }); + this.#rangeEstimator.encode(pass); + this.#stabilizePipeline.with(pass).with(this.#rangeBindGroup).dispatchWorkgroups(1); + this.#depthPipeline + .with(pass) + .with(attachment.depthBindGroup) + .dispatchWorkgroups(attachment.depthWorkgroups); + const [fieldX, fieldY] = attachment.fieldWorkgroups; + this.#surfacePipeline + .with(pass) + .with(attachment.surfaceBindGroup) + .dispatchWorkgroups(fieldX, fieldY); + pass.end(); + } + + const frameGroup = this.#root.createBindGroup(paintFrameLayout, { frame: externalFrame }); + const copyPass = encoder.beginRenderPass({ colorAttachments: { view: this.#mipViews[0] } }); + this.#copyPipeline.with(copyPass).with(attachment.paintBindGroup).with(frameGroup).draw(3); + copyPass.end(); + for (let level = 1; level < this.#mipViews.length; level++) { + const mipPass = encoder.beginRenderPass({ + colorAttachments: { view: this.#mipViews[level] }, + }); + this.#mipPipeline + .with(mipPass) + .with(this.#mipGroups[level - 1]) + .draw(3); + mipPass.end(); + } + const strokePass = encoder.beginComputePass(); + this.#strokePipeline + .with(strokePass) + .with(attachment.paintBindGroup) + .with(frameGroup) + .with(this.#strokeWriteGroup) + .with(this.#underpaintGroup) + .dispatchWorkgroups( + Math.ceil(this.#canvas.width / this.#settings.spacing / 8), + Math.ceil(this.#canvas.height / this.#settings.spacing / 8), + 2, + ); + strokePass.end(); + + const nextHistory = 1 - this.#historyIndex; + // Explicit ping-pong history survives presentation; canvas swapchain contents do not. + const pass = encoder.beginRenderPass({ + colorAttachments: { view: this.#historyViews[nextHistory] }, + }); + this.#paintPipeline + .with(pass) + .with(attachment.paintBindGroup) + .with(frameGroup) + .with(this.#strokeReadGroups[this.#historyIndex]) + .with(this.#underpaintGroup) + .draw(3); + pass.end(); + const present = encoder.beginRenderPass({ colorAttachments: { view: this.#context } }); + this.#presentPipeline.with(present).with(this.#presentGroups[nextHistory]).draw(3); + present.end(); + encoder.submit(); + this.#historyIndex = nextHistory; + this.#paintDirty = false; + this.#firstFrame = false; + } + + destroy(): void { + this.detach(); + this.#rangeEstimator.destroy(); + this.#frameRange.destroy(); + this.#stableRange.destroy(); + this.#depthParams.destroy(); + this.#paintParams.destroy(); + this.#strokes.destroy(); + this.#underpainting.destroy(); + for (const texture of this.#paintHistory) { + texture.destroy(); + } + this.#context.unconfigure(); + } + + #syncCanvasSize(): void { + const displayWidth = this.#canvas.clientWidth; + if (displayWidth <= 0) { + return; + } + const ratio = Math.min(globalThis.devicePixelRatio || 1, MAX_PIXEL_RATIO); + const side = Math.min(MAX_CANVAS_SIDE, Math.max(1, Math.round(displayWidth * ratio))); + if (this.#canvas.width !== side || this.#canvas.height !== side) { + this.#paintDirty = true; + this.#canvas.width = side; + this.#canvas.height = side; + } + } + + #writePaintParams(): void { + this.#paintParams.write({ + uvTransform: this.#uvTransform, + canvasSize: d.vec2f(this.#canvas.width, this.#canvas.height), + spacing: this.#settings.spacing, + detail: this.#settings.detail, + texture: this.#settings.texture, + opacity: this.#settings.opacity, + normalInfluence: this.#settings.normalInfluence, + swapAxes: this.#swapAxes ? 1 : 0, + mirror: this.#settings.mirror ? 1 : 0, + mode: this.#settings.mode, + resetPaint: this.#paintDirty ? 1 : 0, + }); + } +} diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts new file mode 100644 index 0000000000..d71f99fea2 --- /dev/null +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -0,0 +1,312 @@ +import { randf } from '@typegpu/noise'; +import { tgpu, d, std } from 'typegpu'; + +export const PaintMode = { PAINTING: 0, CAMERA: 1, DEPTH: 2, NORMALS: 3, DETAIL: 4 }; +export const STROKES_PER_LAYER = 32768; +export const Stroke = d.struct({ + center: d.vec2f, + axis: d.vec2f, + halfSize: d.vec2f, + grain: d.f32, + detail: d.f32, + color: d.vec4f, + memory: d.vec4f, + dirty: d.u32, +}); +export const PaintParams = d.struct({ + uvTransform: d.mat2x2f, + canvasSize: d.vec2f, + spacing: d.f32, + detail: d.f32, + texture: d.f32, + opacity: d.f32, + normalInfluence: d.f32, + swapAxes: d.u32, + mirror: d.u32, + mode: d.u32, + resetPaint: d.u32, +}); +export const paintLayout = tgpu.bindGroupLayout({ + params: { uniform: PaintParams }, + surface: { texture: d.texture2d() }, + sampler: { sampler: 'filtering' }, +}); +export const paintFrameLayout = tgpu.bindGroupLayout({ + frame: { externalTexture: d.textureExternal() }, +}); +export const strokeWriteLayout = tgpu.bindGroupLayout({ + strokes: { storage: d.arrayOf(Stroke), access: 'mutable' }, +}); +export const strokeReadLayout = tgpu.bindGroupLayout({ + history: { texture: d.texture2d() }, + strokes: { storage: d.arrayOf(Stroke), access: 'readonly' }, +}); + +function cameraUvAt(uv: d.v2f): d.v2f { + 'use gpu'; + let size = d.vec2f(std.textureDimensions(paintFrameLayout.$.frame)); + if (paintLayout.$.params.swapAxes !== 0) { + size = d.vec2f(size.yx); + } + let framed = d.vec2f(uv); + if (paintLayout.$.params.mirror !== 0) { + framed = d.vec2f(1 - uv.x, uv.y); + } + const side = std.min(size.x, size.y); + const pixel = size.sub(side).mul(0.5).add(framed.mul(side)).sub(0.5); + const sourceUv = std.clamp(pixel, d.vec2f(0), size.sub(1)).add(0.5).div(size); + return std.mul(paintLayout.$.params.uvTransform, sourceUv.sub(0.5)).add(0.5); +} + +function colorAt(uv: d.v2f): d.v3f { + 'use gpu'; + return std.textureSampleBaseClampToEdge( + paintFrameLayout.$.frame, + paintLayout.$.sampler, + cameraUvAt(uv), + ).rgb; +} + +// The source is copied into mip zero before bilinear downsampling each successive level. +export const underpaintLayout = tgpu.bindGroupLayout({ + image: { texture: d.texture2d() }, + sampler: { sampler: 'filtering' }, +}); +export const copyFrameFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f })(({ + uv, +}) => { + 'use gpu'; + return d.vec4f(colorAt(uv), 1); +}); +export const downsampleFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f })(({ + uv, +}) => { + 'use gpu'; + return std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, uv, 0); +}); + +/** Signed edge distance (pixels), paint height (pixels), and bristle pigment variation. */ +function brushField(local: d.v2f, halfSize: d.v2f, seed: number): d.v3f { + 'use gpu'; + const p = local.div(halfSize); + const bend = 0.14 * std.sin(p.x * 2.4 + seed * 6) * std.max(1 - p.x * p.x, d.f32(0)); + const across = p.y - bend; + const width = 0.82 + 0.12 * std.sin(p.x * 3 + seed * 17); + const bristles = std.sin(across * halfSize.y * 1.1 + seed * 83 + std.sin(p.x * 3) * 0.8); + const fine = std.sin(across * halfSize.y * 2.7 + seed * 39); + // Keep the body quiet; dry strands and grooves emerge only at the two ends. + const endTexture = std.smoothstep(0.4, 0.9, std.abs(p.x)); + const texture = paintLayout.$.params.texture * endTexture; + // A cubic superellipse sits between an oval and a rectangle. Bristles shorten + // individual strands near the ends, leaving a solid body and a dry, uneven tip. + const tip = std.smoothstep(0.55, 0.95, std.abs(p.x)); + const strandLength = 1 - texture * tip * (0.07 + 0.07 * bristles + 0.035 * fine); + const q = std.abs(d.vec2f(p.x / strandLength, across / width)); + const outline = std.pow(q.x * q.x * q.x + q.y * q.y * q.y, 1 / 3); + const distance = (outline - 1) * halfSize.y + texture * (bristles * 0.25 + fine * 0.08); + const body = std.smoothstep(d.f32(0), d.f32(2.4), -distance); + const ridges = 0.75 + texture * (0.025 * bristles + 0.006 * fine); + return d.vec3f(distance, body * ridges * 1.7, (bristles * 0.7 + fine * 0.3) * endTexture); +} + +function layerSpacing(layer: number): number { + 'use gpu'; + return paintLayout.$.params.spacing * std.select(d.f32(2), d.f32(1), layer === 1); +} + +/** Compute expensive image/normal queries once per stroke, rather than once per pixel. */ +export const prepareStrokes = tgpu.computeFn({ + workgroupSize: [8, 8, 1], + in: { gid: d.builtin.globalInvocationId }, +})(({ gid }) => { + 'use gpu'; + const spacing = layerSpacing(gid.z); + const grid = d.vec2u(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + if (gid.x >= grid.x || gid.y >= grid.y) { + return; + } + // Seed only by the cell and layer: no frame time and no pixel-dependent jitter. + randf.seed3(d.vec3f(d.vec2f(gid.xy).mul(0.013), d.f32(gid.z) + 0.37)); + const jitter = d.vec2f(randf.sample(), randf.sample()).sub(0.5).mul(0.8); + const center = d.vec2f(gid.xy).add(0.5).add(jitter).mul(spacing); + const uv = center.div(paintLayout.$.params.canvasSize); + const index = gid.z * STROKES_PER_LAYER + gid.y * grid.x + gid.x; + // Mip 6 is a 16x16 memory: coarse enough to ignore sensor noise but still local. + // The absolute last mip (1x1) cannot tell which part of the image moved. + const memory = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 6, + ); + if ( + paintLayout.$.params.resetPaint === 0 && + std.distance(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb) < 0.035 + ) { + strokeWriteLayout.$.strokes[index].dirty = 0; + return; + } + const color = colorAt(uv); + const offset = d.vec2f(spacing * 0.35).div(paintLayout.$.params.canvasSize); + const left = colorAt(uv.sub(d.vec2f(offset.x, 0))); + const right = colorAt(uv.add(d.vec2f(offset.x, 0))); + const up = colorAt(uv.sub(d.vec2f(0, offset.y))); + const down = colorAt(uv.add(d.vec2f(0, offset.y))); + // Local high-pass energy also sees chromatic detail and symmetric thin features. + const mean = left.add(right).add(up).add(down).mul(0.25); + const energy = + std.length(color.sub(mean)) + + 0.5 * std.max(std.length(right.sub(left)), std.length(down.sub(up))); + const detail = std.saturate(energy * paintLayout.$.params.detail * 4); + const luminance = d.vec3f(0.2126, 0.7152, 0.0722); + const gradient = d.vec2f(std.dot(right.sub(left), luminance), std.dot(down.sub(up), luminance)); + const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); + // Surface.xy stores depth slopes. The projected normal's perpendicular follows contours. + const normal = std.normalize(d.vec3f(surface.xy.mul(-80), 1)); + let angle = d.f32(-0.65); + if (std.length(gradient) > 0.005) { + angle = std.atan2(gradient.x, -gradient.y); + } + const fallback = d.vec2f(std.cos(angle), std.sin(angle)); + let tangent = d.vec2f(fallback); + if (std.length(normal.xy) > 0.01) { + tangent = std.normalize(d.vec2f(-normal.y, normal.x)); + // Axes are unoriented: align their signs before interpolating to avoid cancellation. + if (std.dot(tangent, fallback) < 0) { + tangent = tangent.mul(-1); + } + } + const axis = std.normalize(std.mix(fallback, tangent, paintLayout.$.params.normalInfluence)); + const size = std.mix(d.vec2f(1.55, 0.85), d.vec2f(0.9, 0.52), detail).mul(spacing); + strokeWriteLayout.$.strokes[index] = Stroke({ + center, + axis, + memory, + dirty: 1, + halfSize: size, + grain: randf.sample(), + detail, + color: d.vec4f(std.mix(color, mean, (1 - detail) * 0.35), 1), + }); +}); + +export const paintFragment = tgpu.fragmentFn({ + in: { uv: d.vec2f }, + out: d.vec4f, +})(({ uv }) => { + 'use gpu'; + const camera = colorAt(uv); + const mode = paintLayout.$.params.mode; + if (mode === PaintMode.CAMERA) { + return d.vec4f(camera, 1); + } + const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); + if (mode === PaintMode.DEPTH) { + return d.vec4f(d.vec3f(surface.w), 1); + } + if (mode === PaintMode.NORMALS) { + return d.vec4f( + std + .normalize(d.vec3f(surface.xy.mul(-80), 1)) + .mul(0.5) + .add(0.5), + 1, + ); + } + const pixel = uv.mul(paintLayout.$.params.canvasSize); + if (mode === PaintMode.DETAIL) { + const grid = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + const cell = std.min(d.vec2u(pixel.div(paintLayout.$.params.spacing)), grid.sub(1)); + const detail = strokeReadLayout.$.strokes[STROKES_PER_LAYER + cell.y * grid.x + cell.x].detail; + return d.vec4f(d.vec3f(detail), 1); + } + // Only the low-frequency image reaches the gaps between strokes (about 8x8 texels). + let painted = std.textureSampleLevel( + strokeReadLayout.$.history, + paintLayout.$.sampler, + uv, + 0, + ).rgb; + if (paintLayout.$.params.resetPaint !== 0) { + painted = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 7.3, + ).rgb; + } + // Coarse underpainting first, finer marks last. Within a layer, ascending row-major + // cell indices give every overlap the same global order, regardless of query pixel. + for (const layer of std.range(2)) { + const spacing = layerSpacing(layer); + const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + const cell = d.vec2i(std.floor(pixel.div(spacing))); + // Including the short shadow, stroke radius stays below 2.1 cells. An excluded cell's + // center is at least 3 - 0.5 (pixel) - 0.4 (jitter) = 2.1 cells away. + for (const y of std.range(-2, 3)) { + for (const x of std.range(-2, 3)) { + const neighbor = cell.add(d.vec2i(x, y)); + if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { + continue; + } + const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); + const stroke = strokeReadLayout.$.strokes[index]; + if (stroke.dirty === 0) { + continue; + } + const delta = pixel.sub(stroke.center); + const local = d.vec2f( + std.dot(delta, stroke.axis), + std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ); + // Include the tiny downward shadow in the conservative stroke footprint. + if (std.length(delta) > spacing * 2.05) { + continue; + } + const field = brushField(local, stroke.halfSize, stroke.grain); + const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); + // A light above the canvas casts a short shadow downward onto the already + // composited (lower-index) paint. Applying it before this stroke prevents self-shadowing. + const shadowDelta = d.vec2f(0, 1.5); + const shadowLocal = local.sub( + d.vec2f( + std.dot(shadowDelta, stroke.axis), + std.dot(shadowDelta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ), + ); + const shadowField = brushField(shadowLocal, stroke.halfSize, stroke.grain); + const shadow = + (1 - std.smoothstep(-0.5, 1.2, shadowField.x)) * + (1 - coverage) * + 0.09 * + paintLayout.$.params.opacity; + painted = painted.mul(1 - shadow); + if (coverage <= 0) { + continue; + } + // The per-stroke height field supplies its own bevel and bristle normals. + const dx = + brushField(local.add(d.vec2f(0.5, 0)), stroke.halfSize, stroke.grain).y - field.y; + const dy = + brushField(local.add(d.vec2f(0, 0.5)), stroke.halfSize, stroke.grain).y - field.y; + const slope = stroke.axis + .mul(dx) + .add(d.vec2f(-stroke.axis.y, stroke.axis.x).mul(dy)) + .mul(2); + const normal = std.normalize(d.vec3f(slope.mul(-1), 1)); + const light = std.normalize(d.vec3f(-0.2, -0.85, 1)); + const halfLight = std.normalize(light.add(d.vec3f(0, 0, 1))); + const diffuse = std.max(std.dot(normal, light), d.f32(0)); + const specular = std.pow(std.max(std.dot(normal, halfLight), d.f32(0)), d.f32(26)); + const pigment = stroke.color.rgb + .mul((0.84 + 0.2 * diffuse) * (1 + paintLayout.$.params.texture * field.z * 0.025)) + .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.045)); + const opacity = coverage * paintLayout.$.params.opacity; + painted = std.mix(painted, pigment, opacity); + } + } + } + return d.vec4f(std.saturate(painted), 1); +}); diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts new file mode 100644 index 0000000000..786429c6e6 --- /dev/null +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -0,0 +1,42 @@ +import { describe, expect } from 'vitest'; +import { it } from 'typegpu-testing-utility'; +import { common } from 'typegpu'; +import { + prepareStrokes, + copyFrameFragment, + downsampleFragment, + paintFragment, +} from '../../src/examples/image-processing/monocular-painting/shaders.ts'; + +describe('monocular painting shaders', () => { + it('resolves frame-copy and bilinear mip pipelines', async ({ root, device }) => { + for (const fragment of [copyFrameFragment, downsampleFragment]) { + await root + .createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment, + targets: { format: 'rgba8unorm' }, + }) + .initAsync(); + } + expect(device.mock.createShaderModule).toHaveBeenCalledTimes(2); + }); + it('resolves stroke preparation and painting pipelines', async ({ root, device }) => { + await root + .createComputePipeline({ compute: prepareStrokes }) + .$name('prepare strokes') + .initAsync(); + await root + .createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }) + .$name('paint strokes') + .initAsync(); + expect(device.mock.createShaderModule).toHaveBeenCalledTimes(2); + expect(device.mock.createShaderModule).toHaveBeenCalledWith( + expect.objectContaining({ code: expect.stringContaining('texture_external') }), + ); + }); +}); diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts new file mode 100644 index 0000000000..3f11a4bef1 --- /dev/null +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -0,0 +1,97 @@ +import { tgpu, d } from 'typegpu'; +import { + PaintParams, + Stroke, + STROKES_PER_LAYER, + prepareStrokes, + paintLayout, + paintFrameLayout, + strokeWriteLayout, + underpaintLayout, +} from '../src/examples/image-processing/monocular-painting/shaders.ts'; +import { expect, test } from 'vitest'; + +test('redraws only strokes over changed coarse camera colors', async () => { + const root = await tgpu.init(); + try { + const errors: string[] = []; + root.device.addEventListener('uncapturederror', (event) => errors.push(event.error.message)); + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 1024; + const ctx = canvas.getContext('2d')!; + ctx.fillStyle = '#666666'; + ctx.fillRect(0, 0, 1024, 1024); + const image = root + .createTexture({ size: [1024, 1024], format: 'rgba8unorm', mipLevelCount: 11 }) + .$usage('sampled', 'render'); + const surface = root.createTexture({ size: [1, 1], format: 'rgba8unorm' }).$usage('sampled'); + const sampler = root.createSampler({ + minFilter: 'linear', + magFilter: 'linear', + mipmapFilter: 'linear', + }); + const params = root + .createBuffer(PaintParams, { + uvTransform: d.mat2x2f.identity(), + canvasSize: d.vec2f(256), + spacing: 32, + detail: 1.4, + texture: 1, + opacity: 1, + normalInfluence: 0.9, + swapAxes: 0, + mirror: 0, + mode: 0, + resetPaint: 1, + }) + .$usage('uniform'); + const strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER + 64)).$usage('storage'); + const group = root.createBindGroup(paintLayout, { + params, + surface: surface.createView(), + sampler, + }); + const writeGroup = root.createBindGroup(strokeWriteLayout, { strokes }); + const mipGroup = root.createBindGroup(underpaintLayout, { image: image.createView(), sampler }); + const pipeline = root.createComputePipeline({ compute: prepareStrokes }); + async function frame() { + image.write(canvas); + image.generateMipmaps(); + const source = new VideoFrame(canvas, { timestamp: 0 }); + const external = root.device.importExternalTexture({ source }); + pipeline + .with(group) + .with(writeGroup) + .with(mipGroup) + .with(root.createBindGroup(paintFrameLayout, { frame: external })) + .dispatchWorkgroups(1, 1, 2); + const all = await strokes.read(); + source.close(); + return [...all.slice(0, 16), ...all.slice(STROKES_PER_LAYER, STROKES_PER_LAYER + 64)]; + } + const first = await frame(); + params.patch({ resetPaint: 0 }); + const quiet = await frame(); + ctx.fillStyle = '#cc4433'; + ctx.fillRect(0, 0, 512, 1024); + const moved = await frame(); + const settled = await frame(); + const results = { + first: first.filter((s) => s.dirty).length, + quiet: quiet.filter((s) => s.dirty).length, + moved: moved.filter((s) => s.dirty).length, + farRight: moved.filter((s) => s.center.x > 160 && s.dirty).length, + settled: settled.filter((s) => s.dirty).length, + errors, + }; + expect(results.first).toBe(80); + expect(results.quiet).toBe(0); + expect(results.moved).toBeGreaterThan(0); + expect(results.moved).toBeLessThan(80); + expect(results.farRight).toBe(0); + expect(results.settled).toBe(0); + expect(errors).toEqual([]); + } finally { + root.destroy(); + } +}); From e3afdf1bec28cc208ea201ee089de7d1a8b99b68 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Mon, 14 Sep 2026 18:06:31 +0200 Subject: [PATCH 3/8] Better brush texture --- .../monocular-painting/renderer.ts | 32 ++++++++- .../monocular-painting/shaders.ts | 65 ++++++++++++------- 2 files changed, 74 insertions(+), 23 deletions(-) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index 8d40beca74..cb82d9422f 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -1,3 +1,4 @@ +import { randf } from '@typegpu/noise'; import { DEPTH_WORKGROUP_SIZE, DepthParams, @@ -9,7 +10,7 @@ import { surfaceKernel, surfaceLayout, } from '../monocular-light-injection/shaders.ts'; -import { common, d } from 'typegpu'; +import { common, d, std } from 'typegpu'; import type { SampledFlag, RenderFlag, @@ -95,6 +96,9 @@ export class DepthPaintingRenderer { readonly #depthParams: TgpuBuffer & UniformFlag; readonly #paintParams: TgpuBuffer & UniformFlag; readonly #sampler: TgpuSampler; + readonly #brushGrain: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & + SampledFlag & + StorageFlag; readonly #rangeBindGroup: TgpuBindGroup; readonly #stabilizePipeline: TgpuComputePipeline; readonly #depthPipeline: TgpuComputePipeline; @@ -167,10 +171,35 @@ export class DepthPaintingRenderer { .$usage('sampled', 'render'), ); this.#historyViews = this.#paintHistory.map((texture) => root.unwrap(texture).createView()); + // Stable, independent noise channels sampled at different scales in brush space. + // Build once; no per-frame random changes that could make the paint sparkle. + this.#brushGrain = root + .createTexture({ size: [128, 128], format: 'rgba8unorm' }) + .$usage('sampled', 'storage'); + const grainOutput = this.#brushGrain.createView(d.textureStorage2d('rgba8unorm', 'write-only')); + root + .createGuardedComputePipeline((x: number, y: number) => { + 'use gpu'; + randf.seed2(d.vec2f(x, y).add(0.731).div(128)); + std.textureStore( + grainOutput.$, + d.vec2u(x, y), + d.vec4f(randf.sample(), randf.sample(), randf.sample(), randf.sample()), + ); + }) + .dispatchThreads(128, 128); + const grainSampler = root.createSampler({ + magFilter: 'linear', + minFilter: 'linear', + addressModeU: 'repeat', + addressModeV: 'repeat', + }); this.#strokeReadGroups = this.#paintHistory.map((texture) => root.createBindGroup(strokeReadLayout, { strokes: this.#strokes, history: texture.createView(), + grain: this.#brushGrain.createView(), + grainSampler, }), ); this.#presentGroups = this.#paintHistory.map((texture) => @@ -399,6 +428,7 @@ export class DepthPaintingRenderer { this.#paintParams.destroy(); this.#strokes.destroy(); this.#underpainting.destroy(); + this.#brushGrain.destroy(); for (const texture of this.#paintHistory) { texture.destroy(); } diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts index d71f99fea2..327ac5d33f 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -38,6 +38,8 @@ export const strokeWriteLayout = tgpu.bindGroupLayout({ strokes: { storage: d.arrayOf(Stroke), access: 'mutable' }, }); export const strokeReadLayout = tgpu.bindGroupLayout({ + grain: { texture: d.texture2d() }, + grainSampler: { sampler: 'filtering' }, history: { texture: d.texture2d() }, strokes: { storage: d.arrayOf(Stroke), access: 'readonly' }, }); @@ -85,28 +87,46 @@ export const downsampleFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d. return std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, uv, 0); }); -/** Signed edge distance (pixels), paint height (pixels), and bristle pigment variation. */ -function brushField(local: d.v2f, halfSize: d.v2f, seed: number): d.v3f { +function grainAt(position: d.v2f): d.v4f { + 'use gpu'; + return std.textureSampleLevel( + strokeReadLayout.$.grain, + strokeReadLayout.$.grainSampler, + position.div(128), + 0, + ); +} + +/** Edge distance, irregular paint height, pigment variation, and pigment coverage. */ +function brushField(local: d.v2f, halfSize: d.v2f, seed: number): d.v4f { 'use gpu'; const p = local.div(halfSize); - const bend = 0.14 * std.sin(p.x * 2.4 + seed * 6) * std.max(1 - p.x * p.x, d.f32(0)); + const origin = d.vec2f(seed * 97, seed * 61); + const pressure = grainAt(p.mul(d.vec2f(2.4, 1.6)).add(origin)); + const bend = (pressure.x - 0.5) * 0.24 * std.max(1 - p.x * p.x, d.f32(0)); const across = p.y - bend; - const width = 0.82 + 0.12 * std.sin(p.x * 3 + seed * 17); - const bristles = std.sin(across * halfSize.y * 1.1 + seed * 83 + std.sin(p.x * 3) * 0.8); - const fine = std.sin(across * halfSize.y * 2.7 + seed * 39); - // Keep the body quiet; dry strands and grooves emerge only at the two ends. - const endTexture = std.smoothstep(0.4, 0.9, std.abs(p.x)); - const texture = paintLayout.$.params.texture * endTexture; - // A cubic superellipse sits between an oval and a rectangle. Bristles shorten - // individual strands near the ends, leaving a solid body and a dry, uneven tip. - const tip = std.smoothstep(0.55, 0.95, std.abs(p.x)); - const strandLength = 1 - texture * tip * (0.07 + 0.07 * bristles + 0.035 * fine); + // Unequal bristle bundles, with fine grain breaking the tracks into deposits. + // These random fields have no periodic wave or evenly spaced comb pattern. + const bundles = grainAt( + d.vec2f(p.x * 2.2 + pressure.y * 0.6, across * halfSize.y * 0.65).add(origin), + ); + const tooth = grainAt(local.mul(0.85).add(origin.mul(1.7))); + const endTexture = std.smoothstep(0.35, 0.95, std.abs(p.x)); + const texture = paintLayout.$.params.texture; + const width = 0.84 + texture * (pressure.z - 0.5) * 0.24; + const strandLength = 1 - texture * endTexture * (0.04 + 0.24 * bundles.x); const q = std.abs(d.vec2f(p.x / strandLength, across / width)); const outline = std.pow(q.x * q.x * q.x + q.y * q.y * q.y, 1 / 3); - const distance = (outline - 1) * halfSize.y + texture * (bristles * 0.25 + fine * 0.08); - const body = std.smoothstep(d.f32(0), d.f32(2.4), -distance); - const ridges = 0.75 + texture * (0.025 * bristles + 0.006 * fine); - return d.vec3f(distance, body * ridges * 1.7, (bristles * 0.7 + fine * 0.3) * endTexture); + const distance = (outline - 1) * halfSize.y + texture * (tooth.x - 0.5) * 0.65; + const edge = std.smoothstep(d.f32(0), d.f32(1.8), -distance); + const dry = std.smoothstep(0.24, 0.64, bundles.y * 0.75 + tooth.y * 0.25); + const deposit = 1 - texture * endTexture * (1 - dry) * 0.9; + // Low, uneven deposits instead of an extruded stamp with a uniform shiny rim. + const height = edge * deposit * (0.16 + texture * pressure.w * 0.22); + const pigment = + texture * + ((pressure.y - 0.5) * 0.12 + (tooth.z - 0.5) * 0.035 + endTexture * (bundles.z - 0.5) * 0.07); + return d.vec4f(distance, height, pigment, deposit); } function layerSpacing(layer: number): number { @@ -269,7 +289,7 @@ export const paintFragment = tgpu.fragmentFn({ const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); // A light above the canvas casts a short shadow downward onto the already // composited (lower-index) paint. Applying it before this stroke prevents self-shadowing. - const shadowDelta = d.vec2f(0, 1.5); + const shadowDelta = d.vec2f(0, 0.7); const shadowLocal = local.sub( d.vec2f( std.dot(shadowDelta, stroke.axis), @@ -280,7 +300,8 @@ export const paintFragment = tgpu.fragmentFn({ const shadow = (1 - std.smoothstep(-0.5, 1.2, shadowField.x)) * (1 - coverage) * - 0.09 * + shadowField.w * + 0.04 * paintLayout.$.params.opacity; painted = painted.mul(1 - shadow); if (coverage <= 0) { @@ -301,9 +322,9 @@ export const paintFragment = tgpu.fragmentFn({ const diffuse = std.max(std.dot(normal, light), d.f32(0)); const specular = std.pow(std.max(std.dot(normal, halfLight), d.f32(0)), d.f32(26)); const pigment = stroke.color.rgb - .mul((0.84 + 0.2 * diffuse) * (1 + paintLayout.$.params.texture * field.z * 0.025)) - .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.045)); - const opacity = coverage * paintLayout.$.params.opacity; + .mul((0.95 + 0.07 * diffuse) * (1 + field.z)) + .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.008)); + const opacity = coverage * field.w * paintLayout.$.params.opacity; painted = std.mix(painted, pigment, opacity); } } From 4f4e6efcb5409ad05e90d50d2743a5dbdf348b06 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Mon, 14 Sep 2026 18:17:58 +0200 Subject: [PATCH 4/8] Staggered painting, animated strokes --- .../monocular-painting/renderer.ts | 7 +++ .../monocular-painting/shaders.ts | 45 ++++++++++++++++++- .../monocular-painting.test.ts | 23 ++++++++++ .../tests/monocular-painting.test.browser.ts | 20 +++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index cb82d9422f..71be9e65ec 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -115,6 +115,8 @@ export class DepthPaintingRenderer { readonly #presentPipeline: TgpuRenderPipeline; #historyIndex = 0; #paintDirty = true; + #lastPaintTime = performance.now(); + #revealStep = 1 / 6; readonly #strokePipeline: TgpuComputePipeline; readonly #underpainting: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & SampledFlag & @@ -338,6 +340,10 @@ export class DepthPaintingRenderer { } const updateDepth = !options?.skipDepth || this.#firstFrame; + const now = performance.now(); + // A 400ms stroke; cap long gaps so returning to the tab does not pop marks in. + this.#revealStep = Math.min(Math.max(now - this.#lastPaintTime, 1), 50) / 100; + this.#lastPaintTime = now; this.#syncCanvasSize(); this.#uvTransform = frame.uvTransform; this.#swapAxes = frame.swapAxes; @@ -462,6 +468,7 @@ export class DepthPaintingRenderer { mirror: this.#settings.mirror ? 1 : 0, mode: this.#settings.mode, resetPaint: this.#paintDirty ? 1 : 0, + revealStep: this.#revealStep, }); } } diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts index 327ac5d33f..ed316aa285 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -12,6 +12,8 @@ export const Stroke = d.struct({ color: d.vec4f, memory: d.vec4f, dirty: d.u32, + progress: d.f32, + previousProgress: d.f32, }); export const PaintParams = d.struct({ uvTransform: d.mat2x2f, @@ -25,6 +27,7 @@ export const PaintParams = d.struct({ mirror: d.u32, mode: d.u32, resetPaint: d.u32, + revealStep: d.f32, }); export const paintLayout = tgpu.bindGroupLayout({ params: { uniform: PaintParams }, @@ -159,6 +162,18 @@ export const prepareStrokes = tgpu.computeFn({ uv, 6, ); + // Finish a mark before sampling a new one, so continuous movement cannot + // repeatedly restart the fade or change its color/orientation halfway through. + if (paintLayout.$.params.resetPaint === 0 && strokeWriteLayout.$.strokes[index].progress < 1) { + const previous = strokeWriteLayout.$.strokes[index].progress; + strokeWriteLayout.$.strokes[index].previousProgress = previous; + strokeWriteLayout.$.strokes[index].progress = std.min( + previous + paintLayout.$.params.revealStep, + d.f32(1), + ); + strokeWriteLayout.$.strokes[index].dirty = 1; + return; + } if ( paintLayout.$.params.resetPaint === 0 && std.distance(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb) < 0.035 @@ -203,6 +218,9 @@ export const prepareStrokes = tgpu.computeFn({ axis, memory, dirty: 1, + previousProgress: 0, + // Independent 5–15 ms delay from the moment this mark is needed. + progress: std.min(paintLayout.$.params.revealStep - (0.05 + randf.sample() * 0.1), d.f32(1)), halfSize: size, grain: randf.sample(), detail, @@ -210,6 +228,21 @@ export const prepareStrokes = tgpu.computeFn({ }); }); +/** Soft brush front traveling from local -X to +X, exactly empty/full at 0/1. */ +export function revealAt(along: number, progress: number): number { + 'use gpu'; + const position = std.saturate(along * 0.5 + 0.5); + const remaining = 1 - std.saturate(progress); + const eased = 1 - remaining * remaining * remaining; + return std.smoothstep(position - 0.12, position + 0.12, eased * 1.24 - 0.12); +} + +/** Add just the missing coverage to a persistent surface, independent of frame count. */ +export function incrementalAlpha(alpha: number, before: number, after: number): number { + 'use gpu'; + return std.saturate((alpha * (after - before)) / std.max(1 - alpha * before, 0.00001)); +} + export const paintFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f, @@ -303,7 +336,9 @@ export const paintFragment = tgpu.fragmentFn({ shadowField.w * 0.04 * paintLayout.$.params.opacity; - painted = painted.mul(1 - shadow); + const shadowBefore = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.previousProgress); + const shadowAfter = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.progress); + painted = painted.mul(1 - incrementalAlpha(shadow, shadowBefore, shadowAfter)); if (coverage <= 0) { continue; } @@ -324,7 +359,13 @@ export const paintFragment = tgpu.fragmentFn({ const pigment = stroke.color.rgb .mul((0.95 + 0.07 * diffuse) * (1 + field.z)) .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.008)); - const opacity = coverage * field.w * paintLayout.$.params.opacity; + const before = revealAt(local.x / stroke.halfSize.x, stroke.previousProgress); + const after = revealAt(local.x / stroke.halfSize.x, stroke.progress); + const opacity = incrementalAlpha( + coverage * field.w * paintLayout.$.params.opacity, + before, + after, + ); painted = std.mix(painted, pigment, opacity); } } diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts index 786429c6e6..5e53245222 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -3,12 +3,35 @@ import { it } from 'typegpu-testing-utility'; import { common } from 'typegpu'; import { prepareStrokes, + revealAt, + incrementalAlpha, copyFrameFragment, downsampleFragment, paintFragment, } from '../../src/examples/image-processing/monocular-painting/shaders.ts'; describe('monocular painting shaders', () => { + it('reveals in stroke direction without multiplying opacity across frames', () => { + expect(revealAt(-0.5, 0.2)).toBeGreaterThan(revealAt(0.5, 0.2)); + expect(revealAt(0.5, 0.5)).toBe(1); + for (const position of [-1, -0.5, 0, 0.5, 1]) { + expect(revealAt(position, -0.1)).toBe(0); + expect(revealAt(position, 0)).toBe(0); + expect(revealAt(position, 1)).toBe(1); + for (const steps of [8, 24, 60]) { + let color = 1; + for (let step = 1; step <= steps; step++) { + const alpha = incrementalAlpha( + 0.45, + revealAt(position, (step - 1) / steps), + revealAt(position, step / steps), + ); + color = color * (1 - alpha) + 0.3 * alpha; + } + expect(color).toBeCloseTo(1 * 0.55 + 0.3 * 0.45, 5); + } + } + }); it('resolves frame-copy and bilinear mip pipelines', async ({ root, device }) => { for (const fragment of [copyFrameFragment, downsampleFragment]) { await root diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts index 3f11a4bef1..d68d4f60da 100644 --- a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -43,6 +43,7 @@ test('redraws only strokes over changed coarse camera colors', async () => { mirror: 0, mode: 0, resetPaint: 1, + revealStep: 10, }) .$usage('uniform'); const strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER + 64)).$usage('storage'); @@ -84,6 +85,25 @@ test('redraws only strokes over changed coarse camera colors', async () => { settled: settled.filter((s) => s.dirty).length, errors, }; + // Animation keeps a stationary stroke active until it finishes, then stops. + params.patch({ resetPaint: 1, revealStep: 0.25 }); + const start = await frame(); + params.patch({ resetPaint: 0 }); + const middle = await frame(); + await frame(); + const end = await frame(); + expect(start.every((s) => s.progress >= 0.1 && s.progress <= 0.2)).toBe(true); + expect(new Set(start.map((s) => s.progress)).size).toBeGreaterThan(1); + expect(middle.every((s, i) => Math.abs(s.progress - start[i].progress - 0.25) < 0.00001)).toBe(true); + expect(end.every((s) => s.progress >= 0.85 && s.progress <= 0.95)).toBe(true); + const complete = await frame(); + const idle = await frame(); + expect(complete.every((s) => s.progress === 1 && s.dirty === 1)).toBe(true); + expect(idle.every((s) => s.dirty === 0)).toBe(true); + // At 200 FPS the first frame is still inside every stroke's delay. + params.patch({ resetPaint: 1, revealStep: 0.05 }); + const delayed = await frame(); + expect(delayed.every((s) => s.progress <= 0 && s.previousProgress === 0)).toBe(true); expect(results.first).toBe(80); expect(results.quiet).toBe(0); expect(results.moved).toBeGreaterThan(0); From 11553f8646b82e87a26fd12260e3daab88b4d2e2 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Mon, 14 Sep 2026 18:57:06 +0200 Subject: [PATCH 5/8] Cool but slow --- .../monocular-painting/renderer.ts | 2 +- .../monocular-painting/shaders.ts | 252 +++++++++++++----- .../monocular-painting.test.ts | 34 ++- .../tests/monocular-painting.test.browser.ts | 35 ++- 4 files changed, 246 insertions(+), 77 deletions(-) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index 71be9e65ec..93e802805f 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -341,7 +341,7 @@ export class DepthPaintingRenderer { const updateDepth = !options?.skipDepth || this.#firstFrame; const now = performance.now(); - // A 400ms stroke; cap long gaps so returning to the tab does not pop marks in. + // A 100ms stroke; cap long gaps so returning to the tab does not pop marks in. this.#revealStep = Math.min(Math.max(now - this.#lastPaintTime, 1), 50) / 100; this.#lastPaintTime = now; this.#syncCanvasSize(); diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts index ed316aa285..2d4b3ddb92 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -9,8 +9,11 @@ export const Stroke = d.struct({ halfSize: d.vec2f, grain: d.f32, detail: d.f32, + depth: d.f32, + opacity: d.f32, color: d.vec4f, memory: d.vec4f, + footprintMemory: d.arrayOf(d.vec4f, 4), dirty: d.u32, progress: d.f32, previousProgress: d.f32, @@ -137,6 +140,48 @@ function layerSpacing(layer: number): number { return paintLayout.$.params.spacing * std.select(d.f32(2), d.f32(1), layer === 1); } +/** Normalized disparity is 0 at the far plane and 1 at the near plane. */ +export function depthBlurAmount(depth: number): number { + 'use gpu'; + // Keep the near half in focus; ramp only over the far half. + return std.saturate(1 - 2 * depth); +} + +export function depthStrokeScale(depth: number): number { + 'use gpu'; + // UI depth runs near-to-far; the model's disparity runs in the opposite direction. + const near = std.saturate(2 * depth - 1); + const far = depthBlurAmount(depth); + const remaining = 1 - far; + // Grow gently toward the camera, and quickly toward the larger background marks. + return 1 + 1.4 * near * near + 2.4 * (1 - remaining * remaining * remaining); +} + +export function strokeComesBefore(depth: number, index: number, otherDepth: number, otherIndex: number): boolean { + 'use gpu'; + return depth > otherDepth || (depth === otherDepth && index > otherIndex); +} + +/** Luma dominates motion; each chroma axis has one quarter of its weight. */ +export function cameraChange(current: d.v3f, previous: d.v3f): number { + 'use gpu'; + const delta = current.sub(previous); + const luma = std.dot(delta, d.vec3f(0.2126, 0.7152, 0.0722)); + const cb = (delta.z - luma) / 1.8556; + const cr = (delta.x - luma) / 1.5748; + // Keep neutral brightness changes on the existing threshold/opacity scale. + return std.length(d.vec3f(luma, cb * 0.25, cr * 0.25)) * 1.7320508; +} + +/** Preserve gentle repainting near the motion threshold; cover large changes fully. */ +export function changeOpacity(change: number, baseOpacity: number): number { + 'use gpu'; + if (baseOpacity <= 0) { + return 0; + } + return std.mix(baseOpacity, d.f32(1), std.smoothstep(0.035, 0.2, change)); +} + /** Compute expensive image/normal queries once per stroke, rather than once per pixel. */ export const prepareStrokes = tgpu.computeFn({ workgroupSize: [8, 8, 1], @@ -154,13 +199,12 @@ export const prepareStrokes = tgpu.computeFn({ const center = d.vec2f(gid.xy).add(0.5).add(jitter).mul(spacing); const uv = center.div(paintLayout.$.params.canvasSize); const index = gid.z * STROKES_PER_LAYER + gid.y * grid.x + gid.x; - // Mip 6 is a 16x16 memory: coarse enough to ignore sensor noise but still local. - // The absolute last mip (1x1) cannot tell which part of the image moved. + // Mip 4 retains local motion that vanished in the old 16x16 average. const memory = std.textureSampleLevel( underpaintLayout.$.image, underpaintLayout.$.sampler, uv, - 6, + 4, ); // Finish a mark before sampling a new one, so continuous movement cannot // repeatedly restart the fade or change its color/orientation halfway through. @@ -174,10 +218,22 @@ export const prepareStrokes = tgpu.computeFn({ strokeWriteLayout.$.strokes[index].dirty = 1; return; } - if ( - paintLayout.$.params.resetPaint === 0 && - std.distance(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb) < 0.035 - ) { + const footprintMemory = d.arrayOf(d.vec4f, 4)(); + let change = cameraChange(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb); + for (const probe of std.range(4)) { + let offset = d.vec2f(0.7, 0); + if (probe === 1) { offset = d.vec2f(-0.7, 0); } + if (probe === 2) { offset = d.vec2f(0, 0.7); } + if (probe === 3) { offset = d.vec2f(0, -0.7); } + const sample = std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, + uv.add(offset.mul(spacing).div(paintLayout.$.params.canvasSize)), 4); + footprintMemory[probe] = d.vec4f(sample); + change = std.max(change, cameraChange(sample.rgb, strokeWriteLayout.$.strokes[index].footprintMemory[probe].rgb)); + } + const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); + // A surface leaving the foreground must release its old depth occlusion too. + change = std.max(change, std.abs(surface.w - strokeWriteLayout.$.strokes[index].depth) * 0.5); + if (paintLayout.$.params.resetPaint === 0 && change < 0.035) { strokeWriteLayout.$.strokes[index].dirty = 0; return; } @@ -195,7 +251,6 @@ export const prepareStrokes = tgpu.computeFn({ const detail = std.saturate(energy * paintLayout.$.params.detail * 4); const luminance = d.vec3f(0.2126, 0.7152, 0.0722); const gradient = d.vec2f(std.dot(right.sub(left), luminance), std.dot(down.sub(up), luminance)); - const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); // Surface.xy stores depth slopes. The projected normal's perpendicular follows contours. const normal = std.normalize(d.vec3f(surface.xy.mul(-80), 1)); let angle = d.f32(-0.65); @@ -212,11 +267,28 @@ export const prepareStrokes = tgpu.computeFn({ } } const axis = std.normalize(std.mix(fallback, tangent, paintLayout.$.params.normalInfluence)); - const size = std.mix(d.vec2f(1.55, 0.85), d.vec2f(0.9, 0.52), detail).mul(spacing); + // The model returns normalized disparity: larger values are closer. + const depth = std.saturate(surface.w); + const distanceScale = depthStrokeScale(depth); + const blur = depthBlurAmount(depth); + const size = std + .mix(d.vec2f(1.55, 0.85), d.vec2f(0.9, 0.52), detail) + .mul(spacing * distanceScale); + const softened = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + blur * 3.5, + ).rgb; + let opacity = paintLayout.$.params.opacity; + if (paintLayout.$.params.resetPaint === 0) { + opacity = changeOpacity(change, opacity); + } strokeWriteLayout.$.strokes[index] = Stroke({ center, axis, memory, + footprintMemory, dirty: 1, previousProgress: 0, // Independent 5–15 ms delay from the moment this mark is needed. @@ -224,7 +296,12 @@ export const prepareStrokes = tgpu.computeFn({ halfSize: size, grain: randf.sample(), detail, - color: d.vec4f(std.mix(color, mean, (1 - detail) * 0.35), 1), + depth, + opacity, + color: d.vec4f( + std.mix(std.mix(color, mean, (1 - detail) * 0.35), softened, blur * 0.7), + 1, + ), }); }); @@ -290,85 +367,116 @@ export const paintFragment = tgpu.fragmentFn({ 7.3, ).rgb; } - // Coarse underpainting first, finer marks last. Within a layer, ascending row-major - // cell indices give every overlap the same global order, regardless of query pixel. + // Gather both grids into one depth order. Equal depths retain fine-over-coarse + // and row-major ordering. Near-to-far compositing also shields existing near paint + // from distant marks that start their animation on a later frame. + const candidates = d.arrayOf(d.u32, 450)(); + let count = d.u32(0); + let changing = false; for (const layer of std.range(2)) { const spacing = layerSpacing(layer); const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); const cell = d.vec2i(std.floor(pixel.div(spacing))); - // Including the short shadow, stroke radius stays below 2.1 cells. An excluded cell's - // center is at least 3 - 0.5 (pixel) - 0.4 (jitter) = 2.1 cells away. - for (const y of std.range(-2, 3)) { - for (const x of std.range(-2, 3)) { + // Cover 3.4x strokes, including jitter, bent edges and the small shadow. + for (const y of std.range(-7, 8)) { + for (const x of std.range(-7, 8)) { const neighbor = cell.add(d.vec2i(x, y)); if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { continue; } const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); const stroke = strokeReadLayout.$.strokes[index]; - if (stroke.dirty === 0) { + // Invalidate a conservative area, including the previous, possibly larger mark. + changing = changing || stroke.dirty !== 0; + if (stroke.progress <= 0 || std.length(pixel.sub(stroke.center)) > spacing * 6.5) { continue; } + // Reject outside the rotated brush bounds before sorting. The padding + // includes the bent edge, antialiasing, grain displacement and tiny shadow. const delta = pixel.sub(stroke.center); - const local = d.vec2f( - std.dot(delta, stroke.axis), - std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x)), - ); - // Include the tiny downward shadow in the conservative stroke footprint. - if (std.length(delta) > spacing * 2.05) { - continue; - } - const field = brushField(local, stroke.halfSize, stroke.grain); - const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); - // A light above the canvas casts a short shadow downward onto the already - // composited (lower-index) paint. Applying it before this stroke prevents self-shadowing. - const shadowDelta = d.vec2f(0, 0.7); - const shadowLocal = local.sub( - d.vec2f( - std.dot(shadowDelta, stroke.axis), - std.dot(shadowDelta, d.vec2f(-stroke.axis.y, stroke.axis.x)), - ), - ); - const shadowField = brushField(shadowLocal, stroke.halfSize, stroke.grain); - const shadow = - (1 - std.smoothstep(-0.5, 1.2, shadowField.x)) * - (1 - coverage) * - shadowField.w * - 0.04 * - paintLayout.$.params.opacity; - const shadowBefore = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.previousProgress); - const shadowAfter = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.progress); - painted = painted.mul(1 - incrementalAlpha(shadow, shadowBefore, shadowAfter)); - if (coverage <= 0) { + const along = std.abs(std.dot(delta, stroke.axis)); + const across = std.abs(std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x))); + if (along > stroke.halfSize.x + 2 || across > stroke.halfSize.y * 1.15 + 2) { continue; } - // The per-stroke height field supplies its own bevel and bristle normals. - const dx = - brushField(local.add(d.vec2f(0.5, 0)), stroke.halfSize, stroke.grain).y - field.y; - const dy = - brushField(local.add(d.vec2f(0, 0.5)), stroke.halfSize, stroke.grain).y - field.y; - const slope = stroke.axis - .mul(dx) - .add(d.vec2f(-stroke.axis.y, stroke.axis.x).mul(dy)) - .mul(2); - const normal = std.normalize(d.vec3f(slope.mul(-1), 1)); - const light = std.normalize(d.vec3f(-0.2, -0.85, 1)); - const halfLight = std.normalize(light.add(d.vec3f(0, 0, 1))); - const diffuse = std.max(std.dot(normal, light), d.f32(0)); - const specular = std.pow(std.max(std.dot(normal, halfLight), d.f32(0)), d.f32(26)); - const pigment = stroke.color.rgb - .mul((0.95 + 0.07 * diffuse) * (1 + field.z)) - .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.008)); - const before = revealAt(local.x / stroke.halfSize.x, stroke.previousProgress); - const after = revealAt(local.x / stroke.halfSize.x, stroke.progress); - const opacity = incrementalAlpha( - coverage * field.w * paintLayout.$.params.opacity, - before, - after, - ); - painted = std.mix(painted, pigment, opacity); + candidates[count] = index; + count++; } } } + if (!changing) { + return d.vec4f(painted, 1); + } + for (let position = d.u32(1); position < count; position++) { + const index = candidates[position]; + const depth = strokeReadLayout.$.strokes[index].depth; + let insert = position; + while (insert > 0) { + const otherIndex = candidates[insert - 1]; + const otherDepth = strokeReadLayout.$.strokes[otherIndex].depth; + if (strokeComesBefore(otherDepth, otherIndex, depth, index)) { + break; + } + candidates[insert] = otherIndex; + insert--; + } + candidates[insert] = index; + } + // Rebuild only invalidated pixels from remembered current strokes. Blending new + // paint over flattened history retained obsolete foreground paint indefinitely. + painted = std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, uv, 7.3).rgb; + let visibility = d.f32(1); + let added = d.vec3f(0); + for (let candidate = d.u32(0); candidate < count; candidate++) { + const stroke = strokeReadLayout.$.strokes[candidates[candidate]]; + const delta = pixel.sub(stroke.center); + const local = d.vec2f( + std.dot(delta, stroke.axis), + std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ); + const field = brushField(local, stroke.halfSize, stroke.grain); + const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); + // A light above the canvas casts a short shadow downward onto the already + // composited more distant paint. Applying it before this stroke prevents self-shadowing. + const shadowDelta = d.vec2f(0, 0.7); + const shadowLocal = local.sub( + d.vec2f( + std.dot(shadowDelta, stroke.axis), + std.dot(shadowDelta, d.vec2f(-stroke.axis.y, stroke.axis.x)), + ), + ); + const shadowField = brushField(shadowLocal, stroke.halfSize, stroke.grain); + const shadow = + (1 - std.smoothstep(-0.5, 1.2, shadowField.x)) * + (1 - coverage) * + shadowField.w * + 0.04 * + paintLayout.$.params.opacity; + const shadowAfter = revealAt(shadowLocal.x / stroke.halfSize.x, stroke.progress); + visibility *= 1 - shadow * shadowAfter; + if (coverage <= 0) { + continue; + } + // The per-stroke height field supplies its own bevel and bristle normals. + const dx = brushField(local.add(d.vec2f(0.5, 0)), stroke.halfSize, stroke.grain).y - field.y; + const dy = brushField(local.add(d.vec2f(0, 0.5)), stroke.halfSize, stroke.grain).y - field.y; + const slope = stroke.axis.mul(dx).add(d.vec2f(-stroke.axis.y, stroke.axis.x).mul(dy)).mul(2); + const normal = std.normalize(d.vec3f(slope.mul(-1), 1)); + const light = std.normalize(d.vec3f(-0.2, -0.85, 1)); + const halfLight = std.normalize(light.add(d.vec3f(0, 0, 1))); + const diffuse = std.max(std.dot(normal, light), d.f32(0)); + const specular = std.pow(std.max(std.dot(normal, halfLight), d.f32(0)), d.f32(26)); + const pigment = stroke.color.rgb + .mul((0.95 + 0.07 * diffuse) * (1 + field.z)) + .add(d.vec3f(1, 0.96, 0.86).mul(specular * 0.008)); + const after = revealAt(local.x / stroke.halfSize.x, stroke.progress); + const opacity = coverage * field.w * stroke.opacity * after; + added += pigment * opacity * visibility; + visibility *= 1 - opacity; + if (visibility < 0.001) { + break; + } + } + painted = painted * visibility + added; return d.vec4f(std.saturate(painted), 1); }); diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts index 5e53245222..546de08ac9 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -1,8 +1,12 @@ import { describe, expect } from 'vitest'; import { it } from 'typegpu-testing-utility'; -import { common } from 'typegpu'; +import { common, d } from 'typegpu'; import { prepareStrokes, + changeOpacity, + cameraChange, + depthStrokeScale, + strokeComesBefore, revealAt, incrementalAlpha, copyFrameFragment, @@ -11,6 +15,34 @@ import { } from '../../src/examples/image-processing/monocular-painting/shaders.ts'; describe('monocular painting shaders', () => { + it('weights luma above chroma while preserving neutral-change sensitivity', () => { + const gray = d.vec3f(0.5); + const brightness = cameraChange(d.vec3f(0.6), gray); + // A 0.1 Cb shift with unchanged Rec.709 luma. + const chroma = cameraChange(gray.add(d.vec3f(0, -0.018556 * 0.722 / 0.7152, 0.18556)), gray); + expect(cameraChange(gray, gray)).toBe(0); + expect(brightness).toBeCloseTo(Math.sqrt(3) * 0.1); + expect(chroma).toBeCloseTo(brightness * 0.25); + expect(cameraChange(gray, d.vec3f(0.6))).toBeCloseTo(brightness); + }); + it('raises repaint opacity smoothly for stronger camera changes', () => { + expect(changeOpacity(0.035, 0.45)).toBeCloseTo(0.45); + expect(changeOpacity(0.1175, 0.45)).toBeCloseTo(0.725); + expect(changeOpacity(0.2, 0.45)).toBe(1); + expect(changeOpacity(1, 0.45)).toBe(1); + expect(changeOpacity(0.2, 0)).toBe(0); + }); + it('orders nearer strokes first across grids and enlarges distant strokes', () => { + expect(strokeComesBefore(0.9, 0, 0.2, 32768)).toBe(true); + expect(strokeComesBefore(0.2, 32768, 0.9, 0)).toBe(false); + expect(strokeComesBefore(0.5, 32768, 0.5, 0)).toBe(true); + expect(depthStrokeScale(1)).toBeCloseTo(2.4); + expect(depthStrokeScale(0.75)).toBeCloseTo(1.35); + expect(depthStrokeScale(0.5)).toBe(1); + expect(depthStrokeScale(0.25)).toBeCloseTo(3.1); + expect(depthStrokeScale(0)).toBe(3.4); + expect(depthStrokeScale(0.25)).toBeGreaterThan(depthStrokeScale(0.75)); + }); it('reveals in stroke direction without multiplying opacity across frames', () => { expect(revealAt(-0.5, 0.2)).toBeGreaterThan(revealAt(0.5, 0.2)); expect(revealAt(0.5, 0.5)).toBe(1); diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts index d68d4f60da..6515130081 100644 --- a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -1,9 +1,11 @@ -import { tgpu, d } from 'typegpu'; +import { tgpu, d, common } from 'typegpu'; import { PaintParams, Stroke, STROKES_PER_LAYER, prepareStrokes, + paintFragment, + strokeReadLayout, paintLayout, paintFrameLayout, strokeWriteLayout, @@ -11,7 +13,7 @@ import { } from '../src/examples/image-processing/monocular-painting/shaders.ts'; import { expect, test } from 'vitest'; -test('redraws only strokes over changed coarse camera colors', async () => { +test('detects local changes and removes a departed foreground mark without disturbing idle paint', async () => { const root = await tgpu.init(); try { const errors: string[] = []; @@ -55,6 +57,13 @@ test('redraws only strokes over changed coarse camera colors', async () => { const writeGroup = root.createBindGroup(strokeWriteLayout, { strokes }); const mipGroup = root.createBindGroup(underpaintLayout, { image: image.createView(), sampler }); const pipeline = root.createComputePipeline({ compute: prepareStrokes }); + const history = root.createTexture({size: [256,256], format: 'rgba8unorm'}).$usage('sampled'); + const target = root.createTexture({size: [256,256], format: 'rgba8unorm'}).$usage('render'); + const grain = root.createTexture({size: [128,128], format: 'rgba8unorm'}).$usage('sampled'); + grain.write(new Uint8Array(128*128*4).fill(128)); + const paint = root.createRenderPipeline({vertex: common.fullScreenTriangle, fragment: paintFragment, targets: {format: 'rgba8unorm'}}); + const readGroup = root.createBindGroup(strokeReadLayout, {strokes, history, grain, grainSampler: sampler}); + let lastPixels = new Uint8Array(); async function frame() { image.write(canvas); image.generateMipmaps(); @@ -66,11 +75,23 @@ test('redraws only strokes over changed coarse camera colors', async () => { .with(mipGroup) .with(root.createBindGroup(paintFrameLayout, { frame: external })) .dispatchWorkgroups(1, 1, 2); + paint.with(group).with(mipGroup).with(readGroup) + .with(root.createBindGroup(paintFrameLayout, {frame: external})) + .withColorAttachment({view: target}).draw(3); + const buffer = root.device.createBuffer({size: 256*256*4, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ}); + const encoder = root.device.createCommandEncoder(); + encoder.copyTextureToTexture({texture: root.unwrap(target)}, {texture: root.unwrap(history)}, [256,256]); + encoder.copyTextureToBuffer({texture: root.unwrap(target)}, {buffer, bytesPerRow: 1024}, [256,256]); + root.device.queue.submit([encoder.finish()]); + await buffer.mapAsync(GPUMapMode.READ); + lastPixels = new Uint8Array(buffer.getMappedRange()).slice(); + buffer.unmap(); buffer.destroy(); const all = await strokes.read(); source.close(); return [...all.slice(0, 16), ...all.slice(STROKES_PER_LAYER, STROKES_PER_LAYER + 64)]; } const first = await frame(); + const originalPixels = lastPixels.slice(); params.patch({ resetPaint: 0 }); const quiet = await frame(); ctx.fillStyle = '#cc4433'; @@ -81,10 +102,18 @@ test('redraws only strokes over changed coarse camera colors', async () => { first: first.filter((s) => s.dirty).length, quiet: quiet.filter((s) => s.dirty).length, moved: moved.filter((s) => s.dirty).length, - farRight: moved.filter((s) => s.center.x > 160 && s.dirty).length, + farRight: moved.filter((s) => s.center.x > 208 && s.dirty).length, settled: settled.filter((s) => s.dirty).length, errors, }; + ctx.fillStyle = '#666666'; + ctx.fillRect(0, 0, 1024, 1024); + await frame(); + // The departed object must leave no residual color, even in dry brush gaps. + expect(lastPixels).toEqual(originalPixels); + const restored = lastPixels.slice(); + await frame(); + expect(lastPixels).toEqual(restored); // Animation keeps a stationary stroke active until it finishes, then stops. params.patch({ resetPaint: 1, revealStep: 0.25 }); const start = await frame(); From 7db55c0e10e83fcfbc273ff4d90fc2e365672929 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Mon, 14 Sep 2026 19:18:48 +0200 Subject: [PATCH 6/8] It's faster --- .../monocular-painting/renderer.ts | 23 +- .../monocular-painting/shaders.ts | 207 ++++++++++++------ .../monocular-painting.test.ts | 6 +- .../tests/monocular-painting.test.browser.ts | 65 ++++-- 4 files changed, 218 insertions(+), 83 deletions(-) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index 93e802805f..194eedfbf4 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -35,6 +35,9 @@ import { Stroke, STROKES_PER_LAYER, prepareStrokes, + prepareStrokeCells, + StrokeCell, + cellWriteLayout, strokeWriteLayout, strokeReadLayout, PaintParams, @@ -78,7 +81,7 @@ type PaintingSettings = Partial; export const defaultPaintingSettings: PaintingState = { mirror: true, - spacing: 16, + spacing: 7, detail: 1.5, texture: 1, opacity: 0.45, @@ -117,6 +120,9 @@ export class DepthPaintingRenderer { #paintDirty = true; #lastPaintTime = performance.now(); #revealStep = 1 / 6; + readonly #cells: TgpuBuffer> & StorageFlag; + readonly #cellGroup: TgpuBindGroup; + readonly #cellPipeline: TgpuComputePipeline; readonly #strokePipeline: TgpuComputePipeline; readonly #underpainting: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & SampledFlag & @@ -163,6 +169,9 @@ export class DepthPaintingRenderer { targets: { format: 'rgba8unorm' }, }); this.#strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER * 2)).$usage('storage'); + this.#cells = root.createBuffer(d.arrayOf(StrokeCell, STROKES_PER_LAYER)).$usage('storage'); + this.#cellGroup = root.createBindGroup(cellWriteLayout, { cells: this.#cells }); + this.#cellPipeline = root.createComputePipeline({ compute: prepareStrokeCells }); this.#strokeWriteGroup = root.createBindGroup(strokeWriteLayout, { strokes: this.#strokes }); this.#paintHistory = Array.from({ length: 2 }, () => root @@ -199,6 +208,7 @@ export class DepthPaintingRenderer { this.#strokeReadGroups = this.#paintHistory.map((texture) => root.createBindGroup(strokeReadLayout, { strokes: this.#strokes, + cells: this.#cells, history: texture.createView(), grain: this.#brushGrain.createView(), grainSampler, @@ -256,6 +266,7 @@ export class DepthPaintingRenderer { this.#surfacePipeline.initAsync(), this.#paintPipeline.initAsync(), this.#strokePipeline.initAsync(), + this.#cellPipeline.initAsync(), this.#copyPipeline.initAsync(), this.#mipPipeline.initAsync(), this.#presentPipeline.initAsync(), @@ -401,6 +412,15 @@ export class DepthPaintingRenderer { Math.ceil(this.#canvas.height / this.#settings.spacing / 8), 2, ); + this.#cellPipeline + .with(strokePass) + .with(attachment.paintBindGroup) + .with(this.#strokeWriteGroup) + .with(this.#cellGroup) + .dispatchWorkgroups( + Math.ceil(this.#canvas.width / this.#settings.spacing / 8), + Math.ceil(this.#canvas.height / this.#settings.spacing / 8), + ); strokePass.end(); const nextHistory = 1 - this.#historyIndex; @@ -433,6 +453,7 @@ export class DepthPaintingRenderer { this.#depthParams.destroy(); this.#paintParams.destroy(); this.#strokes.destroy(); + this.#cells.destroy(); this.#underpainting.destroy(); this.#brushGrain.destroy(); for (const texture of this.#paintHistory) { diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts index 2d4b3ddb92..052572fc56 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -43,7 +43,16 @@ export const paintFrameLayout = tgpu.bindGroupLayout({ export const strokeWriteLayout = tgpu.bindGroupLayout({ strokes: { storage: d.arrayOf(Stroke), access: 'mutable' }, }); +export const StrokeCell = d.struct({ + count: d.u32, + changing: d.u32, + indices: d.arrayOf(d.u32, 450), +}); +export const cellWriteLayout = tgpu.bindGroupLayout({ + cells: { storage: d.arrayOf(StrokeCell), access: 'mutable' }, +}); export const strokeReadLayout = tgpu.bindGroupLayout({ + cells: { storage: d.arrayOf(StrokeCell), access: 'readonly' }, grain: { texture: d.texture2d() }, grainSampler: { sampler: 'filtering' }, history: { texture: d.texture2d() }, @@ -157,7 +166,12 @@ export function depthStrokeScale(depth: number): number { return 1 + 1.4 * near * near + 2.4 * (1 - remaining * remaining * remaining); } -export function strokeComesBefore(depth: number, index: number, otherDepth: number, otherIndex: number): boolean { +export function strokeComesBefore( + depth: number, + index: number, + otherDepth: number, + otherIndex: number, +): boolean { 'use gpu'; return depth > otherDepth || (depth === otherDepth && index > otherIndex); } @@ -222,13 +236,26 @@ export const prepareStrokes = tgpu.computeFn({ let change = cameraChange(memory.rgb, strokeWriteLayout.$.strokes[index].memory.rgb); for (const probe of std.range(4)) { let offset = d.vec2f(0.7, 0); - if (probe === 1) { offset = d.vec2f(-0.7, 0); } - if (probe === 2) { offset = d.vec2f(0, 0.7); } - if (probe === 3) { offset = d.vec2f(0, -0.7); } - const sample = std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, - uv.add(offset.mul(spacing).div(paintLayout.$.params.canvasSize)), 4); + if (probe === 1) { + offset = d.vec2f(-0.7, 0); + } + if (probe === 2) { + offset = d.vec2f(0, 0.7); + } + if (probe === 3) { + offset = d.vec2f(0, -0.7); + } + const sample = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv.add(offset.mul(spacing).div(paintLayout.$.params.canvasSize)), + 4, + ); footprintMemory[probe] = d.vec4f(sample); - change = std.max(change, cameraChange(sample.rgb, strokeWriteLayout.$.strokes[index].footprintMemory[probe].rgb)); + change = std.max( + change, + cameraChange(sample.rgb, strokeWriteLayout.$.strokes[index].footprintMemory[probe].rgb), + ); } const surface = std.textureSampleLevel(paintLayout.$.surface, paintLayout.$.sampler, uv, 0); // A surface leaving the foreground must release its old depth occlusion too. @@ -298,10 +325,7 @@ export const prepareStrokes = tgpu.computeFn({ detail, depth, opacity, - color: d.vec4f( - std.mix(std.mix(color, mean, (1 - detail) * 0.35), softened, blur * 0.7), - 1, - ), + color: d.vec4f(std.mix(std.mix(color, mean, (1 - detail) * 0.35), softened, blur * 0.7), 1), }); }); @@ -320,6 +344,91 @@ export function incrementalAlpha(alpha: number, before: number, after: number): return std.saturate((alpha * (after - before)) / std.max(1 - alpha * before, 0.00001)); } +/** Share one conservative sorted candidate list across all pixels in a grid cell. */ +export const prepareStrokeCells = tgpu.computeFn({ + workgroupSize: [8, 8], + in: { gid: d.builtin.globalInvocationId }, +})(({ gid }) => { + 'use gpu'; + const gridSize = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + if (gid.x >= gridSize.x || gid.y >= gridSize.y) { + return; + } + const cellIndex = gid.y * gridSize.x + gid.x; + const pixel = d.vec2f(gid.xy).add(0.5).mul(paintLayout.$.params.spacing); + const padding = paintLayout.$.params.spacing * Math.SQRT1_2; + // Gather both grids into one depth order. Equal depths retain fine-over-coarse + // and row-major ordering. Near-to-far compositing also shields existing near paint + // from distant marks that start their animation on a later frame. + const candidates = d.arrayOf(d.vec2f, 450)(); + let count = d.u32(0); + let changing = false; + for (const layer of std.range(2)) { + const spacing = layerSpacing(layer); + const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + const cell = d.vec2i(std.floor(pixel.div(spacing))); + // Cover 3.4x strokes, including jitter, bent edges and the small shadow. + for (const y of std.range(-7, 8)) { + for (const x of std.range(-7, 8)) { + const neighbor = cell.add(d.vec2i(x, y)); + if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { + continue; + } + const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); + const stroke = strokeWriteLayout.$.strokes[index]; + // Invalidate a conservative area, including the previous, possibly larger mark. + changing = changing || stroke.dirty !== 0; + if ( + stroke.progress <= 0 || + std.length(pixel.sub(stroke.center)) > spacing * 6.5 + padding + ) { + continue; + } + // Reject outside the rotated brush bounds before sorting. The padding + // includes the bent edge, antialiasing, grain displacement and tiny shadow. + const delta = pixel.sub(stroke.center); + const along = std.abs(std.dot(delta, stroke.axis)); + const across = std.abs(std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x))); + if ( + along > stroke.halfSize.x + 2 + padding || + across > stroke.halfSize.y * 1.15 + 2 + padding + ) { + continue; + } + candidates[count] = d.vec2f(d.f32(index), stroke.depth); + count++; + } + } + } + cellWriteLayout.$.cells[cellIndex].changing = std.select(d.u32(0), d.u32(1), changing); + if (!changing) { + return; + } + for (let position = d.u32(1); position < count; position++) { + const item = d.vec2f(candidates[position]); + const index = item.x; + const depth = item.y; + let insert = position; + while (insert > 0) { + const other = d.vec2f(candidates[insert - 1]); + const otherIndex = other.x; + const otherDepth = other.y; + if (strokeComesBefore(otherDepth, otherIndex, depth, index)) { + break; + } + candidates[insert] = d.vec2f(other); + insert--; + } + candidates[insert] = d.vec2f(item); + } + cellWriteLayout.$.cells[cellIndex].count = count; + for (let i = d.u32(0); i < count; i++) { + cellWriteLayout.$.cells[cellIndex].indices[i] = d.u32(candidates[i].x); + } +}); + export const paintFragment = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f, @@ -367,73 +476,39 @@ export const paintFragment = tgpu.fragmentFn({ 7.3, ).rgb; } - // Gather both grids into one depth order. Equal depths retain fine-over-coarse - // and row-major ordering. Near-to-far compositing also shields existing near paint - // from distant marks that start their animation on a later frame. - const candidates = d.arrayOf(d.u32, 450)(); - let count = d.u32(0); - let changing = false; - for (const layer of std.range(2)) { - const spacing = layerSpacing(layer); - const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); - const cell = d.vec2i(std.floor(pixel.div(spacing))); - // Cover 3.4x strokes, including jitter, bent edges and the small shadow. - for (const y of std.range(-7, 8)) { - for (const x of std.range(-7, 8)) { - const neighbor = cell.add(d.vec2i(x, y)); - if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { - continue; - } - const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); - const stroke = strokeReadLayout.$.strokes[index]; - // Invalidate a conservative area, including the previous, possibly larger mark. - changing = changing || stroke.dirty !== 0; - if (stroke.progress <= 0 || std.length(pixel.sub(stroke.center)) > spacing * 6.5) { - continue; - } - // Reject outside the rotated brush bounds before sorting. The padding - // includes the bent edge, antialiasing, grain displacement and tiny shadow. - const delta = pixel.sub(stroke.center); - const along = std.abs(std.dot(delta, stroke.axis)); - const across = std.abs(std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x))); - if (along > stroke.halfSize.x + 2 || across > stroke.halfSize.y * 1.15 + 2) { - continue; - } - candidates[count] = index; - count++; - } - } - } - if (!changing) { + const cellGrid = d.vec2u( + std.ceil(paintLayout.$.params.canvasSize.div(paintLayout.$.params.spacing)), + ); + const cell = std.min(d.vec2u(pixel.div(paintLayout.$.params.spacing)), cellGrid.sub(1)); + const cellIndex = cell.y * cellGrid.x + cell.x; + if (strokeReadLayout.$.cells[cellIndex].changing === 0) { return d.vec4f(painted, 1); } - for (let position = d.u32(1); position < count; position++) { - const index = candidates[position]; - const depth = strokeReadLayout.$.strokes[index].depth; - let insert = position; - while (insert > 0) { - const otherIndex = candidates[insert - 1]; - const otherDepth = strokeReadLayout.$.strokes[otherIndex].depth; - if (strokeComesBefore(otherDepth, otherIndex, depth, index)) { - break; - } - candidates[insert] = otherIndex; - insert--; - } - candidates[insert] = index; - } + const count = strokeReadLayout.$.cells[cellIndex].count; // Rebuild only invalidated pixels from remembered current strokes. Blending new // paint over flattened history retained obsolete foreground paint indefinitely. - painted = std.textureSampleLevel(underpaintLayout.$.image, underpaintLayout.$.sampler, uv, 7.3).rgb; + painted = std.textureSampleLevel( + underpaintLayout.$.image, + underpaintLayout.$.sampler, + uv, + 7.3, + ).rgb; let visibility = d.f32(1); let added = d.vec3f(0); for (let candidate = d.u32(0); candidate < count; candidate++) { - const stroke = strokeReadLayout.$.strokes[candidates[candidate]]; + const stroke = + strokeReadLayout.$.strokes[strokeReadLayout.$.cells[cellIndex].indices[candidate]]; const delta = pixel.sub(stroke.center); const local = d.vec2f( std.dot(delta, stroke.axis), std.dot(delta, d.vec2f(-stroke.axis.y, stroke.axis.x)), ); + if ( + std.abs(local.x) > stroke.halfSize.x + 2 || + std.abs(local.y) > stroke.halfSize.y * 1.15 + 2 + ) { + continue; + } const field = brushField(local, stroke.halfSize, stroke.grain); const coverage = 1 - std.smoothstep(-0.65, 0.65, field.x); // A light above the canvas casts a short shadow downward onto the already diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts index 546de08ac9..05e2b8b98a 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -3,6 +3,7 @@ import { it } from 'typegpu-testing-utility'; import { common, d } from 'typegpu'; import { prepareStrokes, + prepareStrokeCells, changeOpacity, cameraChange, depthStrokeScale, @@ -19,7 +20,7 @@ describe('monocular painting shaders', () => { const gray = d.vec3f(0.5); const brightness = cameraChange(d.vec3f(0.6), gray); // A 0.1 Cb shift with unchanged Rec.709 luma. - const chroma = cameraChange(gray.add(d.vec3f(0, -0.018556 * 0.722 / 0.7152, 0.18556)), gray); + const chroma = cameraChange(gray.add(d.vec3f(0, (-0.018556 * 0.722) / 0.7152, 0.18556)), gray); expect(cameraChange(gray, gray)).toBe(0); expect(brightness).toBeCloseTo(Math.sqrt(3) * 0.1); expect(chroma).toBeCloseTo(brightness * 0.25); @@ -81,6 +82,7 @@ describe('monocular painting shaders', () => { .createComputePipeline({ compute: prepareStrokes }) .$name('prepare strokes') .initAsync(); + await root.createComputePipeline({ compute: prepareStrokeCells }).initAsync(); await root .createRenderPipeline({ vertex: common.fullScreenTriangle, @@ -89,7 +91,7 @@ describe('monocular painting shaders', () => { }) .$name('paint strokes') .initAsync(); - expect(device.mock.createShaderModule).toHaveBeenCalledTimes(2); + expect(device.mock.createShaderModule).toHaveBeenCalledTimes(3); expect(device.mock.createShaderModule).toHaveBeenCalledWith( expect.objectContaining({ code: expect.stringContaining('texture_external') }), ); diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts index 6515130081..3fe7164fc1 100644 --- a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -4,6 +4,9 @@ import { Stroke, STROKES_PER_LAYER, prepareStrokes, + prepareStrokeCells, + StrokeCell, + cellWriteLayout, paintFragment, strokeReadLayout, paintLayout, @@ -57,12 +60,27 @@ test('detects local changes and removes a departed foreground mark without distu const writeGroup = root.createBindGroup(strokeWriteLayout, { strokes }); const mipGroup = root.createBindGroup(underpaintLayout, { image: image.createView(), sampler }); const pipeline = root.createComputePipeline({ compute: prepareStrokes }); - const history = root.createTexture({size: [256,256], format: 'rgba8unorm'}).$usage('sampled'); - const target = root.createTexture({size: [256,256], format: 'rgba8unorm'}).$usage('render'); - const grain = root.createTexture({size: [128,128], format: 'rgba8unorm'}).$usage('sampled'); - grain.write(new Uint8Array(128*128*4).fill(128)); - const paint = root.createRenderPipeline({vertex: common.fullScreenTriangle, fragment: paintFragment, targets: {format: 'rgba8unorm'}}); - const readGroup = root.createBindGroup(strokeReadLayout, {strokes, history, grain, grainSampler: sampler}); + const history = root + .createTexture({ size: [256, 256], format: 'rgba8unorm' }) + .$usage('sampled'); + const target = root.createTexture({ size: [256, 256], format: 'rgba8unorm' }).$usage('render'); + const grain = root.createTexture({ size: [128, 128], format: 'rgba8unorm' }).$usage('sampled'); + grain.write(new Uint8Array(128 * 128 * 4).fill(128)); + const paint = root.createRenderPipeline({ + vertex: common.fullScreenTriangle, + fragment: paintFragment, + targets: { format: 'rgba8unorm' }, + }); + const cells = root.createBuffer(d.arrayOf(StrokeCell, 64)).$usage('storage'); + const cellPipeline = root.createComputePipeline({ compute: prepareStrokeCells }); + const cellGroup = root.createBindGroup(cellWriteLayout, { cells }); + const readGroup = root.createBindGroup(strokeReadLayout, { + strokes, + cells, + history, + grain, + grainSampler: sampler, + }); let lastPixels = new Uint8Array(); async function frame() { image.write(canvas); @@ -75,17 +93,34 @@ test('detects local changes and removes a departed foreground mark without distu .with(mipGroup) .with(root.createBindGroup(paintFrameLayout, { frame: external })) .dispatchWorkgroups(1, 1, 2); - paint.with(group).with(mipGroup).with(readGroup) - .with(root.createBindGroup(paintFrameLayout, {frame: external})) - .withColorAttachment({view: target}).draw(3); - const buffer = root.device.createBuffer({size: 256*256*4, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ}); + cellPipeline.with(group).with(writeGroup).with(cellGroup).dispatchWorkgroups(1, 1); + paint + .with(group) + .with(mipGroup) + .with(readGroup) + .with(root.createBindGroup(paintFrameLayout, { frame: external })) + .withColorAttachment({ view: target }) + .draw(3); + const buffer = root.device.createBuffer({ + size: 256 * 256 * 4, + usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ, + }); const encoder = root.device.createCommandEncoder(); - encoder.copyTextureToTexture({texture: root.unwrap(target)}, {texture: root.unwrap(history)}, [256,256]); - encoder.copyTextureToBuffer({texture: root.unwrap(target)}, {buffer, bytesPerRow: 1024}, [256,256]); + encoder.copyTextureToTexture( + { texture: root.unwrap(target) }, + { texture: root.unwrap(history) }, + [256, 256], + ); + encoder.copyTextureToBuffer( + { texture: root.unwrap(target) }, + { buffer, bytesPerRow: 1024 }, + [256, 256], + ); root.device.queue.submit([encoder.finish()]); await buffer.mapAsync(GPUMapMode.READ); lastPixels = new Uint8Array(buffer.getMappedRange()).slice(); - buffer.unmap(); buffer.destroy(); + buffer.unmap(); + buffer.destroy(); const all = await strokes.read(); source.close(); return [...all.slice(0, 16), ...all.slice(STROKES_PER_LAYER, STROKES_PER_LAYER + 64)]; @@ -123,7 +158,9 @@ test('detects local changes and removes a departed foreground mark without distu const end = await frame(); expect(start.every((s) => s.progress >= 0.1 && s.progress <= 0.2)).toBe(true); expect(new Set(start.map((s) => s.progress)).size).toBeGreaterThan(1); - expect(middle.every((s, i) => Math.abs(s.progress - start[i].progress - 0.25) < 0.00001)).toBe(true); + expect(middle.every((s, i) => Math.abs(s.progress - start[i].progress - 0.25) < 0.00001)).toBe( + true, + ); expect(end.every((s) => s.progress >= 0.85 && s.progress <= 0.95)).toBe(true); const complete = await frame(); const idle = await frame(); From 0bf32733d06f860a0076cada6bf9edf2fb550ff2 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 15 Sep 2026 13:12:10 +0200 Subject: [PATCH 7/8] Even faster --- .../monocular-light-injection/shaders.ts | 23 ++++++---- .../monocular-painting/renderer.ts | 40 ++++++++++++++--- .../monocular-painting/shaders.ts | 44 ++++++++++++------- .../monocular-painting.test.ts | 15 +------ apps/typegpu-docs/vitest.config.mts | 2 + 5 files changed, 80 insertions(+), 44 deletions(-) diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts index 5fa1f1fb7a..80bbd7dfeb 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts @@ -198,6 +198,9 @@ function surfaceSlope(gradient: d.v2f): d.v2f { return gradient * (ceiling / steepness); } +/** Compile out occlusion taps for consumers that only need slopes and depth. */ +export const surfaceOcclusionSlot = tgpu.slot(true); + /** Derives the surface slope and a height-field occlusion term from the depth field */ export const surfaceKernel = tgpu.computeFn({ in: { gid: d.builtin.globalInvocationId }, @@ -221,15 +224,17 @@ export const surfaceKernel = tgpu.computeFn({ ); let occlusion = d.f32(0); - for (const radius of tgpu.unroll(OCCLUSION_RADII)) { - for (const stepY of tgpu.unroll(RING_OFFSETS)) { - for (const stepX of tgpu.unroll(RING_OFFSETS)) { - if (stepX !== 0 || stepY !== 0) { - const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); - const difference = neighbor - center; - const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); - const cleared = std.max(difference - OCCLUSION_FLOOR, 0); - occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; + if (surfaceOcclusionSlot.$) { + for (const radius of tgpu.unroll(OCCLUSION_RADII)) { + for (const stepY of tgpu.unroll(RING_OFFSETS)) { + for (const stepX of tgpu.unroll(RING_OFFSETS)) { + if (stepX !== 0 || stepY !== 0) { + const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); + const difference = neighbor - center; + const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); + const cleared = std.max(difference - OCCLUSION_FLOOR, 0); + occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; + } } } } diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index 194eedfbf4..db1434d09b 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -8,6 +8,7 @@ import { rangeStabilityLayout, stabilizeRangeKernel, surfaceKernel, + surfaceOcclusionSlot, surfaceLayout, } from '../monocular-light-injection/shaders.ts'; import { common, d, std } from 'typegpu'; @@ -99,6 +100,8 @@ export class DepthPaintingRenderer { readonly #depthParams: TgpuBuffer & UniformFlag; readonly #paintParams: TgpuBuffer & UniformFlag; readonly #sampler: TgpuSampler; + readonly #grainSampler: TgpuSampler; + #cellCount = 1; readonly #brushGrain: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & SampledFlag & StorageFlag; @@ -109,7 +112,7 @@ export class DepthPaintingRenderer { readonly #paintPipeline: TgpuRenderPipeline; readonly #strokes: TgpuBuffer> & StorageFlag; readonly #strokeWriteGroup: TgpuBindGroup; - readonly #strokeReadGroups: TgpuBindGroup[]; + #strokeReadGroups: TgpuBindGroup[]; readonly #paintHistory: (TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & SampledFlag & RenderFlag)[]; @@ -120,8 +123,8 @@ export class DepthPaintingRenderer { #paintDirty = true; #lastPaintTime = performance.now(); #revealStep = 1 / 6; - readonly #cells: TgpuBuffer> & StorageFlag; - readonly #cellGroup: TgpuBindGroup; + #cells: TgpuBuffer> & StorageFlag; + #cellGroup: TgpuBindGroup; readonly #cellPipeline: TgpuComputePipeline; readonly #strokePipeline: TgpuComputePipeline; readonly #underpainting: TgpuTexture<{ size: [number, number]; format: 'rgba8unorm' }> & @@ -162,14 +165,16 @@ export class DepthPaintingRenderer { }); this.#stabilizePipeline = root.createComputePipeline({ compute: stabilizeRangeKernel }); this.#depthPipeline = root.createComputePipeline({ compute: depthPrepareKernel }); - this.#surfacePipeline = root.createComputePipeline({ compute: surfaceKernel }); + this.#surfacePipeline = root + .with(surfaceOcclusionSlot, false) + .createComputePipeline({ compute: surfaceKernel }); this.#paintPipeline = root.createRenderPipeline({ vertex: common.fullScreenTriangle, fragment: paintFragment, targets: { format: 'rgba8unorm' }, }); this.#strokes = root.createBuffer(d.arrayOf(Stroke, STROKES_PER_LAYER * 2)).$usage('storage'); - this.#cells = root.createBuffer(d.arrayOf(StrokeCell, STROKES_PER_LAYER)).$usage('storage'); + this.#cells = root.createBuffer(d.arrayOf(StrokeCell, this.#cellCount)).$usage('storage'); this.#cellGroup = root.createBindGroup(cellWriteLayout, { cells: this.#cells }); this.#cellPipeline = root.createComputePipeline({ compute: prepareStrokeCells }); this.#strokeWriteGroup = root.createBindGroup(strokeWriteLayout, { strokes: this.#strokes }); @@ -205,6 +210,7 @@ export class DepthPaintingRenderer { addressModeU: 'repeat', addressModeV: 'repeat', }); + this.#grainSampler = grainSampler; this.#strokeReadGroups = this.#paintHistory.map((texture) => root.createBindGroup(strokeReadLayout, { strokes: this.#strokes, @@ -356,6 +362,7 @@ export class DepthPaintingRenderer { this.#revealStep = Math.min(Math.max(now - this.#lastPaintTime, 1), 50) / 100; this.#lastPaintTime = now; this.#syncCanvasSize(); + this.#syncStrokeCells(); this.#uvTransform = frame.uvTransform; this.#swapAxes = frame.swapAxes; this.#writePaintParams(); @@ -476,6 +483,29 @@ export class DepthPaintingRenderer { } } + #syncStrokeCells(): void { + const count = + Math.ceil(this.#canvas.width / this.#settings.spacing) * + Math.ceil(this.#canvas.height / this.#settings.spacing); + if (count === this.#cellCount) { + return; + } + this.#cells.destroy(); + this.#cellCount = count; + this.#cells = this.#root.createBuffer(d.arrayOf(StrokeCell, count)).$usage('storage'); + this.#cellGroup = this.#root.createBindGroup(cellWriteLayout, { cells: this.#cells }); + this.#strokeReadGroups = this.#paintHistory.map((texture) => + this.#root.createBindGroup(strokeReadLayout, { + strokes: this.#strokes, + cells: this.#cells, + history: texture.createView(), + grain: this.#brushGrain.createView(), + grainSampler: this.#grainSampler, + }), + ); + this.#paintDirty = true; + } + #writePaintParams(): void { this.#paintParams.write({ uvTransform: this.#uvTransform, diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts index 052572fc56..2b8f79ef56 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/shaders.ts @@ -16,7 +16,6 @@ export const Stroke = d.struct({ footprintMemory: d.arrayOf(d.vec4f, 4), dirty: d.u32, progress: d.f32, - previousProgress: d.f32, }); export const PaintParams = d.struct({ uvTransform: d.mat2x2f, @@ -224,7 +223,6 @@ export const prepareStrokes = tgpu.computeFn({ // repeatedly restart the fade or change its color/orientation halfway through. if (paintLayout.$.params.resetPaint === 0 && strokeWriteLayout.$.strokes[index].progress < 1) { const previous = strokeWriteLayout.$.strokes[index].progress; - strokeWriteLayout.$.strokes[index].previousProgress = previous; strokeWriteLayout.$.strokes[index].progress = std.min( previous + paintLayout.$.params.revealStep, d.f32(1), @@ -317,7 +315,6 @@ export const prepareStrokes = tgpu.computeFn({ memory, footprintMemory, dirty: 1, - previousProgress: 0, // Independent 5–15 ms delay from the moment this mark is needed. progress: std.min(paintLayout.$.params.revealStep - (0.05 + randf.sample() * 0.1), d.f32(1)), halfSize: size, @@ -338,12 +335,6 @@ export function revealAt(along: number, progress: number): number { return std.smoothstep(position - 0.12, position + 0.12, eased * 1.24 - 0.12); } -/** Add just the missing coverage to a persistent surface, independent of frame count. */ -export function incrementalAlpha(alpha: number, before: number, after: number): number { - 'use gpu'; - return std.saturate((alpha * (after - before)) / std.max(1 - alpha * before, 0.00001)); -} - /** Share one conservative sorted candidate list across all pixels in a grid cell. */ export const prepareStrokeCells = tgpu.computeFn({ workgroupSize: [8, 8], @@ -358,13 +349,40 @@ export const prepareStrokeCells = tgpu.computeFn({ } const cellIndex = gid.y * gridSize.x + gid.x; const pixel = d.vec2f(gid.xy).add(0.5).mul(paintLayout.$.params.spacing); + // Check only dirty flags before loading brush bounds or building a candidate list. + // Keep the full old neighborhood: a changed stroke may have shrunk or moved. + let changing = false; + for (const layer of std.range(2)) { + const spacing = layerSpacing(layer); + const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); + const cell = d.vec2i(std.floor(pixel.div(spacing))); + for (let y = d.i32(-7); y < 8 && !changing; y++) { + for (let x = d.i32(-7); x < 8; x++) { + const neighbor = cell.add(d.vec2i(x, y)); + if (neighbor.x < 0 || neighbor.y < 0 || neighbor.x >= grid.x || neighbor.y >= grid.y) { + continue; + } + const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); + if (strokeWriteLayout.$.strokes[index].dirty !== 0) { + changing = true; + break; + } + } + } + if (changing) { + break; + } + } + cellWriteLayout.$.cells[cellIndex].changing = std.select(d.u32(0), d.u32(1), changing); + if (!changing) { + return; + } const padding = paintLayout.$.params.spacing * Math.SQRT1_2; // Gather both grids into one depth order. Equal depths retain fine-over-coarse // and row-major ordering. Near-to-far compositing also shields existing near paint // from distant marks that start their animation on a later frame. const candidates = d.arrayOf(d.vec2f, 450)(); let count = d.u32(0); - let changing = false; for (const layer of std.range(2)) { const spacing = layerSpacing(layer); const grid = d.vec2i(std.ceil(paintLayout.$.params.canvasSize.div(spacing))); @@ -378,8 +396,6 @@ export const prepareStrokeCells = tgpu.computeFn({ } const index = d.u32(layer) * STROKES_PER_LAYER + d.u32(neighbor.y * grid.x + neighbor.x); const stroke = strokeWriteLayout.$.strokes[index]; - // Invalidate a conservative area, including the previous, possibly larger mark. - changing = changing || stroke.dirty !== 0; if ( stroke.progress <= 0 || std.length(pixel.sub(stroke.center)) > spacing * 6.5 + padding @@ -402,10 +418,6 @@ export const prepareStrokeCells = tgpu.computeFn({ } } } - cellWriteLayout.$.cells[cellIndex].changing = std.select(d.u32(0), d.u32(1), changing); - if (!changing) { - return; - } for (let position = d.u32(1); position < count; position++) { const item = d.vec2f(candidates[position]); const index = item.x; diff --git a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts index 05e2b8b98a..440a8f1c23 100644 --- a/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts +++ b/apps/typegpu-docs/tests/individual-example-tests/monocular-painting.test.ts @@ -9,7 +9,6 @@ import { depthStrokeScale, strokeComesBefore, revealAt, - incrementalAlpha, copyFrameFragment, downsampleFragment, paintFragment, @@ -44,25 +43,13 @@ describe('monocular painting shaders', () => { expect(depthStrokeScale(0)).toBe(3.4); expect(depthStrokeScale(0.25)).toBeGreaterThan(depthStrokeScale(0.75)); }); - it('reveals in stroke direction without multiplying opacity across frames', () => { + it('reveals in stroke direction with exact empty and full endpoints', () => { expect(revealAt(-0.5, 0.2)).toBeGreaterThan(revealAt(0.5, 0.2)); expect(revealAt(0.5, 0.5)).toBe(1); for (const position of [-1, -0.5, 0, 0.5, 1]) { expect(revealAt(position, -0.1)).toBe(0); expect(revealAt(position, 0)).toBe(0); expect(revealAt(position, 1)).toBe(1); - for (const steps of [8, 24, 60]) { - let color = 1; - for (let step = 1; step <= steps; step++) { - const alpha = incrementalAlpha( - 0.45, - revealAt(position, (step - 1) / steps), - revealAt(position, step / steps), - ); - color = color * (1 - alpha) + 0.3 * alpha; - } - expect(color).toBeCloseTo(1 * 0.55 + 0.3 * 0.45, 5); - } } }); it('resolves frame-copy and bilinear mip pipelines', async ({ root, device }) => { diff --git a/apps/typegpu-docs/vitest.config.mts b/apps/typegpu-docs/vitest.config.mts index 631620a9d3..a9ce78eec4 100644 --- a/apps/typegpu-docs/vitest.config.mts +++ b/apps/typegpu-docs/vitest.config.mts @@ -26,6 +26,7 @@ export default defineConfig({ test: { projects: [ { + extends: true, test: { name: 'browser', include: ['**/*.{test,spec}.browser.ts'], @@ -36,6 +37,7 @@ export default defineConfig({ }, }, { + extends: true, test: { name: 'individual-example-tests', root: './tests/individual-example-tests', From 6e01a3eb9556294a52223852dd88ed8b6e1c8354 Mon Sep 17 00:00:00 2001 From: Iwo Plaza Date: Tue, 15 Sep 2026 13:27:51 +0200 Subject: [PATCH 8/8] docs: Share DepthART helpers without cross-example imports --- .../src/components/CodeEditor.tsx | 6 +- .../components/stackblitz/openInStackBlitz.ts | 3 +- .../depthart}/camera-session.ts | 0 .../depthart}/chooser.ts | 0 .../depthart}/model-store.ts | 0 .../src/examples/common/depthart/surface.ts | 167 ++++++++++++++++++ .../src/examples/exampleContent.ts | 35 +--- .../monocular-light-injection/index.ts | 6 +- .../monocular-light-injection/renderer.ts | 22 +-- .../monocular-light-injection/shaders.ts | 165 ----------------- .../monocular-painting/index.ts | 14 +- .../monocular-painting/renderer.ts | 8 +- .../tests/monocular-painting.test.browser.ts | 2 +- 13 files changed, 196 insertions(+), 232 deletions(-) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection => common/depthart}/camera-session.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection => common/depthart}/chooser.ts (100%) rename apps/typegpu-docs/src/examples/{image-processing/monocular-light-injection => common/depthart}/model-store.ts (100%) create mode 100644 apps/typegpu-docs/src/examples/common/depthart/surface.ts diff --git a/apps/typegpu-docs/src/components/CodeEditor.tsx b/apps/typegpu-docs/src/components/CodeEditor.tsx index 7708846d3d..a7416f69c2 100644 --- a/apps/typegpu-docs/src/components/CodeEditor.tsx +++ b/apps/typegpu-docs/src/components/CodeEditor.tsx @@ -1,5 +1,4 @@ import { useAtomValue } from 'jotai'; -import { normalize } from 'pathe'; import { useEffect, useState } from 'react'; import Editor, { type Monaco, type OnMount } from '@monaco-editor/react'; import catppuccinMocha from '@shikijs/themes/catppuccin-mocha'; @@ -139,9 +138,8 @@ export default function CodeEditor(props: Props) { }, []); // Monaco needs relative paths to work correctly and '../../common/file.ts' will not do - const path = normalize( - 'common' in file ? `common/${file.path}` : `${file.exampleKey.replace('--', '/')}/${file.path}`, - ); + const path = + 'common' in file ? `common/${file.path}` : `${file.exampleKey.replace('--', '/')}/${file.path}`; return (
diff --git a/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts b/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts index 0f6cef82f8..3d4f68ab12 100644 --- a/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts +++ b/apps/typegpu-docs/src/components/stackblitz/openInStackBlitz.ts @@ -1,5 +1,4 @@ import StackBlitzSDK from '@stackblitz/sdk'; -import { normalize } from 'pathe'; import typegpuColorPackageJson from '@typegpu/color/package.json' with { type: 'json' }; import typegpuGlPackageJson from '@typegpu/gl/package.json' with { type: 'json' }; import typegpuNoisePackageJson from '@typegpu/noise/package.json' with { type: 'json' }; @@ -22,7 +21,7 @@ export async function openInStackBlitz( const tsFiles: Record = {}; for (const file of exampleSource.tsFiles) { - tsFiles[normalize(`src/${file.path}`)] = file.tsnotoverContent ?? file.content; + tsFiles[`src/${file.path}`] = file.tsnotoverContent ?? file.content; } for (const file of common) { tsFiles[`src/common/${file.path}`] = file.tsnotoverContent ?? file.content; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/camera-session.ts b/apps/typegpu-docs/src/examples/common/depthart/camera-session.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/camera-session.ts rename to apps/typegpu-docs/src/examples/common/depthart/camera-session.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/chooser.ts b/apps/typegpu-docs/src/examples/common/depthart/chooser.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/chooser.ts rename to apps/typegpu-docs/src/examples/common/depthart/chooser.ts diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/model-store.ts b/apps/typegpu-docs/src/examples/common/depthart/model-store.ts similarity index 100% rename from apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/model-store.ts rename to apps/typegpu-docs/src/examples/common/depthart/model-store.ts diff --git a/apps/typegpu-docs/src/examples/common/depthart/surface.ts b/apps/typegpu-docs/src/examples/common/depthart/surface.ts new file mode 100644 index 0000000000..da1f28949d --- /dev/null +++ b/apps/typegpu-docs/src/examples/common/depthart/surface.ts @@ -0,0 +1,167 @@ +import { d, std, tgpu } from 'typegpu'; + +export const DEPTH_WORKGROUP_SIZE = 64; +export const SURFACE_WORKGROUP_SIZE = 8; +const RING_OFFSETS = [-1, 0, 1] as const; + +const RANGE_BLEND = 0.12; +const TEMPORAL_ALPHA = 0.32; +const MOTION_ALPHA = 0.8; +const MOTION_LOW = 0.02; +const MOTION_HIGH = 0.09; + +const GRADIENT_RADIUS = 7; +const GRADIENT_BACK = -GRADIENT_RADIUS; +const GRADIENT_LIMIT = 0.009; +const GRADIENT_NOISE = 0.0003; +const GRADIENT_NOISE_ENERGY = GRADIENT_NOISE ** 2; +const OCCLUSION_RADII = [3, 9] as const; +const OCCLUSION_TAPS = OCCLUSION_RADII.length * (RING_OFFSETS.length ** 2 - 1); +const OCCLUSION_SCALE = 0.07; +const OCCLUSION_RANGE = 0.25; +const OCCLUSION_FLOOR = 0.012; + +export const DepthParams = d.struct({ + outputSize: d.vec2u, + reset: d.u32, +}); + +export const rangeStabilityLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + frameRange: { storage: d.vec2f, access: 'readonly' }, + stableRange: { storage: d.vec2f, access: 'mutable' }, +}); + +export const depthPrepareLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + disparity: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, + stableRange: { storage: d.vec2f, access: 'readonly' }, + history: { storage: d.arrayOf(d.f32), access: 'mutable' }, +}); + +export const surfaceLayout = tgpu.bindGroupLayout({ + params: { uniform: DepthParams }, + depth: { storage: d.arrayOf(d.f32), access: 'readonly' }, + surface: { storageTexture: d.textureStorage2d('rgba16float', 'write-only') }, +}); + +export const stabilizeRangeKernel = tgpu.computeFn({ workgroupSize: [1] })(() => { + 'use gpu'; + const low = rangeStabilityLayout.$.frameRange.x; + const high = std.max(rangeStabilityLayout.$.frameRange.y, low + 0.001); + if (rangeStabilityLayout.$.params.reset !== 0) { + rangeStabilityLayout.$.stableRange = d.vec2f(low, high); + return; + } + + const previousLow = rangeStabilityLayout.$.stableRange.x; + const previousHigh = rangeStabilityLayout.$.stableRange.y; + rangeStabilityLayout.$.stableRange = d.vec2f( + std.mix(previousLow, low, RANGE_BLEND), + std.mix(previousHigh, high, RANGE_BLEND), + ); +}); + +export const depthPrepareKernel = tgpu.computeFn({ + in: { gid: d.builtin.globalInvocationId }, + workgroupSize: [DEPTH_WORKGROUP_SIZE], +})(({ gid }) => { + 'use gpu'; + const width = depthPrepareLayout.$.params.outputSize.x; + const index = gid.x; + if (index >= width * depthPrepareLayout.$.params.outputSize.y) { + return; + } + + const low = depthPrepareLayout.$.stableRange.x; + const span = std.max(depthPrepareLayout.$.stableRange.y - low, 0.001); + const disparity = depthPrepareLayout.$.disparity[index].x; + let normalized = d.f32(0); + if (disparity === disparity) { + normalized = std.saturate((disparity - low) / span); + } + + let filtered = d.f32(normalized); + if (depthPrepareLayout.$.params.reset === 0) { + const previous = depthPrepareLayout.$.history[index]; + const motion = std.smoothstep(MOTION_LOW, MOTION_HIGH, std.abs(normalized - previous)); + filtered = std.mix(previous, normalized, std.mix(TEMPORAL_ALPHA, MOTION_ALPHA, motion)); + } + + depthPrepareLayout.$.history[index] = filtered; +}); + +function texelIndex(coord: d.v2i, size: d.v2i): number { + 'use gpu'; + const clamped = std.clamp(coord, d.vec2i(0), size - 1); + return d.u32(clamped.y) * d.u32(size.x) + d.u32(clamped.x); +} + +function depthTexelAt(coord: d.v2i, size: d.v2i): number { + 'use gpu'; + return surfaceLayout.$.depth[texelIndex(coord, size)]; +} + +function gentlerDelta(backward: number, forward: number): number { + 'use gpu'; + const back = std.abs(backward); + const front = std.abs(forward); + return (backward * front + forward * back) / std.max(back + front, 0.000000001); +} + +function surfaceSlope(gradient: d.v2f): d.v2f { + 'use gpu'; + const steepness = std.max(std.length(gradient), 0.000000001); + const shrunk = std.sqrt(std.max(steepness * steepness - GRADIENT_NOISE_ENERGY, 0)); + const ceiling = GRADIENT_LIMIT * std.tanh(shrunk / GRADIENT_LIMIT); + return gradient * (ceiling / steepness); +} + +/** Compile out occlusion taps for consumers that only need slopes and depth. */ +export const surfaceOcclusionSlot = tgpu.slot(true); + +/** Derives the surface slope and a height-field occlusion term from the depth field */ +export const surfaceKernel = tgpu.computeFn({ + in: { gid: d.builtin.globalInvocationId }, + workgroupSize: [SURFACE_WORKGROUP_SIZE, SURFACE_WORKGROUP_SIZE], +})(({ gid }) => { + 'use gpu'; + const size = d.vec2i(surfaceLayout.$.params.outputSize); + const coord = d.vec2i(gid.xy); + if (coord.x >= size.x || coord.y >= size.y) { + return; + } + + const center = depthTexelAt(coord, size); + const left = depthTexelAt(coord + d.vec2i(GRADIENT_BACK, 0), size); + const right = depthTexelAt(coord + d.vec2i(GRADIENT_RADIUS, 0), size); + const up = depthTexelAt(coord + d.vec2i(0, GRADIENT_BACK), size); + const down = depthTexelAt(coord + d.vec2i(0, GRADIENT_RADIUS), size); + const gradient = surfaceSlope( + d.vec2f(gentlerDelta(center - left, right - center), gentlerDelta(center - up, down - center)) / + d.f32(GRADIENT_RADIUS), + ); + + let occlusion = d.f32(0); + if (surfaceOcclusionSlot.$) { + for (const radius of tgpu.unroll(OCCLUSION_RADII)) { + for (const stepY of tgpu.unroll(RING_OFFSETS)) { + for (const stepX of tgpu.unroll(RING_OFFSETS)) { + if (stepX !== 0 || stepY !== 0) { + const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); + const difference = neighbor - center; + const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); + const cleared = std.max(difference - OCCLUSION_FLOOR, 0); + occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; + } + } + } + } + } + + std.textureStore( + surfaceLayout.$.surface, + d.vec2u(gid.xy), + d.vec4f(gradient, 1 - std.saturate(occlusion / d.f32(OCCLUSION_TAPS)), center), + ); +}); diff --git a/apps/typegpu-docs/src/examples/exampleContent.ts b/apps/typegpu-docs/src/examples/exampleContent.ts index 4d79cd065a..79d0e4010a 100644 --- a/apps/typegpu-docs/src/examples/exampleContent.ts +++ b/apps/typegpu-docs/src/examples/exampleContent.ts @@ -128,39 +128,6 @@ async function resolveExampleSrcFile(pending: PendingExampleSrcFile): Promise { - const exampleDir = key.replace('--', '/'); - const seen = new Set(files.map((file) => pathe.join(exampleDir, file.path))); - const result = [...files]; - for (const file of result) { - const directory = pathe.dirname(pathe.join(exampleDir, file.path)); - for (const match of file.content.matchAll( - /(?:from\s*|import\s*\(?\s*)['"](\.[^'"]+\.tsx?)['"]/g, - )) { - const path = pathe.normalize(pathe.join(directory, match[1])); - const getContent = tsContentLazy[`./${path}`]; - if (seen.has(path) || !getContent || path.startsWith('common/')) { - continue; - } - seen.add(path); - result.push( - await resolveExampleSrcFile({ - exampleKey: key, - path: pathe.relative(exampleDir, path), - getContent, - getTsnotoverContent: - tsnotoverContentLazy[replaceExt(`./${path}`, `.tsnotover${pathe.extname(path)}`)], - }), - ); - } - } - return result; -} - export const examples = R.pipe( metaFiles, R.mapValues((value, key) => { @@ -178,7 +145,7 @@ export const examples = R.pipe( const tsFilePromises = exampleTsFiles[key].map(resolveExampleSrcFile); const htmlFilePromise = resolveExampleSrcFile(htmlFiles[key]?.[0]); - const tsFiles = await resolveSharedSources(key, await Promise.all(tsFilePromises)); + const tsFiles = await Promise.all(tsFilePromises); const htmlFile = await htmlFilePromise; return { diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts index b2f93c6189..546350a08a 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/index.ts @@ -1,12 +1,12 @@ import { d, tgpu } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; import { defineControls } from '../../common/defineControls.ts'; -import { DepthCameraSession } from './camera-session.ts'; -import { SourceChoice, SourceChooser } from './chooser.ts'; +import { DepthCameraSession } from '../../common/depthart/camera-session.ts'; +import { SourceChoice, SourceChooser } from '../../common/depthart/chooser.ts'; import { parseDepthBundle } from '../../common/depthart-inference/bundle.ts'; import { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { setupLightInput } from './light-input.ts'; -import { fetchModel, modelLabel, type ModelSize } from './model-store.ts'; +import { fetchModel, modelLabel, type ModelSize } from '../../common/depthart/model-store.ts'; import { DepthRelightingRenderer, defaultRelightingSettings } from './renderer.ts'; import { RelightMode } from './shaders.ts'; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts index 30856f502b..b78ad66cc3 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/renderer.ts @@ -1,3 +1,14 @@ +import { + DEPTH_WORKGROUP_SIZE, + DepthParams, + SURFACE_WORKGROUP_SIZE, + depthPrepareKernel, + depthPrepareLayout, + rangeStabilityLayout, + stabilizeRangeKernel, + surfaceKernel, + surfaceLayout, +} from '../../common/depthart/surface.ts'; import { common, d } from 'typegpu'; import type { SampledFlag, @@ -11,25 +22,16 @@ import type { TgpuTexture, UniformFlag, } from 'typegpu'; -import type { DepthCameraFrame } from './camera-session.ts'; +import type { DepthCameraFrame } from '../../common/depthart/camera-session.ts'; import { DepthDisparityRangeEstimator } from '../../common/depthart-inference/disparity-range.ts'; import type { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { - DEPTH_WORKGROUP_SIZE, - DepthParams, RelightMode, RelightParams, SURFACE_FAR_Z, - SURFACE_WORKGROUP_SIZE, - depthPrepareKernel, - depthPrepareLayout, - rangeStabilityLayout, relightFragment, relightFrameLayout, relightLayout, - stabilizeRangeKernel, - surfaceKernel, - surfaceLayout, } from './shaders.ts'; const MAX_CANVAS_SIDE = 1024; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts index 80bbd7dfeb..abc0438b4f 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-light-injection/shaders.ts @@ -1,8 +1,5 @@ import { d, std, tgpu } from 'typegpu'; -export const DEPTH_WORKGROUP_SIZE = 64; -export const SURFACE_WORKGROUP_SIZE = 8; - const RING_OFFSETS = [-1, 0, 1] as const; export const RelightMode = { @@ -12,23 +9,6 @@ export const RelightMode = { NORMALS: 3, } as const; -const RANGE_BLEND = 0.12; -const TEMPORAL_ALPHA = 0.32; -const MOTION_ALPHA = 0.8; -const MOTION_LOW = 0.02; -const MOTION_HIGH = 0.09; - -const GRADIENT_RADIUS = 7; -const GRADIENT_BACK = -GRADIENT_RADIUS; -const GRADIENT_LIMIT = 0.009; -const GRADIENT_NOISE = 0.0003; -const GRADIENT_NOISE_ENERGY = GRADIENT_NOISE ** 2; -const OCCLUSION_RADII = [3, 9] as const; -const OCCLUSION_TAPS = OCCLUSION_RADII.length * (RING_OFFSETS.length ** 2 - 1); -const OCCLUSION_SCALE = 0.07; -const OCCLUSION_RANGE = 0.25; -const OCCLUSION_FLOOR = 0.012; - const NEAR_Z = 0; /** Depth of the furthest surface the relit scene can hold */ export const SURFACE_FAR_Z = -0.7; @@ -76,11 +56,6 @@ const SHADOW_GAIN = 2.5; /** How far above the light plane an occluder may rise before it stops casting */ const SHADOW_FRONT_FADE = 0.2; -export const DepthParams = d.struct({ - outputSize: d.vec2u, - reset: d.u32, -}); - export const RelightParams = d.struct({ uvTransform: d.mat2x2f, lightColor: d.vec4f, @@ -97,25 +72,6 @@ export const RelightParams = d.struct({ mode: d.u32, }); -export const rangeStabilityLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - frameRange: { storage: d.vec2f, access: 'readonly' }, - stableRange: { storage: d.vec2f, access: 'mutable' }, -}); - -export const depthPrepareLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - disparity: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, - stableRange: { storage: d.vec2f, access: 'readonly' }, - history: { storage: d.arrayOf(d.f32), access: 'mutable' }, -}); - -export const surfaceLayout = tgpu.bindGroupLayout({ - params: { uniform: DepthParams }, - depth: { storage: d.arrayOf(d.f32), access: 'readonly' }, - surface: { storageTexture: d.textureStorage2d('rgba16float', 'write-only') }, -}); - export const relightLayout = tgpu.bindGroupLayout({ params: { uniform: RelightParams }, surface: { texture: d.texture2d() }, @@ -126,127 +82,6 @@ export const relightFrameLayout = tgpu.bindGroupLayout({ frame: { externalTexture: d.textureExternal() }, }); -export const stabilizeRangeKernel = tgpu.computeFn({ workgroupSize: [1] })(() => { - 'use gpu'; - const low = rangeStabilityLayout.$.frameRange.x; - const high = std.max(rangeStabilityLayout.$.frameRange.y, low + 0.001); - if (rangeStabilityLayout.$.params.reset !== 0) { - rangeStabilityLayout.$.stableRange = d.vec2f(low, high); - return; - } - - const previousLow = rangeStabilityLayout.$.stableRange.x; - const previousHigh = rangeStabilityLayout.$.stableRange.y; - rangeStabilityLayout.$.stableRange = d.vec2f( - std.mix(previousLow, low, RANGE_BLEND), - std.mix(previousHigh, high, RANGE_BLEND), - ); -}); - -export const depthPrepareKernel = tgpu.computeFn({ - in: { gid: d.builtin.globalInvocationId }, - workgroupSize: [DEPTH_WORKGROUP_SIZE], -})(({ gid }) => { - 'use gpu'; - const width = depthPrepareLayout.$.params.outputSize.x; - const index = gid.x; - if (index >= width * depthPrepareLayout.$.params.outputSize.y) { - return; - } - - const low = depthPrepareLayout.$.stableRange.x; - const span = std.max(depthPrepareLayout.$.stableRange.y - low, 0.001); - const disparity = depthPrepareLayout.$.disparity[index].x; - let normalized = d.f32(0); - if (disparity === disparity) { - normalized = std.saturate((disparity - low) / span); - } - - let filtered = d.f32(normalized); - if (depthPrepareLayout.$.params.reset === 0) { - const previous = depthPrepareLayout.$.history[index]; - const motion = std.smoothstep(MOTION_LOW, MOTION_HIGH, std.abs(normalized - previous)); - filtered = std.mix(previous, normalized, std.mix(TEMPORAL_ALPHA, MOTION_ALPHA, motion)); - } - - depthPrepareLayout.$.history[index] = filtered; -}); - -function texelIndex(coord: d.v2i, size: d.v2i): number { - 'use gpu'; - const clamped = std.clamp(coord, d.vec2i(0), size - 1); - return d.u32(clamped.y) * d.u32(size.x) + d.u32(clamped.x); -} - -function depthTexelAt(coord: d.v2i, size: d.v2i): number { - 'use gpu'; - return surfaceLayout.$.depth[texelIndex(coord, size)]; -} - -function gentlerDelta(backward: number, forward: number): number { - 'use gpu'; - const back = std.abs(backward); - const front = std.abs(forward); - return (backward * front + forward * back) / std.max(back + front, 0.000000001); -} - -function surfaceSlope(gradient: d.v2f): d.v2f { - 'use gpu'; - const steepness = std.max(std.length(gradient), 0.000000001); - const shrunk = std.sqrt(std.max(steepness * steepness - GRADIENT_NOISE_ENERGY, 0)); - const ceiling = GRADIENT_LIMIT * std.tanh(shrunk / GRADIENT_LIMIT); - return gradient * (ceiling / steepness); -} - -/** Compile out occlusion taps for consumers that only need slopes and depth. */ -export const surfaceOcclusionSlot = tgpu.slot(true); - -/** Derives the surface slope and a height-field occlusion term from the depth field */ -export const surfaceKernel = tgpu.computeFn({ - in: { gid: d.builtin.globalInvocationId }, - workgroupSize: [SURFACE_WORKGROUP_SIZE, SURFACE_WORKGROUP_SIZE], -})(({ gid }) => { - 'use gpu'; - const size = d.vec2i(surfaceLayout.$.params.outputSize); - const coord = d.vec2i(gid.xy); - if (coord.x >= size.x || coord.y >= size.y) { - return; - } - - const center = depthTexelAt(coord, size); - const left = depthTexelAt(coord + d.vec2i(GRADIENT_BACK, 0), size); - const right = depthTexelAt(coord + d.vec2i(GRADIENT_RADIUS, 0), size); - const up = depthTexelAt(coord + d.vec2i(0, GRADIENT_BACK), size); - const down = depthTexelAt(coord + d.vec2i(0, GRADIENT_RADIUS), size); - const gradient = surfaceSlope( - d.vec2f(gentlerDelta(center - left, right - center), gentlerDelta(center - up, down - center)) / - d.f32(GRADIENT_RADIUS), - ); - - let occlusion = d.f32(0); - if (surfaceOcclusionSlot.$) { - for (const radius of tgpu.unroll(OCCLUSION_RADII)) { - for (const stepY of tgpu.unroll(RING_OFFSETS)) { - for (const stepX of tgpu.unroll(RING_OFFSETS)) { - if (stepX !== 0 || stepY !== 0) { - const neighbor = depthTexelAt(coord + d.vec2i(stepX * radius, stepY * radius), size); - const difference = neighbor - center; - const contact = 1 - std.saturate(std.abs(difference) / OCCLUSION_RANGE); - const cleared = std.max(difference - OCCLUSION_FLOOR, 0); - occlusion += std.saturate(cleared / OCCLUSION_SCALE) * contact; - } - } - } - } - } - - std.textureStore( - surfaceLayout.$.surface, - d.vec2u(gid.xy), - d.vec4f(gradient, 1 - std.saturate(occlusion / d.f32(OCCLUSION_TAPS)), center), - ); -}); - function surfaceZ(depth: number): number { 'use gpu'; return std.mix(d.f32(SURFACE_FAR_Z), d.f32(NEAR_Z), depth); diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts index 04223786df..4037238fa5 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/index.ts @@ -1,15 +1,11 @@ import { d, tgpu } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; import { defineControls } from '../../common/defineControls.ts'; -import { DepthCameraSession } from '../monocular-light-injection/camera-session.ts'; -import { SourceChoice, SourceChooser } from '../monocular-light-injection/chooser.ts'; -import { parseDepthBundle } from '../monocular-light-injection/inference/bundle.ts'; -import { DepthInferencePlan } from '../monocular-light-injection/inference/depthart.ts'; -import { - fetchModel, - modelLabel, - type ModelSize, -} from '../monocular-light-injection/model-store.ts'; +import { DepthCameraSession } from '../../common/depthart/camera-session.ts'; +import { SourceChoice, SourceChooser } from '../../common/depthart/chooser.ts'; +import { parseDepthBundle } from '../../common/depthart-inference/bundle.ts'; +import { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; +import { fetchModel, modelLabel, type ModelSize } from '../../common/depthart/model-store.ts'; import { DepthPaintingRenderer, defaultPaintingSettings } from './renderer.ts'; import { PaintMode } from './shaders.ts'; diff --git a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts index db1434d09b..7d0ea86abf 100644 --- a/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts +++ b/apps/typegpu-docs/src/examples/image-processing/monocular-painting/renderer.ts @@ -10,7 +10,7 @@ import { surfaceKernel, surfaceOcclusionSlot, surfaceLayout, -} from '../monocular-light-injection/shaders.ts'; +} from '../../common/depthart/surface.ts'; import { common, d, std } from 'typegpu'; import type { SampledFlag, @@ -25,9 +25,9 @@ import type { TgpuTexture, UniformFlag, } from 'typegpu'; -import type { DepthCameraFrame } from '../monocular-light-injection/camera-session.ts'; -import { DepthDisparityRangeEstimator } from '../monocular-light-injection/inference/disparity-range.ts'; -import type { DepthInferencePlan } from '../monocular-light-injection/inference/depthart.ts'; +import type { DepthCameraFrame } from '../../common/depthart/camera-session.ts'; +import { DepthDisparityRangeEstimator } from '../../common/depthart-inference/disparity-range.ts'; +import type { DepthInferencePlan } from '../../common/depthart-inference/depthart.ts'; import { PaintMode, underpaintLayout, diff --git a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts index 3fe7164fc1..20f6906462 100644 --- a/apps/typegpu-docs/tests/monocular-painting.test.browser.ts +++ b/apps/typegpu-docs/tests/monocular-painting.test.browser.ts @@ -169,7 +169,7 @@ test('detects local changes and removes a departed foreground mark without distu // At 200 FPS the first frame is still inside every stroke's delay. params.patch({ resetPaint: 1, revealStep: 0.05 }); const delayed = await frame(); - expect(delayed.every((s) => s.progress <= 0 && s.previousProgress === 0)).toBe(true); + expect(delayed.every((s) => s.progress <= 0)).toBe(true); expect(results.first).toBe(80); expect(results.quiet).toBe(0); expect(results.moved).toBeGreaterThan(0);