From 95ee23c2a071039289c17eead9a645557511fb54 Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Tue, 27 Oct 2026 15:09:25 +1100 Subject: [PATCH] fix: draw clip filters in the preview as the render applies them --- src/components/canvas/players/color-filter.ts | 133 ++++++++++++++++++ src/components/canvas/players/player.ts | 80 +++++++++-- tests/helpers/pixi-mock-filters.ts | 9 +- tests/player-filters.test.ts | 75 ++++++++++ 4 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 src/components/canvas/players/color-filter.ts create mode 100644 tests/player-filters.test.ts diff --git a/src/components/canvas/players/color-filter.ts b/src/components/canvas/players/color-filter.ts new file mode 100644 index 00000000..f6207bcf --- /dev/null +++ b/src/components/canvas/players/color-filter.ts @@ -0,0 +1,133 @@ +import type { ResolvedClip } from "@schemas"; +import { defaultFilterVert, Filter, GlProgram, GpuProgram, UniformGroup } from "pixi.js"; + +/** + * Recolours a clip with the render's brightness, saturation and negate maths. + * + * Per unpremultiplied pixel: contrast `f · (c − mid) + mid`, then `+ brightness`, each clamped; then saturation + * around the HSP brightness `√(.299r² + .587g² + .114b²)`; then inversion. + */ + +interface ColorParams { + /** −255…255 */ + contrast?: number; + brightness?: number; + saturation?: number; + negate?: boolean; +} + +export const COLOR_FILTERS: Partial, ColorParams>> = { + boost: { saturation: 1.75 }, + contrast: { contrast: 15 }, + darken: { brightness: -0.15 }, + greyscale: { saturation: 0 }, + lighten: { brightness: 0.15 }, + muted: { contrast: 85, brightness: -0.05, saturation: 0.6 }, + negative: { negate: true } +}; + +const MID = "0.5019608"; // 128 / 255 +const HSP = "0.299, 0.587, 0.114"; + +const glFragment = ` +in vec2 vTextureCoord; +out vec4 finalColor; + +uniform sampler2D uTexture; +uniform float uContrast; +uniform float uBrightness; +uniform float uSaturation; +uniform float uNegate; + +void main(void) +{ + vec4 color = texture(uTexture, vTextureCoord); + if (color.a == 0.0) { + finalColor = color; + return; + } + vec3 rgb = clamp(uContrast * (color.rgb / color.a - ${MID}) + ${MID}, 0.0, 1.0); + rgb = clamp(rgb + uBrightness, 0.0, 1.0); + float p = sqrt(dot(rgb * rgb, vec3(${HSP}))); + rgb = clamp(p + (rgb - p) * uSaturation, 0.0, 1.0); + rgb = mix(rgb, 1.0 - rgb, uNegate); + finalColor = vec4(rgb * color.a, color.a); +} +`; + +const gpuSource = ` +struct GlobalFilterUniforms { + uInputSize:vec4, + uInputPixel:vec4, + uInputClamp:vec4, + uOutputFrame:vec4, + uGlobalFrame:vec4, + uOutputTexture:vec4, +}; + +struct ColorUniforms { + uContrast:f32, + uBrightness:f32, + uSaturation:f32, + uNegate:f32, +}; + +@group(0) @binding(0) var gfu: GlobalFilterUniforms; +@group(0) @binding(1) var uTexture: texture_2d; +@group(0) @binding(2) var uSampler : sampler; +@group(1) @binding(0) var colorUniforms : ColorUniforms; + +struct VSOutput { + @builtin(position) position: vec4, + @location(0) uv : vec2 +}; + +fn filterVertexPosition(aPosition:vec2) -> vec4 +{ + var position = aPosition * gfu.uOutputFrame.zw + gfu.uOutputFrame.xy; + position.x = position.x * (2.0 / gfu.uOutputTexture.x) - 1.0; + position.y = position.y * (2.0*gfu.uOutputTexture.z / gfu.uOutputTexture.y) - gfu.uOutputTexture.z; + return vec4(position, 0.0, 1.0); +} + +fn filterTextureCoord( aPosition:vec2 ) -> vec2 +{ + return aPosition * (gfu.uOutputFrame.zw * gfu.uInputSize.zw); +} + +@vertex +fn mainVertex(@location(0) aPosition : vec2) -> VSOutput { + return VSOutput(filterVertexPosition(aPosition), filterTextureCoord(aPosition)); +} + +@fragment +fn mainFragment(@location(0) uv: vec2, @builtin(position) position: vec4) -> @location(0) vec4 { + let color = textureSample(uTexture, uSampler, uv); + if (color.a == 0.0) { + return color; + } + var rgb = saturate(colorUniforms.uContrast * (color.rgb / color.a - ${MID}) + ${MID}); + rgb = saturate(rgb + colorUniforms.uBrightness); + let p = sqrt(dot(rgb * rgb, vec3(${HSP}))); + rgb = saturate(p + (rgb - p) * colorUniforms.uSaturation); + rgb = select(rgb, 1.0 - rgb, colorUniforms.uNegate > 0.5); + return vec4(rgb * color.a, color.a); +} +`; + +export class ColorFilter extends Filter { + constructor({ contrast = 0, brightness = 0, saturation = 1, negate = false }: ColorParams) { + const glProgram = GlProgram.from({ vertex: defaultFilterVert, fragment: glFragment, name: "color-filter" }); + const gpuProgram = GpuProgram.from({ + vertex: { source: gpuSource, entryPoint: "mainVertex" }, + fragment: { source: gpuSource, entryPoint: "mainFragment" } + }); + const colorUniforms = new UniformGroup({ + uContrast: { value: (259 * (contrast + 255)) / (255 * (259 - contrast)), type: "f32" }, + uBrightness: { value: brightness, type: "f32" }, + uSaturation: { value: saturation, type: "f32" }, + uNegate: { value: negate ? 1 : 0, type: "f32" } + }); + super({ glProgram, gpuProgram, resources: { colorUniforms }, padding: 0 }); + } +} diff --git a/src/components/canvas/players/player.ts b/src/components/canvas/players/player.ts index d82724bd..5f474eee 100644 --- a/src/components/canvas/players/player.ts +++ b/src/components/canvas/players/player.ts @@ -23,6 +23,8 @@ import * as pixi from "pixi.js"; import { Entity } from "../../../core/shared/entity"; +import { COLOR_FILTERS, ColorFilter } from "./color-filter"; + /** * Tracks a merge field binding for a specific property path. * Used to restore placeholders on export for properties that haven't changed. @@ -52,6 +54,21 @@ export enum PlayerType { RichCaption = "rich-caption" } +// The render filters clips it draws as images and leaves those it plays as video untouched. +const FILTERED_PLAYERS = new Set([ + PlayerType.Image, + PlayerType.Text, + PlayerType.RichText, + PlayerType.Shape, + PlayerType.Caption, + PlayerType.Html, + PlayerType.Svg, + PlayerType.TextToImage +]); + +// pixi's Gaussian at this strength spreads as far as the render's radius-10 box blur (σ ≈ 6.06 px). +const CLIP_BLUR_STRENGTH = 4.69; + /** * Base class for all visual content players in the canvas. * @@ -101,6 +118,9 @@ export abstract class Player extends Entity { private wipeFilter: WipeFilter | null = null; private zoomBlurBuilder?: KeyframeBuilder; private zoomBlurFilter: pixi.BlurFilter | null = null; + private zoomBlurRadius = 0; + private clipFilterName: ResolvedClip["filter"]; + private clipFilter: pixi.Filter | null = null; private rotationOrigin?: { x: KeyframeBuilder; y: KeyframeBuilder }; private wipeMask: pixi.Graphics | null; @@ -126,6 +146,7 @@ export abstract class Player extends Entity { this.contentContainer = new pixi.Container(); this.lumaWrapper.addChild(this.contentContainer); this.getContainer().addChild(this.lumaWrapper); + this.getContainer().onRender = () => this.sizeBlurs(); } public reconfigureAfterRestore(): void { @@ -265,6 +286,8 @@ export abstract class Player extends Entity { this.applyFixedDimensions(); } + this.updateClipFilter(); + // Update wipe/reveal mask animation this.updateWipeMask(); this.updateWipeFilter(); @@ -301,6 +324,35 @@ export abstract class Player extends Entity { this.wipeMask.fill(0xffffff); } + private updateClipFilter(): void { + const { asset } = this.clipConfiguration; + const rendersAsVideo = asset.type === "rich-text" && Boolean(asset.animation?.preset); + const name = FILTERED_PLAYERS.has(this.playerType) && !rendersAsVideo ? this.clipConfiguration.filter : undefined; + + if (name !== this.clipFilterName) { + this.clipFilter?.destroy(); + this.clipFilter = this.createClipFilter(name); + this.clipFilterName = name; + this.syncContentFilters(); + } + } + + private createClipFilter(name: ResolvedClip["filter"]): pixi.Filter | null { + if (name === "blur") { + const blur = new pixi.BlurFilter(); + // The render's box blur clamps at the clip's edges, so they stay sharp. + blur.repeatEdgePixels = true; + return blur; + } + const color = name && COLOR_FILTERS[name]; + return color ? new ColorFilter(color) : null; + } + + private syncContentFilters(): void { + // The render filters a clip before its transitions. + this.contentContainer.filters = [this.clipFilter, this.wipeFilter].filter((filter): filter is pixi.Filter => filter !== null); + } + private updateWipeFilter(): void { if (!this.wipeBrightnessBuilder) { this.removeWipeFilter(); @@ -308,7 +360,7 @@ export abstract class Player extends Entity { } if (!this.wipeFilter) { this.wipeFilter = new WipeFilter(); - this.contentContainer.filters = [this.wipeFilter]; + this.syncContentFilters(); } const time = this.getPlaybackTime(); this.wipeFilter.brightness = this.wipeBrightnessBuilder.getValue(time); @@ -317,9 +369,9 @@ export abstract class Player extends Entity { private removeWipeFilter(): void { if (!this.wipeFilter) return; - this.contentContainer.filters = []; this.wipeFilter.destroy(); this.wipeFilter = null; + this.syncContentFilters(); } private updateZoomBlur(): void { @@ -334,14 +386,19 @@ export abstract class Player extends Entity { this.zoomBlurFilter = new pixi.BlurFilter(); this.getContainer().filters = [this.zoomBlurFilter]; } - // The render blurs by a whole-pixel radius before applying the clip's scale, so on screen the blur grows with scale - // and the canvas zoom. pixi's Gaussian approximates the render's three box-blur passes. - const time = this.getPlaybackTime(); - const radius = Math.trunc(this.zoomBlurBuilder.getValue(time)); - const canvas = this.getContainer().parent?.worldTransform; - const canvasZoom = canvas ? Math.hypot(canvas.a, canvas.b) : 1; - this.zoomBlurFilter.enabled = radius > 0; - this.zoomBlurFilter.strength = radius * (this.scaleKeyframeBuilder?.getValue(time) ?? 1) * canvasZoom; + this.zoomBlurRadius = Math.trunc(this.zoomBlurBuilder.getValue(this.getPlaybackTime())); + this.zoomBlurFilter.enabled = this.zoomBlurRadius > 0; + } + + // The render blurs by whole pixels before applying the clip's scale, so a blur grows with scale and the zoom it is + // drawn at. Sizing it as each frame draws follows that zoom: the editor's on screen, 1:1 in a capture. + private sizeBlurs(): void { + if (!this.zoomBlurFilter && !(this.clipFilter instanceof pixi.BlurFilter)) return; + const canvas = this.getContainer().parent?.getGlobalTransform(); + const scale = (this.scaleKeyframeBuilder?.getValue(this.getPlaybackTime()) ?? 1) * (canvas ? Math.hypot(canvas.a, canvas.b) : 1); + // pixi's Gaussian approximates the render's three box-blur passes. + if (this.zoomBlurFilter) this.zoomBlurFilter.strength = this.zoomBlurRadius * scale; + if (this.clipFilter instanceof pixi.BlurFilter) this.clipFilter.strength = CLIP_BLUR_STRENGTH * scale; } public override dispose(): void { @@ -351,6 +408,9 @@ export abstract class Player extends Entity { this.wipeFilter = null; this.zoomBlurFilter?.destroy(); this.zoomBlurFilter = null; + this.clipFilter?.destroy(); + this.clipFilter = null; + this.clipFilterName = undefined; this.contentContainer?.destroy(); this.lumaWrapper?.destroy(); diff --git a/tests/helpers/pixi-mock-filters.ts b/tests/helpers/pixi-mock-filters.ts index 5fa0fe1c..e763d6b3 100644 --- a/tests/helpers/pixi-mock-filters.ts +++ b/tests/helpers/pixi-mock-filters.ts @@ -1,10 +1,17 @@ -/** pixi.js filter/shader stubs so WipeFilter (extends pixi.Filter at module load) survives a mocked pixi. */ +/** pixi.js filter/shader stubs so the player's filters (some extend pixi.Filter at module load) survive a mocked pixi. */ /* eslint-disable max-classes-per-file, class-methods-use-this, import/prefer-default-export -- minimal pixi API stubs for jest mocks */ export const pixiFilterStubs = { Filter: class { destroy(): void {} }, + BlurFilter: class { + strength = 0; + + repeatEdgePixels = false; + + destroy(): void {} + }, GlProgram: { from: (): Record => ({}) }, GpuProgram: { from: (): Record => ({}) }, UniformGroup: class { diff --git a/tests/player-filters.test.ts b/tests/player-filters.test.ts new file mode 100644 index 00000000..149ff1f2 --- /dev/null +++ b/tests/player-filters.test.ts @@ -0,0 +1,75 @@ +import { WipeFilter } from "@animations/wipe-filter"; +import { ColorFilter } from "@canvas/players/color-filter"; +import { Player, PlayerType } from "@canvas/players/player"; +import type { Edit } from "@core/edit-session"; +import { sec } from "@core/timing/types"; +import type { Size } from "@layouts/geometry"; +import type { ResolvedClip } from "@schemas"; + +jest.mock("pixi.js", () => { + const point = () => ({ set: jest.fn() }); + return { + // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports -- jest.mock factories can't use imports + ...require("./helpers/pixi-mock-filters").pixiFilterStubs, + Container: jest.fn().mockImplementation(() => ({ addChild: jest.fn(), scale: point(), pivot: point(), position: point(), skew: point() })) + }; +}); + +class StillPlayer extends Player { + public override getSize(): Size { + return this.edit.size; + } +} + +const draw = (playerType: PlayerType, fields: Partial): Player => { + const edit = { playbackTime: 1, size: { width: 1920, height: 1080 } }; + const clip = { id: "clip-1", asset: { type: "image", src: "https://example.com/a.png" }, start: sec(0), length: sec(5), ...fields }; + const player = new StillPlayer(edit as unknown as Edit, clip as ResolvedClip, playerType); + player.reconfigureAfterRestore(); + player.applyPlayheadState(); + return player; +}; + +const filtersOf = (player: Player) => player.getContentContainer().filters ?? []; + +describe("Player clip filters", () => { + it("draws an image through its filter", () => { + expect(filtersOf(draw(PlayerType.Image, { filter: "negative" }))).toEqual([expect.any(ColorFilter)]); + }); + + it("blurs with edges clamped like the render's box blur, sized by the clip's scale as it draws", () => { + const player = draw(PlayerType.Image, { filter: "blur", scale: 0.5 }); + player.getContainer().onRender?.({} as never); + const [blur] = filtersOf(player) as unknown as { repeatEdgePixels: boolean; strength: number }[]; + + expect(blur.repeatEdgePixels).toBe(true); + expect(blur.strength).toBeCloseTo(4.69 * 0.5); + }); + + it("leaves clips the render draws as video unfiltered", () => { + expect(filtersOf(draw(PlayerType.Video, { filter: "greyscale" }))).toEqual([]); + expect(filtersOf(draw(PlayerType.Html5, { filter: "greyscale" }))).toEqual([]); + }); + + it("filters rich text only while it renders as an image", () => { + const text = { type: "rich-text", text: "Hi" } as ResolvedClip["asset"]; + const animated = { ...text, animation: { preset: "typewriter" } } as ResolvedClip["asset"]; + + expect(filtersOf(draw(PlayerType.RichText, { asset: text, filter: "darken" }))).toEqual([expect.any(ColorFilter)]); + expect(filtersOf(draw(PlayerType.RichText, { asset: animated, filter: "darken" }))).toEqual([]); + }); + + it("applies the filter before a wipe transition", () => { + const player = draw(PlayerType.Image, { filter: "lighten", transition: { in: "wipeRight" } }); + + expect(filtersOf(player)).toEqual([expect.any(ColorFilter), expect.any(WipeFilter)]); + }); + + it("drops the filter when the clip clears it", () => { + const player = draw(PlayerType.Image, { filter: "muted" }); + player.clipConfiguration.filter = "none"; + player.applyPlayheadState(); + + expect(filtersOf(player)).toEqual([]); + }); +});