Liquid Media

npx atelier-ui add liquid-media

npm install three @react-three/fiber @react-three/drei

import { useFBO, useTexture } from "@react-three/drei"
import { createPortal, useFrame, useThree } from "@react-three/fiber"
import { useCallback, useEffect, useMemo, useRef } from "react"
import type { Group, Mesh, ShaderMaterial, Texture } from "three"
import {
    AdditiveBlending,
    HalfFloatType,
    LinearFilter,
    MathUtils,
    MeshBasicMaterial,
    OrthographicCamera,
    Scene,
} from "three"
import ripple from "../../assets/ripple.png"
import { type Pointer, WebglImage } from "../webgl-image/webgl-image"
import { WebglVideo } from "../webgl-video/webgl-video"

const ROTATION_SPEED = 0.1 as const
const INITIAL_OPACITY = 0.22 as const
const DISPLACEMENT_DAMPING = 6.3 as const
const VELOCITY_DAMPING = 6.3 as const
const MIN_VELOCITY = 0 as const

const vertexShader = /* glsl */ `
varying vec2 vUv;
varying vec2 vScreenUv;

void main() {
    vUv = uv;
    vec4 pos = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
    vScreenUv = pos.xy / pos.w * 0.5 + 0.5;
    gl_Position = pos;
}`

const fragmentShader = /* glsl */ `
precision highp float;
uniform sampler2D uTexture;
uniform sampler2D uDisplacement;
uniform float uDisplacementIntensity;
varying vec2 vUv;
varying vec2 vScreenUv;

#define PI 3.14159265

void main() {
    vec4 displacement = texture2D(uDisplacement, vScreenUv);
    float theta = displacement.r * 2.0 * PI;
    vec2 direction = vec2(sin(theta), cos(theta));
    vec2 displacedUv = vUv + direction * displacement.r * uDisplacementIntensity;
    vec4 color = texture2D(uTexture, displacedUv);
    gl_FragColor = color;
}`

type LiquidMediaMaterialProps = {
    map: Texture
    pointer: Pointer
    rippleMap?: Texture
    intensity?: number
    radius?: number
    expandRate?: number
    decayRate?: number
    maxRipples?: number
}

type Uniforms = {
    uTexture: { value: Texture | null }
    uDisplacement: { value: Texture | null }
    uDisplacementIntensity: { value: number }
}

// Effect props shared by both the image and video variants.
export type LiquidEffectProps = {
    rippleMap?: Texture
    intensity?: number
    radius?: number
    expandRate?: number
    decayRate?: number
    maxRipples?: number
    segments?: number
    webglEnabled?: boolean
}

type LiquidMediaImageProps = LiquidEffectProps & {
    type?: "image"
    src: string
    alt: string
} & Omit<React.ComponentPropsWithoutRef<"img">, "src" | "alt">

type LiquidMediaVideoProps = LiquidEffectProps & {
    type: "video"
    src: string
} & Omit<React.ComponentPropsWithoutRef<"video">, "src">

export type LiquidMediaProps = LiquidMediaImageProps | LiquidMediaVideoProps

function LiquidMediaMaterial({
    map,
    pointer,
    rippleMap,
    intensity = 0.14,
    radius = 3,
    expandRate = 7,
    decayRate = 3,
    maxRipples = 50,
}: LiquidMediaMaterialProps) {
    const { viewport, size, gl } = useThree()

    const defaultBrush = useTexture(typeof ripple === "string" ? ripple : ripple.src)
    const brush = rippleMap ?? defaultBrush
    const anchorRef = useRef<Group>(null)
    const prevMouse = useRef({ x: 0, y: 0, velocity: 0 })
    const splatIndex = useRef(0)
    const spriteRefs = useRef<Mesh[]>([])
    const displacementSmoothed = useRef(0)
    const spriteScene = useMemo(() => new Scene(), [])
    const spriteCamera = useMemo(() => new OrthographicCamera(-1, 1, 1, -1, 0, 1), [])
    const materialRef = useRef<ShaderMaterial>(null)

    const uniforms = useMemo<Uniforms>(
        () => ({
            uTexture: { value: map },
            uDisplacement: { value: null },
            uDisplacementIntensity: { value: 0 },
        }),
        [map],
    )

    const FBO = useFBO(size.width, size.height, {
        minFilter: LinearFilter,
        magFilter: LinearFilter,
        type: HalfFloatType,
    })

    useEffect(() => {
        uniforms.uTexture.value = map
    }, [map, uniforms])

    useEffect(() => {
        spriteCamera.left = -viewport.width / 2
        spriteCamera.right = viewport.width / 2
        spriteCamera.top = viewport.height / 2
        spriteCamera.bottom = -viewport.height / 2
        spriteCamera.updateProjectionMatrix()
    }, [viewport, spriteCamera])

    useFrame((_, delta) => {
        const parent = anchorRef.current?.parent as Mesh | null
        const mat = materialRef.current
        if (!parent || !mat) return

        const sprites = spriteRefs.current
        const pointerX = parent.position.x + (pointer.uv.x - 0.5) * parent.scale.x
        const pointerY = parent.position.y + (pointer.uv.y - 0.5) * parent.scale.y
        const dx = pointerX - prevMouse.current.x
        const dy = pointerY - prevMouse.current.y
        const dist = Math.sqrt(dx * dx + dy * dy)

        prevMouse.current.x = pointerX
        prevMouse.current.y = pointerY

        const hovering = pointer.hover > 0.5

        if (hovering) {
            prevMouse.current.velocity = Math.max(
                MIN_VELOCITY,
                MathUtils.damp(prevMouse.current.velocity, dist, VELOCITY_DAMPING, delta),
            )
        }

        if (hovering && dist > 0.001) {
            const idx = splatIndex.current % maxRipples
            const sprite = sprites[idx]

            if (sprite.material instanceof MeshBasicMaterial) {
                const scale = (radius * Math.min(viewport.width, viewport.height)) / 100
                sprite.visible = true
                sprite.position.set(pointerX, pointerY, 0)
                sprite.scale.set(scale, scale, 1)
                sprite.material.opacity = INITIAL_OPACITY
            }
            splatIndex.current = (splatIndex.current + 1) % maxRipples
        }

        for (const sprite of sprites) {
            if (sprite.material instanceof MeshBasicMaterial) {
                sprite.rotation.z += 2 * delta * ROTATION_SPEED
                sprite.material.opacity = MathUtils.damp(
                    sprite.material.opacity,
                    0,
                    decayRate,
                    delta,
                )
                sprite.scale.x += delta * expandRate
                sprite.scale.y = sprite.scale.x
            }
        }

        gl.setRenderTarget(FBO)
        gl.render(spriteScene, spriteCamera)
        gl.setRenderTarget(null)

        mat.uniforms.uDisplacement.value = FBO.texture

        displacementSmoothed.current = MathUtils.damp(
            displacementSmoothed.current,
            intensity * prevMouse.current.velocity * 5,
            DISPLACEMENT_DAMPING,
            delta,
        )
        mat.uniforms.uDisplacementIntensity.value = displacementSmoothed.current
    })

    const setSprite = useCallback((el: Mesh | null, i: number) => {
        if (!el) return
        spriteRefs.current[i] = el
    }, [])

    const portal = useMemo(
        () =>
            createPortal(
                <group>
                    {Array.from({ length: maxRipples }, (_, i) => (
                        <mesh
                            key={i}
                            ref={(el) => setSprite(el, i)}
                            visible={false}
                            rotation-z={Math.random() * Math.PI * 2}
                        >
                            <planeGeometry args={[1, 1]} />
                            <meshBasicMaterial
                                map={brush}
                                transparent
                                blending={AdditiveBlending}
                                depthTest={false}
                                depthWrite={false}
                            />
                        </mesh>
                    ))}
                </group>,
                spriteScene,
            ),
        [maxRipples, brush, spriteScene, setSprite],
    )

    return (
        <>
            <group ref={anchorRef} />

            <shaderMaterial
                ref={materialRef}
                attach="material"
                vertexShader={vertexShader}
                fragmentShader={fragmentShader}
                uniforms={uniforms}
            />

            {portal}
        </>
    )
}

export function LiquidMedia(props: LiquidMediaProps) {
    const {
        rippleMap,
        intensity,
        radius,
        expandRate,
        decayRate,
        maxRipples,
        segments,
        webglEnabled,
        ...rest
    } = props

    // The ripple displacement is a fragment-only shader, so the same material
    // runs on both the image and video primitives (they share the WebGL plane).
    const material = (map: Texture, pointer: Pointer) => (
        <LiquidMediaMaterial
            map={map}
            pointer={pointer}
            rippleMap={rippleMap}
            intensity={intensity}
            radius={radius}
            expandRate={expandRate}
            decayRate={decayRate}
            maxRipples={maxRipples}
        />
    )

    if (rest.type === "video") {
        const { type: _type, ...videoProps } = rest
        return (
            <WebglVideo
                segments={segments}
                webglEnabled={webglEnabled}
                material={material}
                {...videoProps}
            />
        )
    }

    const { type: _type, ...imageProps } = rest
    return (
        <WebglImage
            segments={segments}
            webglEnabled={webglEnabled}
            material={material}
            {...imageProps}
        />
    )
}
Expand code

import { useTexture } from "@react-three/drei"
import { useFrame, useThree } from "@react-three/fiber"
import {
    type ComponentRef,
    type RefObject,
    useEffect,
    useLayoutEffect,
    useMemo,
    useRef,
} from "react"
import { type Mesh, type Texture, Vector2 } from "three"
import { webglTeleport } from "../webgl-portal/webgl-portal"

export type Pointer = {
    uv: Vector2
    hover: number
}

type WebglImageProps = {
    src: string
    alt: string
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    webglEnabled?: boolean
    segments?: number
    zIndex?: number
    /**
     * Re-measures the DOM rect every frame so the plane follows animated parents (motion, parallax).
     * Costs one layout read per frame, so only enable it when needed.
     */
    autoReflow?: boolean
} & Omit<React.ComponentPropsWithoutRef<"img">, "children" | "src" | "alt">

type PlaneProps = {
    el: RefObject<HTMLImageElement | null>
    src: string
    segments: number
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    pointer: Pointer
    zIndex: number
    autoReflow: boolean
}

function Plane({ el, src, segments, material, pointer, zIndex, autoReflow }: PlaneProps) {
    const mesh = useRef<Mesh>(null)
    const texture = useTexture(src)
    const size = useThree((s) => s.size)
    const viewport = useThree((s) => s.viewport)
    const fitScale = useRef({ x: 1, y: 1 })
    const bounds = useRef({ x: 0, y: 0, width: 0, height: 0 })

    useLayoutEffect(() => {
        const target = el.current
        if (!target) return

        const measure = () => {
            const m = mesh.current
            if (!m) return

            // Rect in document coords (top/left offset by current scroll at measure time),
            // so we can later derive viewport position with just `window.scrollX/Y`,
            // instead of recalculating bounds on every render
            const rect = target.getBoundingClientRect()
            bounds.current.x = rect.left + window.scrollX
            bounds.current.y = rect.top + window.scrollY
            bounds.current.width = rect.width
            bounds.current.height = rect.height

            // Replicate CSS object-fit: cover crops via UV repeat/offset; contain shrinks the mesh scale
            // because UVs alone can't letterbox: the plane would still fill the element.
            const image = texture.image as HTMLImageElement
            const objectFit = getComputedStyle(target).objectFit
            const planeAspect = rect.width / rect.height
            const imageAspect = image.width / image.height

            let repeatU = 1
            let repeatV = 1

            fitScale.current.x = 1
            fitScale.current.y = 1

            if (objectFit === "cover") {
                if (planeAspect > imageAspect) {
                    repeatV = imageAspect / planeAspect
                } else {
                    repeatU = planeAspect / imageAspect
                }
            } else if (objectFit === "contain") {
                if (planeAspect > imageAspect) {
                    fitScale.current.x = imageAspect / planeAspect
                } else {
                    fitScale.current.y = planeAspect / imageAspect
                }
            }

            const offsetU = (1 - repeatU) / 2
            const offsetV = (1 - repeatV) / 2

            const uvAttribute = m.geometry.attributes.uv

            for (let iy = 0; iy <= segments; iy++) {
                for (let ix = 0; ix <= segments; ix++) {
                    const idx = iy * (segments + 1) + ix
                    const u = ix / segments
                    const v = 1 - iy / segments
                    uvAttribute.setXY(idx, u * repeatU + offsetU, v * repeatV + offsetV)
                }
            }

            uvAttribute.needsUpdate = true
        }

        measure()

        const ro = new ResizeObserver(measure)
        ro.observe(target)
        ro.observe(document.body)
        return () => ro.disconnect()
    }, [el, texture, segments])

    useFrame(() => {
        const m = mesh.current
        if (!m) return
        const pxToWorld = viewport.height / size.height

        // autoReflow re-reads the rect each frame so the mesh follows parent
        // CSS transforms (e.g. parallax). One layout read per frame.
        if (autoReflow && el.current) {
            const rect = el.current.getBoundingClientRect()
            m.position.x = (rect.left + rect.width / 2 - size.width / 2) * pxToWorld
            m.position.y = -(rect.top + rect.height / 2 - size.height / 2) * pxToWorld
            m.scale.x = rect.width * pxToWorld * fitScale.current.x
            m.scale.y = rect.height * pxToWorld * fitScale.current.y
            return
        }

        const { x, y, width, height } = bounds.current
        m.position.x = (x + width / 2 - window.scrollX - size.width / 2) * pxToWorld
        m.position.y = -(y + height / 2 - window.scrollY - size.height / 2) * pxToWorld
        m.scale.x = width * pxToWorld * fitScale.current.x
        m.scale.y = height * pxToWorld * fitScale.current.y
    })

    return (
        <mesh ref={mesh} renderOrder={zIndex}>
            <planeGeometry args={[1, 1, segments, segments]} />
            {material ? (
                material(texture, pointer)
            ) : (
                <meshBasicMaterial map={texture} transparent />
            )}
        </mesh>
    )
}

export function WebglImage({
    src,
    alt,
    className,
    style,
    material,
    webglEnabled = true,
    segments = 1,
    zIndex = 0,
    autoReflow = false,
    ...rest
}: WebglImageProps) {
    const el = useRef<ComponentRef<"img">>(null)
    const pointer = useMemo<Pointer>(() => {
        return {
            uv: new Vector2(0.5, 0.5),
            hover: 0,
        }
    }, [])

    useEffect(() => {
        if (!webglEnabled) return
        const target = el.current
        if (!target) return

        // Pointer events still fire on the DOM element through opacity:0,
        // so the browser tells us when the cursor is over it.
        const onMove = (e: PointerEvent) => {
            const { width, left, top, height } = target.getBoundingClientRect()
            const x = (e.clientX - left) / width
            const y = 1 - (e.clientY - top) / height
            pointer.uv.set(x, y)
        }

        const onEnter = () => (pointer.hover = 1)
        const onLeave = () => (pointer.hover = 0)

        target.addEventListener("pointermove", onMove)
        target.addEventListener("pointerenter", onEnter)
        target.addEventListener("pointerleave", onLeave)
        return () => {
            target.removeEventListener("pointermove", onMove)
            target.removeEventListener("pointerenter", onEnter)
            target.removeEventListener("pointerleave", onLeave)
        }
    }, [webglEnabled, pointer])

    return (
        <>
            <img
                ref={el}
                src={src}
                alt={alt}
                className={className}
                style={webglEnabled ? { ...style, opacity: 0 } : style}
                {...rest}
            />

            {webglEnabled && (
                <webglTeleport.In>
                    <Plane
                        el={el}
                        src={src}
                        segments={segments}
                        material={material}
                        pointer={pointer}
                        zIndex={zIndex}
                        autoReflow={autoReflow}
                    />
                </webglTeleport.In>
            )}
        </>
    )
}
Expand code

import { useFrame, useThree } from "@react-three/fiber"
import {
    type ComponentRef,
    type RefObject,
    useEffect,
    useLayoutEffect,
    useMemo,
    useRef,
    useState,
} from "react"
import { type Mesh, SRGBColorSpace, type Texture, Vector2, VideoTexture } from "three"
import { webglTeleport } from "../webgl-portal/webgl-portal"

export type Pointer = {
    uv: Vector2
    hover: number
}

type WebglVideoProps = {
    src: string
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    webglEnabled?: boolean
    segments?: number
    zIndex?: number
    /**
     * Re-measures the DOM rect every frame so the plane follows animated parents (motion, parallax).
     * Costs one layout read per frame, so only enable it when needed.
     */
    autoReflow?: boolean
} & Omit<React.ComponentPropsWithoutRef<"video">, "children" | "src">

type PlaneProps = {
    el: RefObject<HTMLVideoElement | null>
    segments: number
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    pointer: Pointer
    zIndex: number
    autoReflow: boolean
}

function Plane({ el, segments, material, pointer, zIndex, autoReflow }: PlaneProps) {
    const mesh = useRef<Mesh>(null)
    const [texture, setTexture] = useState<VideoTexture | null>(null)
    const size = useThree((state) => state.size)
    const viewport = useThree((state) => state.viewport)
    const fitScale = useRef({ x: 1, y: 1 })
    const bounds = useRef({ x: 0, y: 0, width: 0, height: 0 })

    useLayoutEffect(() => {
        const video = el.current
        if (!video) return

        // Build the texture from the DOM <video> itself so a single element
        // decodes once; VideoTexture pulls each new frame from it.
        const videoTexture = new VideoTexture(video)
        videoTexture.colorSpace = SRGBColorSpace
        setTexture(videoTexture)
        return () => videoTexture.dispose()
    }, [el])

    useLayoutEffect(() => {
        const target = el.current
        if (!target || !texture) return

        const measure = () => {
            const m = mesh.current
            if (!m) return

            // Rect in document coords (top/left offset by current scroll at measure time),
            // so we can later derive viewport position with just `window.scrollX/Y`,
            // instead of recalculating bounds on every render
            const rect = target.getBoundingClientRect()
            bounds.current.x = rect.left + window.scrollX
            bounds.current.y = rect.top + window.scrollY
            bounds.current.width = rect.width
            bounds.current.height = rect.height

            // Replicate CSS object-fit: cover crops via UV repeat/offset; contain shrinks the mesh scale
            // because UVs alone can't letterbox: the plane would still fill the element.
            const video = texture.image as HTMLVideoElement
            const objectFit = getComputedStyle(target).objectFit
            const planeAspect = rect.width / rect.height
            const videoAspect = video.videoWidth / video.videoHeight

            let repeatU = 1
            let repeatV = 1

            fitScale.current.x = 1
            fitScale.current.y = 1

            // videoWidth/Height are 0 until metadata loads; skip cropping until then.
            if (video.videoWidth > 0) {
                if (objectFit === "cover") {
                    if (planeAspect > videoAspect) {
                        repeatV = videoAspect / planeAspect
                    } else {
                        repeatU = planeAspect / videoAspect
                    }
                } else if (objectFit === "contain") {
                    if (planeAspect > videoAspect) {
                        fitScale.current.x = videoAspect / planeAspect
                    } else {
                        fitScale.current.y = planeAspect / videoAspect
                    }
                }
            }

            const offsetU = (1 - repeatU) / 2
            const offsetV = (1 - repeatV) / 2

            const uvAttribute = m.geometry.attributes.uv

            for (let iy = 0; iy <= segments; iy++) {
                for (let ix = 0; ix <= segments; ix++) {
                    const idx = iy * (segments + 1) + ix
                    const u = ix / segments
                    const v = 1 - iy / segments
                    uvAttribute.setXY(idx, u * repeatU + offsetU, v * repeatV + offsetV)
                }
            }

            uvAttribute.needsUpdate = true
        }

        measure()

        // Re-measure once the video reports its intrinsic size.
        target.addEventListener("loadedmetadata", measure)
        target.addEventListener("resize", measure)

        const ro = new ResizeObserver(measure)
        ro.observe(target)
        ro.observe(document.body)
        return () => {
            ro.disconnect()
            target.removeEventListener("loadedmetadata", measure)
            target.removeEventListener("resize", measure)
        }
    }, [el, texture, segments])

    useFrame(() => {
        const m = mesh.current
        if (!m) return

        // Browsers without requestVideoFrameCallback need an explicit pull.
        texture?.update()

        const pxToWorld = viewport.height / size.height

        // autoReflow re-reads the rect each frame so the mesh follows parent
        // CSS transforms (e.g. parallax). One layout read per frame.
        if (autoReflow && el.current) {
            const rect = el.current.getBoundingClientRect()
            m.position.x = (rect.left + rect.width / 2 - size.width / 2) * pxToWorld
            m.position.y = -(rect.top + rect.height / 2 - size.height / 2) * pxToWorld
            m.scale.x = rect.width * pxToWorld * fitScale.current.x
            m.scale.y = rect.height * pxToWorld * fitScale.current.y
            return
        }

        const { x, y, width, height } = bounds.current
        m.position.x = (x + width / 2 - window.scrollX - size.width / 2) * pxToWorld
        m.position.y = -(y + height / 2 - window.scrollY - size.height / 2) * pxToWorld
        m.scale.x = width * pxToWorld * fitScale.current.x
        m.scale.y = height * pxToWorld * fitScale.current.y
    })

    if (!texture) return null

    return (
        <mesh ref={mesh} renderOrder={zIndex}>
            <planeGeometry args={[1, 1, segments, segments]} />
            {material ? (
                material(texture, pointer)
            ) : (
                <meshBasicMaterial map={texture} transparent />
            )}
        </mesh>
    )
}

export function WebglVideo({
    src,
    className,
    style,
    material,
    webglEnabled = true,
    segments = 1,
    zIndex = 0,
    autoReflow = false,
    autoPlay = true,
    muted = true,
    loop = true,
    playsInline = true,
    ...rest
}: WebglVideoProps) {
    const el = useRef<ComponentRef<"video">>(null)
    const pointer = useMemo<Pointer>(() => {
        return {
            uv: new Vector2(0.5, 0.5),
            hover: 0,
        }
    }, [])

    useEffect(() => {
        if (!webglEnabled) return
        const target = el.current
        if (!target) return

        // Pointer events still fire on the DOM element through opacity:0,
        // so the browser tells us when the cursor is over it.
        const onMove = (event: PointerEvent) => {
            const { width, left, top, height } = target.getBoundingClientRect()
            const x = (event.clientX - left) / width
            const y = 1 - (event.clientY - top) / height
            pointer.uv.set(x, y)
        }

        const onEnter = () => (pointer.hover = 1)
        const onLeave = () => (pointer.hover = 0)

        target.addEventListener("pointermove", onMove)
        target.addEventListener("pointerenter", onEnter)
        target.addEventListener("pointerleave", onLeave)
        return () => {
            target.removeEventListener("pointermove", onMove)
            target.removeEventListener("pointerenter", onEnter)
            target.removeEventListener("pointerleave", onLeave)
        }
    }, [webglEnabled, pointer])

    return (
        <>
            <video
                ref={el}
                src={src}
                className={className}
                style={webglEnabled ? { ...style, opacity: 0 } : style}
                autoPlay={autoPlay}
                muted={muted}
                loop={loop}
                playsInline={playsInline}
                {...rest}
            />

            {webglEnabled && (
                <webglTeleport.In>
                    <Plane
                        el={el}
                        segments={segments}
                        material={material}
                        pointer={pointer}
                        zIndex={zIndex}
                        autoReflow={autoReflow}
                    />
                </webglTeleport.In>
            )}
        </>
    )
}
Expand code