film-stills

"use client"

// Stills of a film on a page: the cache of lib/film-stills-core held for a component's
// life, the DECODER that makes a still where none was prepared (one hidden video,
// seeked to each instant asked, its frame kept as a bitmap), and the picture a tile
// draws. Where stills come from, and why, is lib/film-stills-core's header and
// film-stills.notes.md.
//
//   const decoded = useSeekStills("/film.mp4", { width: 480 })
//   const stills = useStills([frameStills(asset.frames), decoded.source])
//   <StillPicture stills={stills} at={12_000} />
//
// A decoder needs the film's frame rate to know its grain (a frame); given none it
// MEASURES it, playing the hidden video muted for a moment and timing the frames the
// browser presents, as a video timebase does (lib/timebase `measureRate`): a file
// picked from a disk says its own rate. A video on another origin decodes too: its
// frames are drawn, never read back, so no CORS is needed.

import * as React from "react"
import { cn } from "@/lib/utils"
import {
  createStills,
  type StillSource,
  type Stills,
} from "@/registry/base-nova/lib/film-stills-core"
import {
  frameAt,
  frameTime,
  measureRate,
  msOf,
  seekSeconds,
} from "@/registry/base-nova/lib/timebase"
import { Image } from "@/registry/base-nova/ui/image"

/** A still as a page holds it: an address for an <img>, or a bitmap decoded here. */
export type Still = string | ImageBitmap

/** How long the decoder plays a video to time its frames before it gives up, ms: long
 *  enough for `measureRate`'s eight gaps at the slowest rate it accepts. */
const MEASURE_MS = 3000

/** The cache over `sources` (coarse to fine; a null rung, a decoder still measuring,
 *  is left out until it is ready), made when the ladder changes and disposed with the
 *  component. `keep`: pictures each rung holds beyond the ones on screen. A stable
 *  list of sources (memoized, or each made once) keeps the cache and its pictures. */
export function useStills(
  sources: readonly (StillSource<Still> | null)[],
  { keep = 64 }: { keep?: number } = {},
): Stills<Still> | null {
  const ladder = sources.filter((s): s is StillSource<Still> => s !== null)
  // The ladder as its sources' identities: a new cache only for new sources.
  const key = ladder.map(identity).join(" ")
  const current = React.useRef(ladder)
  current.current = ladder
  const [stills, setStills] = React.useState<Stills<Still> | null>(null)
  // biome-ignore lint/correctness/useExhaustiveDependencies: the ladder is read through its key
  React.useEffect(() => {
    if (current.current.length === 0) return
    const made = createStills(current.current, { keep })
    setStills(made)
    return () => {
      setStills(null)
      made.dispose()
    }
  }, [key, keep])
  return stills
}

const identities = new WeakMap<object, number>()
let identified = 0
/** A source's identity for a dependency list: one number per object, for its life. */
function identity(source: object): number {
  const known = identities.get(source)
  if (known !== undefined) return known
  identities.set(source, ++identified)
  return identified
}

/** A decoder's state: measuring the film (`source` null), ready, or failed (the file
 *  would not load, or its frames could not be timed). */
export interface SeekStills {
  source: StillSource<Still> | null
  failed: string | null
}

/** The decoder: one hidden video seeked to each instant asked, one at a time, its
 *  frame kept as a bitmap as wide as `width` CSS px on this display (the widest a tile
 *  draws it). `fps` when the film's rate is known; else it is measured (see the
 *  header). Null `src` makes nothing. */
export function useSeekStills(
  src: string | null,
  { fps, width }: { fps?: number; width: number },
): SeekStills {
  const [state, setState] = React.useState<SeekStills>({
    source: null,
    failed: null,
  })
  React.useEffect(() => {
    setState({ source: null, failed: null })
    if (!src) return
    const resize = Math.max(1, Math.round(width * window.devicePixelRatio))
    const v = document.createElement("video")
    v.muted = true
    v.playsInline = true
    v.preload = "auto"
    v.src = src
    let dead = false
    const fail = (why: string) => {
      if (!dead) setState({ source: null, failed: why })
    }
    const loaded = new Promise<void>((resolve, reject) => {
      if (v.readyState >= 1) return resolve()
      v.addEventListener("loadedmetadata", () => resolve(), { once: true })
      v.addEventListener(
        "error",
        () => reject(new Error(`${src} did not load`)),
        {
          once: true,
        },
      )
    })
    /** The film's rate: given, or timed off the frames it presents playing at 1×. */
    const rate = async (): Promise<number> => {
      if (fps) return fps
      const gaps: number[] = []
      let last: number | null = null
      let handle = 0
      const timed = new Promise<number | null>((resolve) => {
        const onFrame = (_: number, meta: VideoFrameCallbackMetadata) => {
          const at = meta.mediaTime * 1000
          if (last !== null && at > last) gaps.push(at - last)
          last = at
          const measured = measureRate(gaps)
          if (measured) return resolve(measured)
          handle = v.requestVideoFrameCallback(onFrame)
        }
        handle = v.requestVideoFrameCallback(onFrame)
        window.setTimeout(() => resolve(null), MEASURE_MS)
      })
      await v.play()
      const measured = await timed
      v.cancelVideoFrameCallback(handle)
      v.pause()
      if (!measured)
        throw new Error(`${src}: its frames could not be timed to a rate`)
      return measured
    }
    loaded
      .then(rate)
      .then((measured) => {
        if (dead) return
        const duration = msOf(v.duration)
        // The last frame is the one at or before its end.
        const last = frameTime(
          frameAt(Math.max(0, duration - 1), measured),
          measured,
        )
        let busy: Promise<unknown> = Promise.resolve()
        const source: StillSource<Still> = {
          grain: 1000 / measured,
          place: (at) =>
            frameTime(
              frameAt(Math.min(last, Math.max(0, at)), measured),
              measured,
            ),
          parallel: 1,
          // One seek at a time, chained: the element is one decoder.
          still: (at, signal) => {
            const made = busy.then(async () => {
              signal.throwIfAborted()
              await new Promise<void>((resolve) => {
                v.addEventListener("seeked", () => resolve(), { once: true })
                v.currentTime = seekSeconds(at)
              })
              signal.throwIfAborted()
              return createImageBitmap(v, {
                resizeWidth: resize,
                resizeQuality: "medium",
              })
            })
            busy = made.catch(() => {})
            return made
          },
          release: (picture) => {
            if (typeof picture !== "string") picture.close()
          },
        }
        setState({ source, failed: null })
      })
      .catch((e: unknown) => fail(e instanceof Error ? e.message : String(e)))
    return () => {
      dead = true
      // A detached element keeps its media: release it (the spec's own procedure).
      v.pause()
      v.removeAttribute("src")
      v.load()
    }
  }, [src, fps, width])
  return state
}

/** The best picture the cache has of `at`, filling its box (object-cover): an address
 *  as an <img> that fades in once, a bitmap drawn on a canvas. Nothing yet: the box's
 *  own ground shows. Wears `data-status` (shown, standing-in, loading, failed). */
export function StillPicture({
  stills,
  at,
  className,
}: {
  stills: Stills<Still>
  at: number
  className?: string
}) {
  const read = () => stills.shown(at)
  const shown = React.useSyncExternalStore(stills.subscribe, read, read)
  const status = React.useSyncExternalStore(
    stills.subscribe,
    () => stills.status(at),
    () => stills.status(at),
  )
  const picture = shown?.picture
  const box = cn(
    "pointer-events-none absolute inset-0 size-full object-cover",
    className,
  )
  if (picture === undefined)
    return <span aria-hidden data-slot="still-picture" data-status={status} />
  if (typeof picture === "string")
    return (
      <Image
        data-slot="still-picture"
        data-status={status}
        src={picture}
        alt=""
        draggable={false}
        decoding="async"
        className={box}
        onError={() => {
          if (shown) stills.broken(shown)
        }}
      />
    )
  return <BitmapPicture bitmap={picture} status={status} className={box} />
}

/** A decoded still on a canvas the bitmap's own size, scaled by CSS like an image. */
function BitmapPicture({
  bitmap,
  status,
  className,
}: {
  bitmap: ImageBitmap
  status: string
  className: string
}) {
  const canvas = React.useRef<HTMLCanvasElement>(null)
  React.useLayoutEffect(() => {
    const c = canvas.current
    if (!c) return
    c.width = bitmap.width
    c.height = bitmap.height
    c.getContext("2d")?.drawImage(bitmap, 0, 0)
  }, [bitmap])
  return (
    <canvas
      ref={canvas}
      aria-hidden
      data-slot="still-picture"
      data-status={status}
      className={className}
    />
  )
}

// STILLS OF A FILM: a picture of it at an instant, from wherever pictures come from, and
// the one cache that asks for them in the order a page needs them. Framework-free;
// ui/film-stills adds the decoder that seeks a video on the page.
//
// WHERE A STILL COMES FROM is a `StillSource`, and it says three things about itself:
//
//   grain   the finest step between two stills it can tell apart, ms: a prepared
//           frame every second is 1000, a decoder seeking a 25 fps video is 40. Below
//           its grain two tiles would show the same picture, so a view is never cut
//           finer than this (lib/film-tiles, lib/film-zoom-core).
//   place   the instant of the still it has at or before an instant, whole ms: the
//           second a prepared frame was cut at, the name of a decoded frame. A tile
//           starts on a placed instant, so the picture it shows is the first frame its
//           span plays.
//   still   the picture at a placed instant: an address answered at once (a prepared
//           frame, a CDN's frame), or a promise of a decoded one (`parallel` at a time,
//           a decoder seeks one at a time).
//
// THE CACHE takes a LADDER of sources, coarse to fine: a prepared frame every second
// paints at once, the frame decoded for the exact instant replaces it when it lands
// (the ladder's promise: the content never swaps for another moment, it only sharpens).
// `want` says what is on screen, in the order it matters; what is not there yet is made
// in that order, the newest want replacing the queue. A still under way is never
// stopped for a newer want (a decoder stopped mid-seek has bought nothing, and a scrub
// that stops every seek never lands one): the staleness is one still, bounded. A still
// that fails is not asked again; its tile shows what stands in for it, or nothing.

/** Where pictures of a film come from (see the header). `P` is what a picture is: an
 *  address for an <img>, a decoded bitmap for a canvas. */
export interface StillSource<P = string> {
  /** The finest step between two stills it tells apart, ms. */
  grain: number
  /** The instant of the still it has at or before `at`, whole ms. */
  place: (at: number) => number
  /** The still at a placed instant: at once, or a promise. `signal` aborts when the
   *  cache is disposed. */
  still: (at: number, signal: AbortSignal) => P | Promise<P>
  /** How many stills it makes at once. Default: every one asked (an address costs
   *  nothing to name; the browser fetches what it shows). */
  parallel?: number
  /** Gives back what a picture holds (a bitmap's memory) when it leaves the cache.
   *  (A method: an address source, which releases nothing, stands in a ladder of
   *  wider pictures.) */
  release?(picture: P): void
}

/** A picture the cache holds: what it is, which rung of the ladder made it (0 the
 *  coarsest), and the placed instant it shows. One object per picture, so a reader
 *  comparing what it got with what it had sees a change only when there is one. */
export interface Shown<P> {
  picture: P
  rung: number
  at: number
}

export type StillStatus = "shown" | "standing-in" | "loading" | "failed"

export interface Stills<P> {
  /** The finest grain of the ladder, ms. */
  grain: number
  /** The finest rung's placement: where a tile starts. */
  place: (at: number) => number
  /** The best picture of `at` now: the finest rung's own once made, else what a
   *  coarser rung has at or before it, else null. */
  shown: (at: number) => Shown<P> | null
  /** Whether `at` shows its own still, a stand-in, nothing yet, or never will. */
  status: (at: number) => StillStatus
  /** The instants a page shows, the first first: what is missing is made in that order
   *  (each rung its own queue), a newer want replacing what is still queued. */
  want: (ats: readonly number[]) => void
  /** A picture that would not draw (a named still whose file is not there): it
   *  fails as a still that could not be made does, and what stands in shows. Said by
   *  whoever drew it, since only a drawing learns that an address answers nothing. */
  broken: (shown: Shown<P>) => void
  subscribe: (listener: () => void) => () => void
  /** Moves on every change: a still landed, one failed. */
  version: () => number
  /** Stops what is under way and gives back every picture. */
  dispose: () => void
}

function assert(ok: unknown, why: string): asserts ok {
  if (!ok) throw new Error(`film-stills: ${why}`)
}

/** Floors that survive a float's last bit: 12 × (1000/30) / (1000/30) is 11.999…, and
 *  the still at or before it must still be the 12th. */
const floorOf = (x: number) => Math.floor(x + 1e-9)

/** A source that names its stills: `address(at)` for every placed instant on a grid of
 *  `grain` ms from 0, the last at `last` (inclusive). Prepared frames, an edge's frame
 *  at a second (media-cdn), a server's still tier. */
export function addressStills(
  grain: number,
  last: number,
  address: (at: number) => string,
): StillSource<string> {
  assert(grain > 0 && Number.isFinite(grain), `a grain of ${grain} ms`)
  assert(last >= 0, `a last still at ${last} ms`)
  const top = floorOf(last / grain)
  return {
    grain,
    place: (at) =>
      Math.round(Math.min(top, Math.max(0, floorOf(at / grain))) * grain),
    still: (at) => address(at),
  }
}

/** The frames a media asset was prepared with (lib/media-asset's `frames`: frame i the
 *  picture at i × every seconds, at `src` with `{i}` replaced): a still every `every`
 *  seconds, named at once. */
export function frameStills(frames: {
  every: number
  count: number
  src: string
}): StillSource<string> {
  assert(frames.src.includes("{i}"), `frames at ${frames.src} name no {i}`)
  assert(frames.count >= 1, `${frames.count} frames`)
  const grain = frames.every * 1000
  return addressStills(grain, (frames.count - 1) * grain, (at) =>
    frames.src.replace("{i}", String(Math.round(at / grain))),
  )
}

/** A rung of the ladder: its source, the pictures it holds (in the order they were
 *  last asked for: a Map keeps insertion order), the ones it could not make, what is
 *  under way, and its queue. */
interface Rung<P> {
  source: StillSource<P>
  held: Map<number, Shown<P>>
  failed: Set<number>
  making: Set<number>
  queue: number[]
  wanted: Set<number>
}

/** The cache over a ladder of sources, coarse to fine (see the header). `keep` is how
 *  many pictures each rung holds beyond the ones wanted now; a wanted picture is never
 *  given back. */
export function createStills<P>(
  sources: readonly StillSource<P>[],
  { keep }: { keep: number },
): Stills<P> {
  assert(sources.length > 0, "a ladder of no sources")
  assert(keep >= 0, `keep ${keep}`)
  for (let i = 1; i < sources.length; i++)
    assert(
      (sources[i] as StillSource<P>).grain <=
        (sources[i - 1] as StillSource<P>).grain,
      "a ladder runs coarse to fine: each rung's grain at most the one before's",
    )
  const rungs: Rung<P>[] = sources.map((source) => ({
    source,
    held: new Map(),
    failed: new Set(),
    making: new Set(),
    queue: [],
    wanted: new Set(),
  }))
  const finest = rungs.length - 1
  const fine = sources[finest] as StillSource<P>
  const listeners = new Set<() => void>()
  const abort = new AbortController()
  let version = 0
  const changed = () => {
    version++
    for (const l of listeners) l()
  }

  /** Over `keep` beyond what is wanted, the least recently asked leave. */
  const trim = (rung: Rung<P>) => {
    const room = keep + rung.wanted.size
    for (const [at, shown] of rung.held) {
      if (rung.held.size <= room) break
      if (rung.wanted.has(at)) continue
      rung.held.delete(at)
      rung.source.release?.(shown.picture)
    }
  }

  const land = (k: number, at: number, picture: P) => {
    const rung = rungs[k] as Rung<P>
    if (abort.signal.aborted) {
      rung.source.release?.(picture)
      return
    }
    rung.held.set(at, { picture, rung: k, at })
    trim(rung)
  }

  const pump = (k: number) => {
    const rung = rungs[k] as Rung<P>
    const parallel = rung.source.parallel ?? Number.POSITIVE_INFINITY
    let landed = false
    while (rung.making.size < parallel && rung.queue.length > 0) {
      const at = rung.queue.shift() as number
      if (rung.held.has(at) || rung.failed.has(at) || rung.making.has(at))
        continue
      let made: P | Promise<P>
      try {
        made = rung.source.still(at, abort.signal)
      } catch {
        rung.failed.add(at)
        landed = true
        continue
      }
      if (!(made instanceof Promise)) {
        land(k, at, made)
        landed = true
        continue
      }
      rung.making.add(at)
      made.then(
        (picture) => {
          rung.making.delete(at)
          land(k, at, picture)
          if (abort.signal.aborted) return
          changed()
          pump(k)
        },
        () => {
          rung.making.delete(at)
          if (abort.signal.aborted) return
          rung.failed.add(at)
          changed()
          pump(k)
        },
      )
    }
    if (landed) changed()
  }

  return {
    grain: fine.grain,
    place: fine.place,
    shown: (at) => {
      for (let k = finest; k >= 0; k--) {
        const rung = rungs[k] as Rung<P>
        const shown = rung.held.get(rung.source.place(at))
        if (shown) return shown
      }
      return null
    },
    status: (at) => {
      const own = fine.place(at)
      const top = rungs[finest] as Rung<P>
      if (top.held.has(own)) return "shown"
      for (let k = finest - 1; k >= 0; k--) {
        const rung = rungs[k] as Rung<P>
        if (rung.held.has(rung.source.place(at)))
          return top.failed.has(own) ? "failed" : "standing-in"
      }
      return top.failed.has(own) ? "failed" : "loading"
    },
    want: (ats) => {
      assert(!abort.signal.aborted, "want after dispose")
      for (let k = 0; k < rungs.length; k++) {
        const rung = rungs[k] as Rung<P>
        const placed = [...new Set(ats.map((at) => rung.source.place(at)))]
        rung.wanted = new Set(placed)
        // Asked again: to the back of the Map's order, the last to leave.
        for (const at of placed) {
          const shown = rung.held.get(at)
          if (!shown) continue
          rung.held.delete(at)
          rung.held.set(at, shown)
        }
        rung.queue = placed.filter(
          (at) =>
            !rung.held.has(at) && !rung.failed.has(at) && !rung.making.has(at),
        )
        pump(k)
      }
    },
    broken: (shown) => {
      const rung = rungs[shown.rung]
      if (!rung || rung.held.get(shown.at) !== shown) return
      rung.held.delete(shown.at)
      rung.failed.add(shown.at)
      rung.source.release?.(shown.picture)
      changed()
    },
    subscribe: (listener) => {
      listeners.add(listener)
      return () => {
        listeners.delete(listener)
      }
    },
    version: () => version,
    dispose: () => {
      abort.abort()
      for (const rung of rungs) {
        for (const shown of rung.held.values())
          rung.source.release?.(shown.picture)
        rung.held.clear()
        rung.queue = []
      }
      listeners.clear()
    },
  }
}