From 788319d098f22dad34d14b8d79d9bc4e06e85476 Mon Sep 17 00:00:00 2001 From: larssand Date: Fri, 11 Sep 2026 17:36:42 +0200 Subject: [PATCH] Add spoken narration to the video guide recorder The clips were silent screen captures - the user's actual ask was for someone talking through what's happening. Each clip step now carries a narration line synthesized offline via espeak-ng; the recording paces itself to each line's real spoken duration, and the final audio track is built from the real measured per-line start times (adelay + amix) so narration and on-screen action stay in sync over a 30+ second clip instead of drifting against an idealized schedule. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 6 + ROADMAP.md | 11 ++ package-lock.json | 4 +- package.json | 2 +- scripts/record-video-guides.mjs | 208 ++++++++++++++++++++++---------- 5 files changed, 166 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f2a42c..c7844b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ The application version is maintained in `package.json` and displayed in the adm No changes recorded yet. +## 1.18.1 - 2026-09-11 + +### Changed + +- `scripts/record-video-guides.mjs` now records narrated video guides, not silent screen captures: each clip is a sequence of spoken lines (synthesized offline with `espeak-ng`) timed to the on-screen action they describe, mixed onto the video's real measured timing so narration and action stay in sync. + ## 1.18.0 - 2026-09-11 ### Added diff --git a/ROADMAP.md b/ROADMAP.md index fa471d1..c55a4fb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -443,6 +443,17 @@ Suggested 2026-09-11 by the user, alongside the permanent-tests entry above: whi Verified 2026-09-11: extracted and visually inspected sample frames from all three clips after the scroll fix - Settings clip clearly shows each tab's own panel switching, Event Manager clip shows the tab bar plus the Open Grid button jumping to the Live tab, and the Schedule clip shows the filled-in form followed by the real cascaded table (Practice 1 12:30-12:40, Qualifying 1 12:43-12:48 with the 3-minute gap applied) - all legible, no garbled or blank frames. `ffprobe` confirmed all three are valid 1600x900 H.264 MKV files (10-13s each, under 750KB). +### Follow-up: the clips needed a spoken narration track, not just a silent screen capture + +The user's original ask ("a video guide showing how to do the various things") meant a narrated walkthrough - someone talking through what's on screen - not a silent recording. Rebuilt the recorder around that: + +- [x] Installed `espeak-ng` (offline, open-source TTS - confirmed with the user first since it's a system package install, not something scoped to the repo) and switched each clip's definition from a flat list of `{ action, pause }` steps to `{ text, action }` steps: `text` is a narration line, synthesized to a WAV up front via `espeak-ng` so its real spoken duration is known before recording starts. +- [x] Rebuilt the timing model around that: the recording pauses for each line's actual measured duration (not a guessed fixed delay) after running its action, and the real wall-clock offset each line actually started at (network/DOM latency included) is recorded live during capture - not assumed from an idealized schedule, which would have drifted out of sync with page-load and IPC latency over a 30+ second clip. +- [x] The narration track is built after recording from those real per-line offsets, via ffmpeg's `adelay` (place each line at its own measured start time) and `amix` (`normalize=0`, since lines never overlap so there's no reason to quietly attenuate them), then muxed onto the silent video as an AAC audio stream. +- [x] Wrote actual narration scripts for all three clips (5-6 short spoken lines each) explaining what each tab/panel is for as it comes into view, not just describing the click. + +Verified 2026-09-11: re-recorded all three clips and confirmed each now has both a video and an AAC audio stream (`ffprobe`). Extracted the volume envelope of the settings-tabs clip's audio and confirmed 5 distinct speech bursts separated by clean silence, matching its 5 narration lines. Cross-checked video frames against the audio timeline: at 8s into settings-tabs (mid second line, "Audio controls race sound announcements...") the Audio tab is the one on screen; at 20s into event-manager-tabs (the Schedule line) the Schedule tab/panel is on screen; at 12s into event-schedule (the "enter a start time" line) the form is visible filled in but not yet saved - narration and visuals line up correctly throughout, not just at the start. + ## Documentation maintenance - [ ] Update the architecture's module/dependency inventory as modules change. diff --git a/package-lock.json b/package-lock.json index 2e8b0a3..a05834a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rc-racecore", - "version": "1.18.0", + "version": "1.18.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rc-racecore", - "version": "1.18.0", + "version": "1.18.1", "hasInstallScript": true, "dependencies": { "better-sqlite3": "^12.11.1", diff --git a/package.json b/package.json index 8fa835f..0ba5da8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rc-racecore", - "version": "1.18.0", + "version": "1.18.1", "private": true, "description": "RC RaceCore live timing and race control with AMMC ingest and SQLite persistence", "engines": { diff --git a/scripts/record-video-guides.mjs b/scripts/record-video-guides.mjs index c9516ca..98b91ce 100644 --- a/scripts/record-video-guides.mjs +++ b/scripts/record-video-guides.mjs @@ -1,17 +1,24 @@ -// Records short MKV screen-capture clips of RaceCore's UI for use as new-user video -// guides, driven against a real, isolated instance of the app (own temp SQLite dir, own -// headless Chrome profile) exactly like the tests/browser_*.js suite - nothing here ever -// touches a real installation's data. +// Records short, narrated MKV screen-capture clips of RaceCore's UI for use as new-user +// video guides, driven against a real, isolated instance of the app (own temp SQLite dir, +// own headless Chrome profile) exactly like the tests/browser_*.js suite - nothing here +// ever touches a real installation's data. +// +// Each clip is a list of steps ({ text, action }). For every step, the narration line's +// audio is synthesized up front (espeak-ng) so its real duration is known, the on-screen +// action runs, and the recording pauses for exactly that duration before moving to the +// next line - the actual wall-clock time each action took is recorded too, so the final +// narration track is built from real measured offsets (via ffmpeg's adelay/amix), not an +// idealized schedule that would drift out of sync with page-load/network latency. // // Usage: node scripts/record-video-guides.mjs [clip-name ...] // With no arguments, records every clip below. Pass one or more names (e.g. // "settings-tabs") to record only those. Output lands in video-guides/.mkv // (gitignored - these are sizeable binaries, not something to commit by default). // -// Requires ffmpeg on PATH (used only to encode the captured JPEG frames into H.264 MKV). +// Requires ffmpeg and espeak-ng on PATH. import { spawn, execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, rmSync, mkdirSync, writeFileSync, copyFileSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -21,6 +28,13 @@ const projectDir = dirname(dirname(fileURLToPath(import.meta.url))); const outputDir = join(projectDir, "video-guides"); const chrome = [process.env.CHROME_PATH, "/usr/bin/google-chrome", "/usr/bin/chromium", "/usr/bin/chromium-browser"].find((p) => p && existsSync(p)); if (!chrome) throw new Error("Chrome/Chromium not found. Set CHROME_PATH to the browser executable."); +try { execFileSync("espeak-ng", ["--version"], { stdio: "ignore" }); } catch { throw new Error("espeak-ng not found on PATH (used for narration audio)."); } + +const NARRATION_VOICE = "en-us"; +const NARRATION_RATE = 160; +const LEAD_MS = 600; // silence before the first line, so the opening frame settles first +const GAP_MS = 450; // silence between narration lines +const TAIL_MS = 1400; // hold on the final frame after the last line finishes const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -41,12 +55,39 @@ async function freePort() { return port; } -// A single recording session: isolated server + headless Chrome + CDP screencast capture. -// `record(ctx)` receives { evaluate, navigate, fill, click, pause } and drives the demo. -async function recordClip(name, { seedState, record }) { +function wavDurationMs(wavPath) { + const out = execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", wavPath]).toString().trim(); + return Math.round(parseFloat(out) * 1000); +} + +function synthesizeNarration(steps, narrationDir) { + return steps.map((step, index) => { + const wavPath = join(narrationDir, `line_${String(index).padStart(2, "0")}.wav`); + execFileSync("espeak-ng", ["-v", NARRATION_VOICE, "-s", String(NARRATION_RATE), "-w", wavPath, step.text]); + return { ...step, wavPath, durationMs: wavDurationMs(wavPath) }; + }); +} + +// Places each narration line at its own real measured start offset (via adelay) and mixes +// them onto one track (amix, normalize=0 since lines never overlap - normalizing would +// quietly halve/third the volume for no reason). `normalize=0` is an amix filter option +// available since ffmpeg 4.x. +function mixNarrationTrack(narrated, outputPath) { + const inputs = narrated.flatMap((line) => ["-i", line.wavPath]); + const delayed = narrated.map((line, i) => `[${i}]adelay=${Math.max(0, Math.round(line.startMs))}|${Math.max(0, Math.round(line.startMs))}[a${i}]`); + const mixInputs = narrated.map((_, i) => `[a${i}]`).join(""); + const filter = `${delayed.join(";")};${mixInputs}amix=inputs=${narrated.length}:duration=longest:normalize=0[aout]`; + execFileSync("ffmpeg", ["-y", ...inputs, "-filter_complex", filter, "-map", "[aout]", "-c:a", "pcm_s16le", outputPath], { stdio: "inherit" }); +} + +// A single recording session: isolated server + headless Chrome + CDP screencast capture, +// narrated by stepping through `steps` in order. +async function recordClip(name, { seedState, setup, steps }) { const temporary = mkdtempSync(join(tmpdir(), "racecore-recguide-")); const framesDir = join(temporary, "frames"); + const narrationDir = join(temporary, "narration"); mkdirSync(framesDir, { recursive: true }); + mkdirSync(narrationDir, { recursive: true }); const children = []; let socket; const launch = (command, args, options = {}) => { @@ -56,6 +97,9 @@ async function recordClip(name, { seedState, record }) { return child; }; try { + console.log(`[${name}] synthesizing narration (${steps.length} lines)...`); + const narrationSteps = synthesizeNarration(steps, narrationDir); + const port = await freePort(); const debugPort = await freePort(); const base = `http://127.0.0.1:${port}`; @@ -87,7 +131,6 @@ async function recordClip(name, { seedState, record }) { let nextId = 0; const requests = new Map(); let frameCount = 0; - let captureStart = 0; socket.addEventListener("message", (event) => { const message = JSON.parse(event.data); if (message.id) { @@ -120,28 +163,53 @@ async function recordClip(name, { seedState, record }) { await evaluate(`(() => { const form = document.getElementById(${JSON.stringify(form)}); for (const [name,value] of Object.entries(${JSON.stringify(fields)})) { const field=form.elements.namedItem(name); if (!field) throw new Error("Missing field "+name); field.value=value; field.dispatchEvent(new Event("input", {bubbles:true})); } })()`); } const click = (selector) => evaluate(`document.querySelector(${JSON.stringify(selector)}).click()`); + const scrollTo = (selector, block = "start") => evaluate(`document.querySelector(${JSON.stringify(selector)})?.scrollIntoView({block: ${JSON.stringify(block)}})`); await send("Runtime.enable"); await send("Page.enable"); await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 900, deviceScaleFactor: 1, mobile: false }); + // Setup (navigation, opening the right event, scrolling to the starting point) happens + // before the screencast starts, so the recording opens already on the right view + // instead of capturing the page load itself. + await setup?.({ evaluate, navigate, fill, click, scrollTo, pause }); + console.log(`[${name}] recording...`); - captureStart = Date.now(); + const captureStart = Date.now(); await send("Page.startScreencast", { format: "jpeg", quality: 80, maxWidth: 1600, maxHeight: 900, everyNthFrame: 1 }); - await record({ evaluate, navigate, fill, click, pause }); + await pause(LEAD_MS); + const narrated = []; + for (const step of narrationSteps) { + const actionStart = Date.now(); + await step.action?.({ evaluate, navigate, fill, click, scrollTo, pause }); + narrated.push({ ...step, startMs: Date.now() - captureStart }); + await pause(step.durationMs); + await pause(GAP_MS); + } + await pause(TAIL_MS); await send("Page.stopScreencast"); await pause(300); const elapsedSec = (Date.now() - captureStart) / 1000; - const inputFps = Math.max(1, frameCount / elapsedSec); - console.log(`[${name}] captured ${frameCount} frames over ${elapsedSec.toFixed(1)}s (~${inputFps.toFixed(1)} fps)`); + console.log(`[${name}] captured ${frameCount} frames over ${elapsedSec.toFixed(1)}s`); mkdirSync(outputDir, { recursive: true }); - const outputPath = join(outputDir, `${name}.mkv`); + const silentVideoPath = join(temporary, "silent.mkv"); + const inputFps = Math.max(1, frameCount / elapsedSec); execFileSync("ffmpeg", [ "-y", "-framerate", inputFps.toFixed(2), "-i", join(framesDir, "frame_%06d.jpg"), - "-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", outputPath, + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", silentVideoPath, + ], { stdio: "inherit" }); + + const audioPath = join(temporary, "narration.wav"); + mixNarrationTrack(narrated, audioPath); + + const outputPath = join(outputDir, `${name}.mkv`); + execFileSync("ffmpeg", [ + "-y", "-i", silentVideoPath, "-i", audioPath, + "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac", "-b:a", "96k", + outputPath, ], { stdio: "inherit" }); console.log(`[${name}] wrote ${outputPath}`); } finally { @@ -154,14 +222,16 @@ async function recordClip(name, { seedState, record }) { const clips = { "settings-tabs": { - async record({ navigate, click, pause }) { + async setup({ navigate }) { await navigate("settings", "settingsForm"); - await pause(1200); - for (const tab of ["audio", "appearance", "system", "connection"]) { - await click(`[data-settings-tab="${tab}"]`); - await pause(1800); - } }, + steps: [ + { text: "This is the Settings page, split into four tabs: Connection, Audio, Appearance, and System." }, + { text: "Audio controls race sound announcements, volume, and which events get a spoken callout.", async action({ click }) { await click('[data-settings-tab="audio"]'); } }, + { text: "Appearance and Overlay covers club branding, track layouts, and the OBS overlay setup.", async action({ click }) { await click('[data-settings-tab="appearance"]'); } }, + { text: "System and Backup handles software updates, backups, and race presets.", async action({ click }) { await click('[data-settings-tab="system"]'); } }, + { text: "And Connection, where we started, is where you link up the decoder.", async action({ click }) { await click('[data-settings-tab="connection"]'); } }, + ], }, "event-manager-tabs": { @@ -180,32 +250,41 @@ const clips = { state.events.push(raceEvent); state.sessions.push(practice, qualifying); }, - async record({ evaluate, navigate, click, pause }) { + async setup({ navigate, click, evaluate, scrollTo }) { await navigate("race_setup", "raceWizardStepForm"); await click("#event-manage-vid_event"); - // Wait for the manage tabs to exist before interacting with them. - const until = Date.now() + 15000; - while (Date.now() < until) { - if (await evaluate(`Boolean(document.querySelector(".manage-tabs"))`)) break; - await pause(100); - } - // The tabs bar sits well below the fold on the Race Setup page - scroll it into view - // first so the recording actually shows the tab-switching, not a static top-of-page - // frame the whole time. - await evaluate(`document.querySelector(".manage-tabs").scrollIntoView({block: "start"})`); - await pause(1200); - for (const tab of ["format", "live", "schedule", "setup"]) { - await click(`[data-manage-tab="${tab}"]`); - await evaluate(`document.querySelector('[data-manage-panel="${tab}"]').scrollIntoView({block: "start"})`); - await pause(1800); - } - await evaluate(`document.getElementById("session-grid-vid_quali").scrollIntoView({block: "center"})`); - await pause(1200); - await click("#session-grid-vid_quali"); - await pause(400); - await evaluate(`document.querySelector('[data-manage-panel="live"]').scrollIntoView({block: "start"})`); - await pause(1800); + await waitFor(async () => Boolean(await evaluate(`document.querySelector(".manage-tabs")`)), "manage tabs rendered"); + await scrollTo(".manage-tabs"); }, + steps: [ + { text: "Opening Manage on a race event splits the page into four tabs: Setup, Format, Live, and Schedule." }, + { + text: "Format is where you configure practice, qualifying, and finals, and generate the session plan.", + async action({ click, scrollTo }) { await click('[data-manage-tab="format"]'); await scrollTo('[data-manage-panel="format"]'); }, + }, + { + text: "Live shows the grid editor and every standings table while the race is running.", + async action({ click, scrollTo }) { await click('[data-manage-tab="live"]'); await scrollTo('[data-manage-panel="live"]'); }, + }, + { + text: "Schedule lets you set a start time and see the whole day's running order.", + async action({ click, scrollTo }) { await click('[data-manage-tab="schedule"]'); await scrollTo('[data-manage-panel="schedule"]'); }, + }, + { + text: "And Setup, where we started, is for managing participants and teams.", + async action({ click, scrollTo }) { await click('[data-manage-tab="setup"]'); await scrollTo('[data-manage-panel="setup"]'); }, + }, + { + text: "The Open Grid button on any session jumps straight to its grid editor on the Live tab.", + async action({ click, scrollTo, pause: pauseFn }) { + await scrollTo("#session-grid-vid_quali", "center"); + await pauseFn(500); + await click("#session-grid-vid_quali"); + await pauseFn(300); + await scrollTo('[data-manage-panel="live"]'); + }, + }, + ], }, "event-schedule": { @@ -223,26 +302,31 @@ const clips = { state.events.push(raceEvent); state.sessions.push(practice, qualifying); }, - async record({ evaluate, navigate, fill, click, pause }) { + async setup({ navigate, click, evaluate, scrollTo }) { await navigate("race_setup", "raceWizardStepForm"); await click("#event-manage-vid_sched_event"); - const until = Date.now() + 15000; - while (Date.now() < until) { - if (await evaluate(`Boolean(document.querySelector(".manage-tabs"))`)) break; - await pause(100); - } - await evaluate(`document.querySelector(".manage-tabs").scrollIntoView({block: "start"})`); - await pause(800); - await click('[data-manage-tab="schedule"]'); - await evaluate(`document.getElementById("scheduleForm").scrollIntoView({block: "start"})`); - await pause(1500); - await fill("scheduleForm", { scheduleStartAt: "2026-09-12T12:30", scheduleGapMin: "3" }); - await pause(800); - await evaluate(`document.getElementById("scheduleForm").requestSubmit()`); - await pause(600); - await evaluate(`document.querySelector('[data-manage-panel="schedule"] table')?.scrollIntoView({block: "start"})`); - await pause(2500); + await waitFor(async () => Boolean(await evaluate(`document.querySelector(".manage-tabs")`)), "manage tabs rendered"); + await scrollTo(".manage-tabs"); }, + steps: [ + { + text: "Let's set up a time-based schedule for race day.", + async action({ click, scrollTo }) { await click('[data-manage-tab="schedule"]'); await scrollTo("#scheduleForm"); }, + }, + { + text: "Enter a start time, and the gap you want between sessions, in minutes.", + async action({ fill }) { await fill("scheduleForm", { scheduleStartAt: "2026-09-12T12:30", scheduleGapMin: "3" }); }, + }, + { + text: "Save it, and RaceCore works out every session's planned clock time automatically.", + async action({ evaluate, scrollTo, pause: pauseFn }) { + await evaluate(`document.getElementById("scheduleForm").requestSubmit()`); + await pauseFn(500); + await scrollTo('[data-manage-panel="schedule"] table'); + }, + }, + { text: "If a session starts late or runs long, everything after it shifts forward on its own." }, + ], }, };