Public/Live_RC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spoken narration to the video guide recorder

Browse Source 
The clips were silent screen captures - the user's actual ask was for someone talking
through what's happening. Each clip step now carries a narration line synthesized
offline via espeak-ng; the recording paces itself to each line's real spoken duration,
and the final audio track is built from the real measured per-line start times (adelay +
amix) so narration and on-screen action stay in sync over a 30+ second clip instead of
drifting against an idealized schedule.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
larssand
2026-09-11 17:36:42 +02:00
parent 4967299b04
commit 788319d098
5 changed files with 166 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
scripts/record-video-guides.mjs
View File

@@ -1,17 +1,24 @@
// Records short MKV screen-capture clips of RaceCore's UI for use as new-user video
// guides, driven against a real, isolated instance of the app (own temp SQLite dir, own
// headless Chrome profile) exactly like the tests/browser_*.js suite - nothing here ever
// touches a real installation's data.
// Records short, narrated MKV screen-capture clips of RaceCore's UI for use as new-user
// video guides, driven against a real, isolated instance of the app (own temp SQLite dir,
// own headless Chrome profile) exactly like the tests/browser_*.js suite - nothing here
// ever touches a real installation's data.
//
// Each clip is a list of steps ({ text, action }). For every step, the narration line's
// audio is synthesized up front (espeak-ng) so its real duration is known, the on-screen
// action runs, and the recording pauses for exactly that duration before moving to the
// next line - the actual wall-clock time each action took is recorded too, so the final
// narration track is built from real measured offsets (via ffmpeg's adelay/amix), not an
// idealized schedule that would drift out of sync with page-load/network latency.
//
// Usage: node scripts/record-video-guides.mjs [clip-name ...]
// With no arguments, records every clip below. Pass one or more names (e.g.
// "settings-tabs") to record only those. Output lands in video-guides/<name>.mkv
// (gitignored - these are sizeable binaries, not something to commit by default).
//
// Requires ffmpeg on PATH (used only to encode the captured JPEG frames into H.264 MKV).
// Requires ffmpeg and espeak-ng on PATH.
import { spawn, execFileSync } from "node:child_process";
import { existsSync, mkdtempSync, rmSync, mkdirSync, writeFileSync, copyFileSync } from "node:fs";
import { existsSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
@@ -21,6 +28,13 @@ const projectDir = dirname(dirname(fileURLToPath(import.meta.url)));
const outputDir = join(projectDir, "video-guides");
const chrome = [process.env.CHROME_PATH, "/usr/bin/google-chrome", "/usr/bin/chromium", "/usr/bin/chromium-browser"].find((p) => p && existsSync(p));
if (!chrome) throw new Error("Chrome/Chromium not found. Set CHROME_PATH to the browser executable.");
try { execFileSync("espeak-ng", ["--version"], { stdio: "ignore" }); } catch { throw new Error("espeak-ng not found on PATH (used for narration audio)."); }
const NARRATION_VOICE = "en-us";
const NARRATION_RATE = 160;
const LEAD_MS = 600; // silence before the first line, so the opening frame settles first
const GAP_MS = 450; // silence between narration lines
const TAIL_MS = 1400; // hold on the final frame after the last line finishes
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -41,12 +55,39 @@ async function freePort() {
return port;
}
// A single recording session: isolated server + headless Chrome + CDP screencast capture.
// `record(ctx)` receives { evaluate, navigate, fill, click, pause } and drives the demo.
async function recordClip(name, { seedState, record }) {
function wavDurationMs(wavPath) {
const out = execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", wavPath]).toString().trim();
return Math.round(parseFloat(out) * 1000);
}
function synthesizeNarration(steps, narrationDir) {
return steps.map((step, index) => {
const wavPath = join(narrationDir, `line_${String(index).padStart(2, "0")}.wav`);
execFileSync("espeak-ng", ["-v", NARRATION_VOICE, "-s", String(NARRATION_RATE), "-w", wavPath, step.text]);
return { ...step, wavPath, durationMs: wavDurationMs(wavPath) };
});
}
// Places each narration line at its own real measured start offset (via adelay) and mixes
// them onto one track (amix, normalize=0 since lines never overlap - normalizing would
// quietly halve/third the volume for no reason). `normalize=0` is an amix filter option
// available since ffmpeg 4.x.
function mixNarrationTrack(narrated, outputPath) {
const inputs = narrated.flatMap((line) => ["-i", line.wavPath]);
const delayed = narrated.map((line, i) => `[${i}]adelay=${Math.max(0, Math.round(line.startMs))}|${Math.max(0, Math.round(line.startMs))}[a${i}]`);
const mixInputs = narrated.map((_, i) => `[a${i}]`).join("");
const filter = `${delayed.join(";")};${mixInputs}amix=inputs=${narrated.length}:duration=longest:normalize=0[aout]`;
execFileSync("ffmpeg", ["-y", ...inputs, "-filter_complex", filter, "-map", "[aout]", "-c:a", "pcm_s16le", outputPath], { stdio: "inherit" });
}
// A single recording session: isolated server + headless Chrome + CDP screencast capture,
// narrated by stepping through `steps` in order.
async function recordClip(name, { seedState, setup, steps }) {
const temporary = mkdtempSync(join(tmpdir(), "racecore-recguide-"));
const framesDir = join(temporary, "frames");
const narrationDir = join(temporary, "narration");
mkdirSync(framesDir, { recursive: true });
mkdirSync(narrationDir, { recursive: true });
const children = [];
let socket;
const launch = (command, args, options = {}) => {
@@ -56,6 +97,9 @@ async function recordClip(name, { seedState, record }) {
return child;
};
try {
console.log(`[${name}] synthesizing narration (${steps.length} lines)...`);
const narrationSteps = synthesizeNarration(steps, narrationDir);
const port = await freePort();
const debugPort = await freePort();
const base = `http://127.0.0.1:${port}`;
@@ -87,7 +131,6 @@ async function recordClip(name, { seedState, record }) {
let nextId = 0;
const requests = new Map();
let frameCount = 0;
let captureStart = 0;
socket.addEventListener("message", (event) => {
const message = JSON.parse(event.data);
if (message.id) {
@@ -120,28 +163,53 @@ async function recordClip(name, { seedState, record }) {
await evaluate(`(() => { const form = document.getElementById(${JSON.stringify(form)}); for (const [name,value] of Object.entries(${JSON.stringify(fields)})) { const field=form.elements.namedItem(name); if (!field) throw new Error("Missing field "+name); field.value=value; field.dispatchEvent(new Event("input", {bubbles:true})); } })()`);
}
const click = (selector) => evaluate(`document.querySelector(${JSON.stringify(selector)}).click()`);
const scrollTo = (selector, block = "start") => evaluate(`document.querySelector(${JSON.stringify(selector)})?.scrollIntoView({block: ${JSON.stringify(block)}})`);
await send("Runtime.enable");
await send("Page.enable");
await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 900, deviceScaleFactor: 1, mobile: false });
// Setup (navigation, opening the right event, scrolling to the starting point) happens
// before the screencast starts, so the recording opens already on the right view
// instead of capturing the page load itself.
await setup?.({ evaluate, navigate, fill, click, scrollTo, pause });
console.log(`[${name}] recording...`);
captureStart = Date.now();
const captureStart = Date.now();
await send("Page.startScreencast", { format: "jpeg", quality: 80, maxWidth: 1600, maxHeight: 900, everyNthFrame: 1 });
await record({ evaluate, navigate, fill, click, pause });
await pause(LEAD_MS);
const narrated = [];
for (const step of narrationSteps) {
const actionStart = Date.now();
await step.action?.({ evaluate, navigate, fill, click, scrollTo, pause });
narrated.push({ ...step, startMs: Date.now() - captureStart });
await pause(step.durationMs);
await pause(GAP_MS);
}
await pause(TAIL_MS);
await send("Page.stopScreencast");
await pause(300);
const elapsedSec = (Date.now() - captureStart) / 1000;
const inputFps = Math.max(1, frameCount / elapsedSec);
console.log(`[${name}] captured ${frameCount} frames over ${elapsedSec.toFixed(1)}s (~${inputFps.toFixed(1)} fps)`);
console.log(`[${name}] captured ${frameCount} frames over ${elapsedSec.toFixed(1)}s`);
mkdirSync(outputDir, { recursive: true });
const outputPath = join(outputDir, `${name}.mkv`);
const silentVideoPath = join(temporary, "silent.mkv");
const inputFps = Math.max(1, frameCount / elapsedSec);
execFileSync("ffmpeg", [
"-y", "-framerate", inputFps.toFixed(2), "-i", join(framesDir, "frame_%06d.jpg"),
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", outputPath,
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", "30", silentVideoPath,
], { stdio: "inherit" });
const audioPath = join(temporary, "narration.wav");
mixNarrationTrack(narrated, audioPath);
const outputPath = join(outputDir, `${name}.mkv`);
execFileSync("ffmpeg", [
"-y", "-i", silentVideoPath, "-i", audioPath,
"-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac", "-b:a", "96k",
outputPath,
], { stdio: "inherit" });
console.log(`[${name}] wrote ${outputPath}`);
} finally {
@@ -154,14 +222,16 @@ async function recordClip(name, { seedState, record }) {
const clips = {
"settings-tabs": {
async record({ navigate, click, pause }) {
async setup({ navigate }) {
await navigate("settings", "settingsForm");
await pause(1200);
for (const tab of ["audio", "appearance", "system", "connection"]) {
await click(`[data-settings-tab="${tab}"]`);
await pause(1800);
}
},
steps: [
{ text: "This is the Settings page, split into four tabs: Connection, Audio, Appearance, and System." },
{ text: "Audio controls race sound announcements, volume, and which events get a spoken callout.", async action({ click }) { await click('[data-settings-tab="audio"]'); } },
{ text: "Appearance and Overlay covers club branding, track layouts, and the OBS overlay setup.", async action({ click }) { await click('[data-settings-tab="appearance"]'); } },
{ text: "System and Backup handles software updates, backups, and race presets.", async action({ click }) { await click('[data-settings-tab="system"]'); } },
{ text: "And Connection, where we started, is where you link up the decoder.", async action({ click }) { await click('[data-settings-tab="connection"]'); } },
],
},
"event-manager-tabs": {
@@ -180,32 +250,41 @@ const clips = {
state.events.push(raceEvent);
state.sessions.push(practice, qualifying);
},
async record({ evaluate, navigate, click, pause }) {
async setup({ navigate, click, evaluate, scrollTo }) {
await navigate("race_setup", "raceWizardStepForm");
await click("#event-manage-vid_event");
// Wait for the manage tabs to exist before interacting with them.
const until = Date.now() + 15000;
while (Date.now() < until) {
if (await evaluate(`Boolean(document.querySelector(".manage-tabs"))`)) break;
await pause(100);
}
// The tabs bar sits well below the fold on the Race Setup page - scroll it into view
// first so the recording actually shows the tab-switching, not a static top-of-page
// frame the whole time.
await evaluate(`document.querySelector(".manage-tabs").scrollIntoView({block: "start"})`);
await pause(1200);
for (const tab of ["format", "live", "schedule", "setup"]) {
await click(`[data-manage-tab="${tab}"]`);
await evaluate(`document.querySelector('[data-manage-panel="${tab}"]').scrollIntoView({block: "start"})`);
await pause(1800);
}
await evaluate(`document.getElementById("session-grid-vid_quali").scrollIntoView({block: "center"})`);
await pause(1200);
await click("#session-grid-vid_quali");
await pause(400);
await evaluate(`document.querySelector('[data-manage-panel="live"]').scrollIntoView({block: "start"})`);
await pause(1800);
await waitFor(async () => Boolean(await evaluate(`document.querySelector(".manage-tabs")`)), "manage tabs rendered");
await scrollTo(".manage-tabs");
},
steps: [
{ text: "Opening Manage on a race event splits the page into four tabs: Setup, Format, Live, and Schedule." },
{
text: "Format is where you configure practice, qualifying, and finals, and generate the session plan.",
async action({ click, scrollTo }) { await click('[data-manage-tab="format"]'); await scrollTo('[data-manage-panel="format"]'); },
},
{
text: "Live shows the grid editor and every standings table while the race is running.",
async action({ click, scrollTo }) { await click('[data-manage-tab="live"]'); await scrollTo('[data-manage-panel="live"]'); },
},
{
text: "Schedule lets you set a start time and see the whole day's running order.",
async action({ click, scrollTo }) { await click('[data-manage-tab="schedule"]'); await scrollTo('[data-manage-panel="schedule"]'); },
},
{
text: "And Setup, where we started, is for managing participants and teams.",
async action({ click, scrollTo }) { await click('[data-manage-tab="setup"]'); await scrollTo('[data-manage-panel="setup"]'); },
},
{
text: "The Open Grid button on any session jumps straight to its grid editor on the Live tab.",
async action({ click, scrollTo, pause: pauseFn }) {
await scrollTo("#session-grid-vid_quali", "center");
await pauseFn(500);
await click("#session-grid-vid_quali");
await pauseFn(300);
await scrollTo('[data-manage-panel="live"]');
},
},
],
},
"event-schedule": {
@@ -223,26 +302,31 @@ const clips = {
state.events.push(raceEvent);
state.sessions.push(practice, qualifying);
},
async record({ evaluate, navigate, fill, click, pause }) {
async setup({ navigate, click, evaluate, scrollTo }) {
await navigate("race_setup", "raceWizardStepForm");
await click("#event-manage-vid_sched_event");
const until = Date.now() + 15000;
while (Date.now() < until) {
if (await evaluate(`Boolean(document.querySelector(".manage-tabs"))`)) break;
await pause(100);
}
await evaluate(`document.querySelector(".manage-tabs").scrollIntoView({block: "start"})`);
await pause(800);
await click('[data-manage-tab="schedule"]');
await evaluate(`document.getElementById("scheduleForm").scrollIntoView({block: "start"})`);
await pause(1500);
await fill("scheduleForm", { scheduleStartAt: "2026-09-12T12:30", scheduleGapMin: "3" });
await pause(800);
await evaluate(`document.getElementById("scheduleForm").requestSubmit()`);
await pause(600);
await evaluate(`document.querySelector('[data-manage-panel="schedule"] table')?.scrollIntoView({block: "start"})`);
await pause(2500);
await waitFor(async () => Boolean(await evaluate(`document.querySelector(".manage-tabs")`)), "manage tabs rendered");
await scrollTo(".manage-tabs");
},
steps: [
{
text: "Let's set up a time-based schedule for race day.",
async action({ click, scrollTo }) { await click('[data-manage-tab="schedule"]'); await scrollTo("#scheduleForm"); },
},
{
text: "Enter a start time, and the gap you want between sessions, in minutes.",
async action({ fill }) { await fill("scheduleForm", { scheduleStartAt: "2026-09-12T12:30", scheduleGapMin: "3" }); },
},
{
text: "Save it, and RaceCore works out every session's planned clock time automatically.",
async action({ evaluate, scrollTo, pause: pauseFn }) {
await evaluate(`document.getElementById("scheduleForm").requestSubmit()`);
await pauseFn(500);
await scrollTo('[data-manage-panel="schedule"] table');
},
},
{ text: "If a session starts late or runs long, everything after it shifts forward on its own." },
],
},
};