test(movie): port regression fixtures to Python

tests/proving-it-works-with-a-movie/README.md

@@ -0,0 +1,42 @@
+# Portable movie regressions
+
+Run an implemented suite from the repository root:
+
+```sh
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite assembly
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite checker
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite narration
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite all
+```
+
+By default, unavailable external capabilities are reported as skips. Add
+`--require-capabilities` when the selected environment is required to provide
+them; any skip then makes the run fail. The names `paths`, `browser`,
+`subtitles`, `processes`, `shells`, `terminal`, and `routes` are reserved for
+later tasks and currently fail explicitly instead of reporting empty success.
+
+The assembly sine wave is only a synthetic timing fixture. The narration drift
+inputs exercise text comparison only. Neither is speech/ASR acceptance.
+
+## Imported assertion equivalence
+
+| Bash suite | Imported assertion | Python test |
+| --- | --- | --- |
+| `test-assemble.sh` | `assemble runs` | `AssemblyRegression.test_narration_padding_and_offsets` |
+| `test-assemble.sh` | `segment = max(narration, visuals)` (8s +/- 0.4s) | `AssemblyRegression.test_narration_padding_and_offsets` |
+| `test-assemble.sh` | `offsets.json places the narrated scene` (2s +/- 0.3s) | `AssemblyRegression.test_narration_padding_and_offsets` |
+| `test-assemble.sh` | `cues start at the scene's real offset, not zero` | `AssemblyRegression.test_narration_padding_and_offsets` |
+| `test-check-movie.sh` | `front-loaded action is rejected` | `CheckerRegression.test_front_loaded_action_is_rejected` |
+| `test-check-movie.sh` | `paced + subtitles is accepted` | `CheckerRegression.test_paced_with_subtitles_is_accepted` |
+| `test-check-movie.sh` | `narrated without subtitles rejected` | `CheckerRegression.test_narrated_without_subtitles_is_rejected` |
+| `test-check-movie.sh` | `subtitles that stop early rejected` | `CheckerRegression.test_subtitles_that_stop_early_are_rejected` |
+| `test-check-movie.sh` | `subtitle check is opt-outable` | `CheckerRegression.test_subtitle_check_is_opt_outable` |
+| `test-check-movie.sh` | `a still with audio is rejected` | `CheckerRegression.test_still_with_audio_is_rejected` |
+| `test-check-movie.sh` | `missing narration is rejected` | `CheckerRegression.test_missing_narration_is_rejected` |
+| `test-check-movie.sh` | `silent movie passes when unnarrated` | `CheckerRegression.test_silent_movie_passes_when_unnarrated` |
+| `test-check-movie.sh` | `a contact sheet is always written` | `CheckerRegression.test_contact_sheet_is_always_written` |
+| `test-narrate.sh` | `mispronounced jargon passes` | `NarrationDriftRegression.test_mispronounced_jargon_passes` |
+| `test-narrate.sh` | `exact transcript passes` | `NarrationDriftRegression.test_exact_transcript_passes` |
+| `test-narrate.sh` | `a dropped clause fails` | `NarrationDriftRegression.test_dropped_clause_fails` |
+| `test-narrate.sh` | `an invented preamble fails` | `NarrationDriftRegression.test_invented_preamble_fails` |
+| `test-narrate.sh` | `an empty clip fails` | `NarrationDriftRegression.test_empty_clip_fails` |

tests/proving-it-works-with-a-movie/fixtures.py

@@ -0,0 +1,272 @@
+"""Portable fixtures for the imported movie regression suites."""
+
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+
+TIMEOUT_SECONDS = 900
+
+
+def missing_executables(*names: str) -> list[str]:
+    """Return executable names that cannot be resolved on PATH."""
+    return [name for name in names if shutil.which(name) is None]
+
+
+def output_text(result: subprocess.CompletedProcess[bytes]) -> str:
+    """Decode a captured command's combined output as UTF-8 evidence."""
+    return (result.stdout + result.stderr).decode("utf-8", errors="replace")
+
+
+def run_tool(
+    name: str,
+    args: list[str],
+    *,
+    cwd: Path,
+    env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[bytes]:
+    """Invoke one extensionless movie tool through its PEP 723 environment."""
+    uv = shutil.which("uv")
+    if uv is None:
+        raise RuntimeError("uv is required for movie tool tests")
+    script = (
+        Path(__file__).resolve().parents[2]
+        / "skills/proving-it-works-with-a-movie/scripts"
+        / name
+    )
+    return subprocess.run(
+        [uv, "run", "--script", str(script), *args],
+        cwd=cwd,
+        env=env,
+        capture_output=True,
+        timeout=TIMEOUT_SECONDS,
+    )
+
+
+def duration(path: Path) -> float:
+    """Measure a media file's container duration with ffprobe."""
+    ffprobe = shutil.which("ffprobe")
+    if ffprobe is None:
+        raise RuntimeError("ffprobe is required for movie tool tests")
+    result = subprocess.run(
+        [
+            ffprobe,
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "csv=p=0",
+            str(path),
+        ],
+        capture_output=True,
+        timeout=TIMEOUT_SECONDS,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ffprobe failed for {path}: {output_text(result)}")
+    return float(result.stdout.decode("utf-8").strip())
+
+
+def _run_ffmpeg(args: list[str], *, cwd: Path) -> None:
+    ffmpeg = shutil.which("ffmpeg")
+    if ffmpeg is None:
+        raise RuntimeError("ffmpeg is required for movie tool tests")
+    result = subprocess.run(
+        [ffmpeg, "-nostdin", "-y", "-v", "error", *args],
+        cwd=cwd,
+        capture_output=True,
+        timeout=TIMEOUT_SECONDS,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ffmpeg fixture generation failed: {output_text(result)}")
+
+
+def assembly_fixture(work: Path) -> Path:
+    """Write the imported image/frames/synthetic-timing-audio assembly case."""
+    shots = work / "shots"
+    shots.mkdir(parents=True)
+    for index in range(1, 5):
+        _run_ffmpeg(
+            [
+                "-f",
+                "lavfi",
+                "-i",
+                f"color=c=0x{index}0{index}0{index}0:size=320x180:d=0.1",
+                "-frames:v",
+                "1",
+                str(shots / f"s0{index}.png"),
+            ],
+            cwd=work,
+        )
+
+    narration = work / "narration"
+    narration.mkdir()
+    wav = narration / "body.wav"
+    _run_ffmpeg(
+        [
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=300:duration=6",
+            str(wav),
+        ],
+        cwd=work,
+    )
+    manifest = [
+        {
+            "id": "body",
+            "text": "one two three four five six seven eight nine ten",
+            "wav": "body.wav",
+            "duration": duration(wav),
+        }
+    ]
+    (narration / "manifest.json").write_text(
+        json.dumps(manifest), encoding="utf-8"
+    )
+
+    scenes = work / "scenes.yaml"
+    scenes.write_text(
+        """resolution: { width: 640, height: 360 }
+fps: 30
+scenes:
+  - id: opener
+    kind: image
+    src: shots/s01.png
+    duration: 2
+  - id: body
+    kind: frames
+    src: shots
+    rate: 1.0
+""",
+        encoding="utf-8",
+    )
+    return scenes
+
+
+def checker_fixture(work: Path) -> dict[str, Path]:
+    """Create the four imported checker movies and their subtitle sidecars."""
+    front_loaded = work / "front-loaded.mp4"
+    _run_ffmpeg(
+        [
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=320x240:rate=10:d=2",
+            "-f",
+            "lavfi",
+            "-i",
+            "color=c=navy:size=320x240:rate=10:d=20",
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=300:duration=22",
+            "-filter_complex",
+            "[0:v][1:v]concat=n=2:v=1:a=0[v]",
+            "-map",
+            "[v]",
+            "-map",
+            "2:a",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-shortest",
+            str(front_loaded),
+        ],
+        cwd=work,
+    )
+
+    paced = work / "paced.mp4"
+    _run_ffmpeg(
+        [
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=320x240:rate=10:d=22",
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=300:duration=22",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-shortest",
+            str(paced),
+        ],
+        cwd=work,
+    )
+
+    still = work / "still.mp4"
+    _run_ffmpeg(
+        [
+            "-f",
+            "lavfi",
+            "-i",
+            "color=c=navy:size=320x240:rate=10:d=12",
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=300:duration=12",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-shortest",
+            str(still),
+        ],
+        cwd=work,
+    )
+
+    silent = work / "silent.mp4"
+    _run_ffmpeg(
+        [
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=320x240:rate=10:d=12",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            str(silent),
+        ],
+        cwd=work,
+    )
+
+    subtitles = """1
+00:00:00,000 --> 00:00:07,000
+A narrated movie needs subtitles:
+plenty of people watch muted.
+
+2
+00:00:07,000 --> 00:00:14,000
+The checker treats their absence
+as a defect, not a nicety.
+
+3
+00:00:14,000 --> 00:00:21,500
+And it notices when they stop
+before the narration does.
+"""
+    (work / "paced.srt").write_text(subtitles, encoding="utf-8")
+    (work / "short.srt").write_text(
+        "\n".join(subtitles.splitlines()[:8]) + "\n", encoding="utf-8"
+    )
+    short = work / "short.mp4"
+    shutil.copyfile(paced, short)
+
+    return {
+        "front-loaded": front_loaded,
+        "paced": paced,
+        "still": still,
+        "silent": silent,
+        "short": short,
+    }

tests/proving-it-works-with-a-movie/run-tests.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["pyyaml", "pillow", "websockets"]
+# ///
+"""Run the proving-it-works-with-a-movie regression suites portably."""
+
+import argparse
+import sys
+import unittest
+from pathlib import Path
+
+
+IMPLEMENTED_SUITES = {
+    "assembly": "test_assembly.py",
+    "checker": "test_checker.py",
+    "narration": "test_narration.py",
+}
+RESERVED_SUITES = {
+    "paths",
+    "browser",
+    "subtitles",
+    "processes",
+    "shells",
+    "terminal",
+    "routes",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--suite",
+        required=True,
+        choices=[*IMPLEMENTED_SUITES, *sorted(RESERVED_SUITES), "all"],
+    )
+    parser.add_argument("--require-capabilities", action="store_true")
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if args.suite in RESERVED_SUITES:
+        print(
+            f"suite {args.suite!r} is reserved but not implemented",
+            file=sys.stderr,
+        )
+        return 2
+
+    patterns = (
+        list(IMPLEMENTED_SUITES.values())
+        if args.suite == "all"
+        else [IMPLEMENTED_SUITES[args.suite]]
+    )
+    test_directory = Path(__file__).resolve().parent
+    loader = unittest.TestLoader()
+    suite = unittest.TestSuite(
+        loader.discover(
+            str(test_directory),
+            pattern=pattern,
+            top_level_dir=str(test_directory),
+        )
+        for pattern in patterns
+    )
+    result = unittest.TextTestRunner(verbosity=2).run(suite)
+    if args.require_capabilities and result.skipped:
+        print(
+            f"required capabilities unavailable: {len(result.skipped)} "
+            "selected test(s) skipped",
+            file=sys.stderr,
+        )
+        return 1
+    return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/proving-it-works-with-a-movie/test_assembly.py

@@ -0,0 +1,50 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import fixtures
+
+
+class AssemblyRegression(unittest.TestCase):
+    def test_narration_padding_and_offsets(self):
+        missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+        if missing:
+            self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+
+        with tempfile.TemporaryDirectory() as directory:
+            work = Path(directory)
+            scenes = fixtures.assembly_fixture(work)
+            result = fixtures.run_tool(
+                "assemble", [str(scenes), str(work / "out.mp4")], cwd=work
+            )
+            self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+            self.assertAlmostEqual(
+                fixtures.duration(work / "out.mp4"), 8, delta=0.4
+            )
+            offsets = json.loads(
+                (work / "segments/offsets.json").read_text(encoding="utf-8")
+            )
+            self.assertAlmostEqual(offsets["body"], 2, delta=0.3)
+
+            subtitles = fixtures.run_tool(
+                "make-subtitles",
+                [
+                    str(work / "narration/manifest.json"),
+                    str(work / "out.srt"),
+                    "--offsets-json",
+                    str(work / "segments/offsets.json"),
+                ],
+                cwd=work,
+            )
+            self.assertEqual(
+                subtitles.returncode, 0, fixtures.output_text(subtitles)
+            )
+            self.assertRegex(
+                (work / "out.srt").read_text(encoding="utf-8"),
+                r"00:00:0[2-9]",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/proving-it-works-with-a-movie/test_checker.py

@@ -0,0 +1,95 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from PIL import Image
+
+import fixtures
+
+
+class CheckerRegression(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+        if missing:
+            raise unittest.SkipTest(
+                f"required executable(s) not on PATH: {', '.join(missing)}"
+            )
+        cls._temporary_directory = tempfile.TemporaryDirectory()
+        cls.addClassCleanup(cls._temporary_directory.cleanup)
+        cls.work = Path(cls._temporary_directory.name)
+        cls.movies = fixtures.checker_fixture(cls.work)
+
+    def check(
+        self, expected_exit: int, needle: str, movie: Path, *extra_args: str
+    ) -> Path:
+        output_directory = self.work / f"{movie.stem}-check"
+        result = fixtures.run_tool(
+            "check-movie",
+            [str(movie), "--out", str(output_directory), *extra_args],
+            cwd=self.work,
+        )
+        output = fixtures.output_text(result)
+        self.assertEqual(result.returncode, expected_exit, output)
+        self.assertIn(needle.casefold(), output.casefold())
+        return output_directory
+
+    def test_front_loaded_action_is_rejected(self):
+        self.check(
+            1,
+            "every visible change happens in the first",
+            self.movies["front-loaded"],
+        )
+
+    def test_paced_with_subtitles_is_accepted(self):
+        self.check(0, "Mechanical checks pass", self.movies["paced"])
+
+    def test_narrated_without_subtitles_is_rejected(self):
+        self.check(
+            1,
+            "no subtitles",
+            self.movies["paced"],
+            "--subs",
+            str(self.work / "nope.srt"),
+        )
+
+    def test_subtitles_that_stop_early_are_rejected(self):
+        self.check(1, "subtitles stop at", self.movies["short"])
+
+    def test_subtitle_check_is_opt_outable(self):
+        self.check(
+            0,
+            "Mechanical checks pass",
+            self.movies["paced"],
+            "--subs",
+            str(self.work / "nope.srt"),
+            "--no-expect-subtitles",
+        )
+
+    def test_still_with_audio_is_rejected(self):
+        self.check(1, "never reaches a new state", self.movies["still"])
+
+    def test_missing_narration_is_rejected(self):
+        self.check(1, "no audio stream", self.movies["silent"])
+
+    def test_silent_movie_passes_when_unnarrated(self):
+        self.check(
+            0,
+            "Mechanical checks pass",
+            self.movies["silent"],
+            "--no-expect-audio",
+        )
+
+    def test_contact_sheet_is_always_written(self):
+        output_directory = self.check(
+            0, "contact-sheet.png", self.movies["paced"]
+        )
+        sheet = output_directory / "contact-sheet.png"
+        self.assertTrue(sheet.is_file(), f"missing contact sheet: {sheet}")
+        with Image.open(sheet) as image:
+            image.load()
+            self.assertEqual(image.format, "PNG")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/proving-it-works-with-a-movie/test_narration.py

@@ -0,0 +1,61 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+import fixtures
+
+
+SCRIPT = (
+    "This is smevals studio. Every eval on the shelf is a folder of tasks and "
+    "graders."
+)
+
+
+class NarrationDriftRegression(unittest.TestCase):
+    def drift(self, expected_exit: int, heard: str) -> None:
+        missing = fixtures.missing_executables("uv")
+        if missing:
+            self.skipTest(
+                f"required executable(s) not on PATH: {', '.join(missing)}"
+            )
+        with tempfile.TemporaryDirectory() as directory:
+            work = Path(directory)
+            script_path = work / "script.txt"
+            heard_path = work / "heard.txt"
+            script_path.write_text(SCRIPT, encoding="utf-8")
+            heard_path.write_text(heard, encoding="utf-8")
+            result = fixtures.run_tool(
+                "narrate",
+                ["--drift-check", str(script_path), str(heard_path)],
+                cwd=work,
+            )
+            self.assertEqual(
+                result.returncode, expected_exit, fixtures.output_text(result)
+            )
+
+    def test_mispronounced_jargon_passes(self):
+        self.drift(
+            0,
+            "This is Mevil studio. Every Yvel on the shelf is a folder of tasks "
+            "and graders.",
+        )
+
+    def test_exact_transcript_passes(self):
+        self.drift(0, SCRIPT)
+
+    def test_dropped_clause_fails(self):
+        self.drift(1, "This is smevals studio.")
+
+    def test_invented_preamble_fails(self):
+        self.drift(
+            1,
+            "Sure, here it is, happy to help with that. This is smevals studio. "
+            "Every eval on the shelf is a folder of tasks and graders.",
+        )
+
+    def test_empty_clip_fails(self):
+        self.drift(1, "you")
+
+
+if __name__ == "__main__":
+    unittest.main()