From e90ef44c321ea8fca0a1bcbb873b629c73b7e461 Mon Sep 17 00:00:00 2001 From: bradautomates Date: Sat, 12 Sep 2026 22:09:43 +1000 Subject: [PATCH] Implement v0.3.0 reliability fixes and managed WhisperX transcription Add opt-in local transcription with managed installation, cache warm-up, and a one-time backend choice while preserving native-caption priority. Correct frame timing and coverage, caption provenance and download ownership, configuration parsing, provider pairing, and partial reports. Update cross-host instructions, packaging, and test-gated release CI. Validate 198 isolated tests, real local English and Spanish inference, network-denied warm-cache inference, live captions/media, and packaging. Record platform and account-dependent acceptance limits separately. --- .claude-plugin/plugin.json | 19 +- .codex-plugin/plugin.json | 11 +- .gitattributes | 6 +- .github/workflows/release.yml | 6 + .github/workflows/tests.yml | 49 +++ AGENTS.md | 4 +- CHANGELOG.md | 24 +- README.md | 386 +++++++----------- hooks/scripts/check-setup.sh | 70 +--- skills/watch/.gitattributes | 6 + skills/watch/SKILL.md | 290 ++++---------- skills/watch/scripts/config.py | 176 ++++++--- skills/watch/scripts/download.py | 322 ++++++++------- skills/watch/scripts/frames.py | 321 +++++++-------- skills/watch/scripts/local_whisperx.py | 147 +++++++ skills/watch/scripts/runtime.py | 35 ++ skills/watch/scripts/setup.py | 524 +++++++++++-------------- skills/watch/scripts/transcribe.py | 163 ++++---- skills/watch/scripts/watch.py | 339 +++++++--------- skills/watch/scripts/whisper.py | 163 +++----- tests/conftest.py | 30 ++ tests/test_config.py | 41 +- tests/test_dedup.py | 13 +- tests/test_download.py | 175 ++++++--- tests/test_frames.py | 71 ++++ tests/test_hook.py | 37 ++ tests/test_local_whisperx.py | 226 +++++++++++ tests/test_packaging.py | 28 ++ tests/test_release_regressions.py | 57 +++ tests/test_runtime.py | 82 ++++ tests/test_setup.py | 153 +++++--- tests/test_transcribe.py | 40 ++ tests/test_watch.py | 107 ++++- tests/test_whisper.py | 59 +++ 34 files changed, 2531 insertions(+), 1649 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 skills/watch/.gitattributes create mode 100644 skills/watch/scripts/local_whisperx.py create mode 100644 skills/watch/scripts/runtime.py create mode 100644 tests/test_hook.py create mode 100644 tests/test_local_whisperx.py create mode 100644 tests/test_packaging.py create mode 100644 tests/test_release_regressions.py create mode 100644 tests/test_runtime.py create mode 100644 tests/test_transcribe.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 2d0c824..9109913 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,12 +1,25 @@ { "name": "watch", - "version": "0.2.0", - "description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.", + "version": "0.3.0", + "description": "Give an agent video evidence: timestamped frames and native captions, with optional local WhisperX or Groq/OpenAI transcription.", "author": { "name": "Bradley Bonanno" }, "homepage": "https://github.com/bradautomates/claude-video", "repository": "https://github.com/bradautomates/claude-video", "license": "MIT", - "keywords": ["video", "watch", "youtube", "vimeo", "tiktok", "transcription", "whisper", "yt-dlp", "ffmpeg", "multimodal", "frames"] + "keywords": [ + "video", + "watch", + "youtube", + "vimeo", + "tiktok", + "transcription", + "whisper", + "yt-dlp", + "ffmpeg", + "multimodal", + "frames", + "whisperx" + ] } diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 5c1f569..936832f 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "watch", - "version": "0.2.0", - "description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.", + "version": "0.3.0", + "description": "Give an agent video evidence: timestamped frames and native captions, with optional local WhisperX or Groq/OpenAI transcription.", "author": { "name": "Bradley Bonanno" }, @@ -19,13 +19,14 @@ "yt-dlp", "ffmpeg", "multimodal", - "frames" + "frames", + "whisperx" ], "skills": "./skills/", "interface": { "displayName": "watch", - "shortDescription": "Give Claude a video input — paste a URL or path and ask about it.", - "longDescription": "watch adds a skill that lets the agent watch any video: it downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls a timestamped transcript from native captions (or the Whisper API as a fallback), and hands frames + transcript to the model so it can answer questions grounded in what's actually on screen and in the audio.", + "shortDescription": "Give your agent a video input \u2014 paste a URL or local path and ask about it.", + "longDescription": "watch gives an agent timestamped video frames and a transcript from native captions or an optional local WhisperX / cloud Whisper fallback. The agent views the selected frames and answers from the available visual and spoken evidence.", "developerName": "Bradley Bonanno", "category": "Productivity", "capabilities": [ diff --git a/.gitattributes b/.gitattributes index 5119f8d..9144b6c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -22,10 +22,8 @@ V2_PLAN.md export-ignore V2_CONCERNS.md export-ignore .agents/ export-ignore -# Dev-only build script + scanner config inside the skill tree: keep them out of -# the claude.ai .skill bundle (built via `git archive HEAD:skills/watch`). -skills/watch/scripts/build-skill.sh export-ignore -skills/watch/.skillignore export-ignore +# Skill-local archive exclusions live in skills/watch/.gitattributes so they +# also apply when archiving HEAD:skills/watch directly. # NOTE: hooks/ and .claude-plugin/ are NOT export-ignored because Claude Code's # /plugin install fetches the full-repo git archive — they must be present for diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80db5da..cebe9dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,13 @@ permissions: contents: write jobs: + tests: + uses: ./.github/workflows/tests.yml + permissions: + contents: read + build-and-release: + needs: tests runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..064cb33 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,49 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + workflow_call: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + python: '3.10' + - os: ubuntu-latest + python: '3.12' + - os: macos-latest + python: '3.12' + - os: windows-latest + python: '3.12' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install FFmpeg (Linux, including the retained 4.4 compatibility path) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ffmpeg + - name: Install FFmpeg (macOS) + if: runner.os == 'macOS' + run: brew list ffmpeg >/dev/null 2>&1 || brew install ffmpeg + - name: Install FFmpeg (Windows) + if: runner.os == 'Windows' + run: choco install ffmpeg --no-progress -y + - name: Install test tools + run: python -m pip install pytest "yt-dlp[default]==2026.8.19" + - name: Verify executables + run: | + ffmpeg -version + ffprobe -version + yt-dlp --version + - name: Run isolated tests (no model downloads or live APIs) + run: python -m pytest -q diff --git a/AGENTS.md b/AGENTS.md index 6068f24..6edc8ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,12 @@ # claude-video / watch skill -Agent Skills package that gives an agent a video input. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, and 50+ other [Agent Skills](https://agentskills.io) hosts. Pure-stdlib Python that orchestrates `yt-dlp` + `ffmpeg` and an optional Whisper API. +Agent Skills package that gives an agent a video input. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, and 50+ other [Agent Skills](https://agentskills.io) hosts. Pure-stdlib Python that orchestrates `yt-dlp` + `ffmpeg` and an optional Whisper API or separately managed local WhisperX process. ## Structure - `skills/watch/SKILL.md` — canonical skill contract the model reads when `/watch` fires. Source of truth for behavior across every host. - `skills/watch/scripts/watch.py` — entry point; orchestrates download → frames → transcript. -- `skills/watch/scripts/{download,frames,transcribe,whisper,setup,config}.py` — yt-dlp wrapper, ffmpeg frame extraction + auto-fps, caption/Whisper transcription, preflight/installer, shared config. +- `skills/watch/scripts/{download,frames,transcribe,whisper,local_whisperx,setup,config,runtime}.py` — yt-dlp wrapper, ffmpeg frame extraction + auto-fps, caption/Whisper transcription, preflight/installer, shared config. - `skills/watch/scripts/build-skill.sh` — builds `dist/watch.skill` for claude.ai upload (dev-only). - `hooks/` — Claude Code SessionStart setup-status hook (Claude Code only). - `.claude-plugin/` — `plugin.json` + `marketplace.json` (Claude Code plugin + local marketplace). diff --git a/CHANGELOG.md b/CHANGELOG.md index 41691c8..be5c192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,28 @@ All notable changes to `/watch` are documented here. +## [0.3.0] — Unreleased + +### Added +- Managed, optional WhisperX 3.8.6 setup with uv, a separate Python 3.12 environment, cache warm-up, recoverable installation state, and local CPU transcription. Default model: small, int8, batch 8; language and timeout are configurable. Local failures never select a cloud provider. +- One-time fallback selection in the skill wizard: WhisperX, Groq, OpenAI, or captions only. Existing installations retain the `auto` cloud-key preference. +- Original-language caption selection and provenance, `--sub-lang`, and explicit cookie-file/browser options shared across downloader stages. +- Reports for unavailable modalities, no speech, and missing cloud-transcription intervals. Offline dependency diagnostics and a cross-platform CI gate for release publication. + +### Fixed +- Uniform sampling now spans the actual capped range and labels selected source frames with their real timestamps, including fractional seeks and cue frames. +- FFmpeg sync-option capability detection, empty keyframe-window fallback, RGB thumbnail deduplication, and actionable probe/launch errors. +- UTF-8 output and subprocess decoding, BOM-aware shared config parsing, provider-key pairing, atomic config updates, keyless readiness, and advisory setup hooks. +- Bounded caption downloads respect watch-owned output templates while retaining intentional yt-dlp configuration. Fresh run directories and completed final-path validation prevent stale/partial media reuse. +- WebVTT optional-hour timestamps, entities, cue/block parsing, and overlap-scoped deduplication preserve independent repeated speech. +- Full-track caption availability is checked before focus filtering; successful captions survive media/probe failures. +- Conservative 24,000,000-byte cloud upload budgets, actual chunk/multipart size checks, and explicit partial/no-speech metadata. + +### Changed +- First-success installation documentation separates media tools from optional speech backends and describes host/network limitations. +- User-provided output directories now contain a disposable run child; cleanup preserves the parent and source files. +- The base runtime remains standard-library-only; local model environments and caches live outside the self-contained skill folder. + ## [0.2.0] — 2026-06-29 ### Added @@ -9,7 +31,7 @@ All notable changes to `/watch` are documented here. - **Frame deduplication** (default on; `--no-dedup` to disable). Before the budget cap, a pass downscales each frame to a 16×16 grayscale thumbnail and drops frames whose mean per-pixel difference from the last *kept* frame is within threshold — so the budget goes to distinct content instead of held slides and static recordings. The **Frames** report line shows how many near-duplicates were dropped. - **Whisper auto-chunking.** Audio over the 25 MB upload cap is split into evenly sized chunks, transcribed per chunk, with segment timestamps shifted back into source time. Partial failures are tolerated — transcription only fails if *every* chunk fails, so length alone no longer breaks it. - **`--timestamps T1,T2,…`** — grab a frame at each absolute timestamp; reserved against the cap, and the only frames produced under `--detail transcript`. -- **`--no-whisper`** — disable transcription entirely (frames only). +- **`--no-whisper`** — disable speech fallback while retaining native captions. - pytest suite covering config, dedup, download, fixtures, frames, setup, timestamps, watch, and whisper (no network; ffmpeg-synthesized clips). ### Changed diff --git a/README.md b/README.md index 8c44cf2..70f2809 100644 --- a/README.md +++ b/README.md @@ -1,266 +1,178 @@ # /watch -**Give Claude the ability to watch any video.** +Give an agent video evidence: a URL or local file becomes timestamped frames and a transcript. Works with Claude Code and other [Agent Skills](https://agentskills.io) hosts, including Codex, Cursor, and Copilot. Native captions come first; optional local WhisperX or Groq/OpenAI transcription handles videos without captions. -Claude Code (recommended — auto-updates via marketplace): -``` +## Install and get a first result + +**Claude Code:** type these in Claude Code, then reload the host and check skill autocomplete. Depending on the host version, the plugin skill may appear as `/watch:watch`. + +```text /plugin marketplace add bradautomates/claude-video /plugin install watch@claude-video ``` -Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts: -```bash -npx skills add bradautomates/claude-video -g -``` -(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.) - -More install options (claude.ai web, manual) in the [Install](#install) section below. - -Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions. - ---- - -Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen. - -With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude fetches captions first, downloads only what it needs, extracts frames (scene-aware, or fast keyframes at `efficient` detail), pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio. - -``` -/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark? -``` - -## What people actually use it for - -**Analyze someone else's content.** `/watch https://youtu.be/ what hook did they open with?` Claude looks at the first frames, reads the opening transcript, breaks down the structure. Same for ad creative, competitor launches, podcast intros, anything where the *how* matters as much as the *what*. - -**Diagnose a bug from a video.** Someone sends you a screen recording of something broken. `/watch bug-repro.mov what's going wrong?` Claude watches the recording, finds the frame where the issue appears, describes what's on screen, often catches the cause without you ever opening the file. - -**Summarize a video.** `/watch https://youtu.be/ summarize this` does the obvious thing — pulls the structure, the key moments, what was actually said and shown. Faster than watching at 2x. - -**Cut the hype out of an update video.** `/watch https://youtu.be/ what's actually new — skip the hype` Strip a "game-changer" feature drop down to the few things that matter, so you get the substance without ten minutes of intro and overselling. - -**Turn a playlist into notes.** `/watch https://youtu.be/