mirror of
https://github.com/Luchexx/claude-video.git
synced 2026-11-02 17:29:52 +00:00
e90ef44c32
Add opt-in local transcription with managed installation, cache warm-up, and a one-time backend choice while preserving native-caption priority. Correct frame timing and coverage, caption provenance and download ownership, configuration parsing, provider pairing, and partial reports. Update cross-host instructions, packaging, and test-gated release CI. Validate 198 isolated tests, real local English and Spanish inference, network-denied warm-cache inference, live captions/media, and packaging. Record platform and account-dependent acceptance limits separately.
426 lines
16 KiB
Python
Executable File
426 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Transcribe a video via local WhisperX or the Groq/OpenAI Whisper APIs.
|
|
|
|
Strategy: extract audio (mono 16kHz mp3, tiny payload), upload to whichever
|
|
API has a key. Returns segments in the same shape as transcribe.parse_vtt so
|
|
the rest of the pipeline (filter_range, format_transcript) doesn't care where
|
|
the transcript came from.
|
|
|
|
Pure stdlib — no `pip install groq` or `pip install openai` needed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import math
|
|
import mimetypes
|
|
import shutil
|
|
import ssl
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.request import Request, urlopen
|
|
|
|
from config import load_api_key, ConfigError
|
|
from runtime import configure_stdio, run_text
|
|
from transcribe import Segments, normalize_segments
|
|
|
|
|
|
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/audio/transcriptions"
|
|
GROQ_MODEL = "whisper-large-v3"
|
|
|
|
OPENAI_ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
|
|
OPENAI_MODEL = "whisper-1"
|
|
|
|
# Both Groq's free tier and OpenAI whisper-1 cap uploads at 25 MB. We target a
|
|
# margin under that so multipart framing overhead never pushes a chunk over.
|
|
MAX_UPLOAD_BYTES = 24_000_000
|
|
MAX_MULTIPART_BYTES = 25_000_000
|
|
|
|
|
|
def plan_chunks(
|
|
total_seconds: float,
|
|
total_bytes: int,
|
|
max_bytes: int = MAX_UPLOAD_BYTES,
|
|
) -> list[tuple[float, float]]:
|
|
"""Split a duration into contiguous (offset, duration) chunks under max_bytes.
|
|
|
|
Size scales linearly with duration (constant-bitrate mono mp3), so an even
|
|
time split yields evenly-sized chunks. Returns a single full-length chunk
|
|
when the audio already fits.
|
|
"""
|
|
if not math.isfinite(total_seconds) or total_seconds < 0 or max_bytes <= 0 or total_bytes < 0:
|
|
raise SystemExit("Invalid audio chunk duration or byte budget")
|
|
if total_bytes > max_bytes and total_seconds <= 0:
|
|
raise SystemExit("Cannot split oversized audio with unknown duration")
|
|
if total_bytes <= max_bytes or total_seconds <= 0:
|
|
return [(0.0, total_seconds)]
|
|
|
|
n = math.ceil(total_bytes / max_bytes)
|
|
chunk = total_seconds / n
|
|
plan: list[tuple[float, float]] = []
|
|
for i in range(n):
|
|
offset = i * chunk
|
|
# The last chunk absorbs any rounding remainder so durations sum exactly.
|
|
duration = (total_seconds - offset) if i == n - 1 else chunk
|
|
plan.append((round(offset, 3), round(duration, 3)))
|
|
return plan
|
|
|
|
|
|
def extract_audio(video_path: str, out_path: Path) -> Path:
|
|
"""Extract mono 16kHz 64kbps mp3 — ~480 kB/min, fits any Whisper limit."""
|
|
if shutil.which("ffmpeg") is None:
|
|
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
|
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-hide_banner",
|
|
"-loglevel", "error",
|
|
"-y",
|
|
"-i", str(Path(video_path).resolve()),
|
|
"-vn",
|
|
"-acodec", "libmp3lame",
|
|
"-ar", "16000",
|
|
"-ac", "1",
|
|
"-b:a", "64k",
|
|
str(out_path.resolve()),
|
|
]
|
|
result = run_text(cmd)
|
|
if result.returncode != 0:
|
|
raise SystemExit(f"ffmpeg audio extraction failed: {result.stderr.strip()}")
|
|
if not out_path.exists() or out_path.stat().st_size == 0:
|
|
raise SystemExit("ffmpeg produced no audio — video may have no audio track")
|
|
return out_path
|
|
|
|
|
|
def audio_duration(audio_path: Path) -> float:
|
|
from frames import get_metadata
|
|
return get_metadata(str(audio_path))["duration_seconds"]
|
|
|
|
|
|
def split_audio(
|
|
full_audio: Path,
|
|
work_dir: Path,
|
|
plan: list[tuple[float, float]],
|
|
) -> list[tuple[Path, float]]:
|
|
"""Slice full_audio into per-plan chunk files, returning (path, offset) pairs.
|
|
|
|
Uses stream copy (`-c copy`) so there is no re-encode and no quality loss;
|
|
mp3 frame boundaries are close enough for transcription's purposes.
|
|
"""
|
|
if shutil.which("ffmpeg") is None:
|
|
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
|
|
|
work_dir.mkdir(parents=True, exist_ok=True)
|
|
chunks: list[tuple[Path, float]] = []
|
|
for index, (offset, duration) in enumerate(plan):
|
|
out_path = work_dir / f"chunk_{index:03d}.mp3"
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-hide_banner",
|
|
"-loglevel", "error",
|
|
"-y",
|
|
"-ss", f"{offset:.3f}",
|
|
"-i", str(full_audio.resolve()),
|
|
"-t", f"{duration:.3f}",
|
|
"-c", "copy",
|
|
str(out_path.resolve()),
|
|
]
|
|
result = run_text(cmd)
|
|
if result.returncode != 0 or not out_path.exists() or out_path.stat().st_size == 0:
|
|
raise SystemExit(
|
|
f"ffmpeg failed to split audio chunk {index + 1}: {result.stderr.strip()}"
|
|
)
|
|
chunks.append((out_path, offset))
|
|
return chunks
|
|
|
|
|
|
def _build_multipart(fields: dict[str, str], file_path: Path) -> tuple[bytes, str]:
|
|
"""Assemble a multipart/form-data body the Whisper APIs accept.
|
|
|
|
Whisper's multipart upload is small and predictable — doing it by hand
|
|
keeps us on pure stdlib instead of pulling requests/groq/openai SDKs.
|
|
"""
|
|
boundary = f"----WatchBoundary{uuid.uuid4().hex}"
|
|
eol = b"\r\n"
|
|
buf = io.BytesIO()
|
|
|
|
for name, value in fields.items():
|
|
buf.write(f"--{boundary}".encode()); buf.write(eol)
|
|
buf.write(f'Content-Disposition: form-data; name="{name}"'.encode()); buf.write(eol)
|
|
buf.write(eol)
|
|
buf.write(str(value).encode()); buf.write(eol)
|
|
|
|
mimetype = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
|
|
buf.write(f"--{boundary}".encode()); buf.write(eol)
|
|
buf.write(
|
|
f'Content-Disposition: form-data; name="file"; filename="{file_path.name}"'.encode()
|
|
)
|
|
buf.write(eol)
|
|
buf.write(f"Content-Type: {mimetype}".encode()); buf.write(eol)
|
|
buf.write(eol)
|
|
buf.write(file_path.read_bytes())
|
|
buf.write(eol)
|
|
buf.write(f"--{boundary}--".encode()); buf.write(eol)
|
|
|
|
return buf.getvalue(), boundary
|
|
|
|
|
|
MAX_ATTEMPTS = 4 # initial + 3 retries
|
|
MAX_429_RETRIES = 2
|
|
RETRY_BASE_DELAY = 2.0
|
|
|
|
|
|
def _post_whisper(endpoint: str, api_key: str, model: str, audio_path: Path) -> dict:
|
|
if audio_path.stat().st_size > MAX_UPLOAD_BYTES:
|
|
raise SystemExit("Audio file exceeds the 24,000,000-byte upload budget; split it before upload.")
|
|
fields = {
|
|
"model": model,
|
|
"response_format": "verbose_json",
|
|
"temperature": "0",
|
|
}
|
|
body, boundary = _build_multipart(fields, audio_path)
|
|
if len(body) > MAX_MULTIPART_BYTES:
|
|
raise SystemExit("Multipart request exceeds the 25,000,000-byte limit.")
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
|
# Groq sits behind Cloudflare — the default `Python-urllib/3.x` UA
|
|
# trips WAF rule 1010 (403) before auth even runs. Any non-default
|
|
# UA clears it; we identify honestly.
|
|
"User-Agent": "watch-skill/1.0 (+claude-code; python-urllib)",
|
|
}
|
|
|
|
context = ssl.create_default_context()
|
|
rate_limit_hits = 0
|
|
last_exc: Exception | None = None
|
|
last_detail = ""
|
|
|
|
for attempt in range(MAX_ATTEMPTS):
|
|
request = Request(endpoint, data=body, headers=headers, method="POST")
|
|
try:
|
|
with urlopen(request, timeout=300, context=context) as response:
|
|
payload = response.read().decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as exc:
|
|
detail = _read_error_body(exc)
|
|
last_exc, last_detail = exc, detail
|
|
|
|
# 4xx other than 429 are client errors — no retry will fix them.
|
|
if 400 <= exc.code < 500 and exc.code != 429:
|
|
raise SystemExit(f"Whisper request failed: {exc}{detail}")
|
|
|
|
if exc.code == 429:
|
|
rate_limit_hits += 1
|
|
if rate_limit_hits >= MAX_429_RETRIES:
|
|
raise SystemExit(f"Whisper request failed: {exc}{detail}")
|
|
delay = _retry_after(exc) or RETRY_BASE_DELAY * (2 ** attempt) + 1
|
|
else:
|
|
delay = RETRY_BASE_DELAY * (2 ** attempt)
|
|
|
|
if attempt < MAX_ATTEMPTS - 1:
|
|
print(
|
|
f"[watch] whisper HTTP {exc.code} — retrying in {delay:.1f}s "
|
|
f"(attempt {attempt + 2}/{MAX_ATTEMPTS})",
|
|
file=sys.stderr,
|
|
)
|
|
time.sleep(delay)
|
|
continue
|
|
except (urllib.error.URLError, TimeoutError, ConnectionResetError, OSError) as exc:
|
|
last_exc, last_detail = exc, ""
|
|
if attempt < MAX_ATTEMPTS - 1:
|
|
delay = RETRY_BASE_DELAY * (attempt + 1)
|
|
print(
|
|
f"[watch] whisper network error ({type(exc).__name__}: {exc}) — "
|
|
f"retrying in {delay:.1f}s (attempt {attempt + 2}/{MAX_ATTEMPTS})",
|
|
file=sys.stderr,
|
|
)
|
|
time.sleep(delay)
|
|
continue
|
|
|
|
try:
|
|
return json.loads(payload)
|
|
except json.JSONDecodeError as exc:
|
|
raise SystemExit(f"Whisper returned non-JSON response: {exc}")
|
|
|
|
raise SystemExit(
|
|
f"Whisper request failed after {MAX_ATTEMPTS} attempts: {last_exc}{last_detail}"
|
|
)
|
|
|
|
|
|
def _read_error_body(exc: urllib.error.HTTPError) -> str:
|
|
try:
|
|
body = exc.read()
|
|
except Exception:
|
|
return ""
|
|
if not body:
|
|
return ""
|
|
try:
|
|
return f" — {body.decode('utf-8', errors='replace')[:400]}"
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _retry_after(exc: urllib.error.HTTPError) -> float | None:
|
|
header = exc.headers.get("Retry-After") if getattr(exc, "headers", None) else None
|
|
if not header:
|
|
return None
|
|
try:
|
|
return float(header)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def shift_segments(segments: list[dict], offset_seconds: float) -> list[dict]:
|
|
"""Return a copy of segments with start/end shifted by offset_seconds.
|
|
|
|
Each chunk is transcribed in isolation, so Whisper returns 0-based timestamps
|
|
per chunk; shifting by the chunk's offset stitches them into source time.
|
|
"""
|
|
if offset_seconds == 0:
|
|
return segments
|
|
return [
|
|
{
|
|
"start": round(seg["start"] + offset_seconds, 2),
|
|
"end": round(seg["end"] + offset_seconds, 2),
|
|
"text": seg["text"],
|
|
}
|
|
for seg in segments
|
|
]
|
|
|
|
|
|
def _segments_from_response(data: dict) -> list[dict]:
|
|
try:
|
|
return normalize_segments(data)
|
|
except (ValueError, TypeError) as exc:
|
|
raise SystemExit(f"Whisper returned malformed segments: {exc}") from None
|
|
|
|
|
|
def transcribe_chunks(
|
|
chunks: list[tuple[Path, float]],
|
|
transcribe_one,
|
|
) -> list[dict]:
|
|
"""Transcribe each chunk, shift its segments by the chunk offset, concatenate.
|
|
|
|
A chunk that fails after its own retries is logged and skipped so one bad
|
|
slice doesn't discard the whole transcript. Raises only if every chunk fails.
|
|
"""
|
|
segments = Segments()
|
|
failures = 0
|
|
for index, (path, offset) in enumerate(chunks):
|
|
try:
|
|
chunk_segments = transcribe_one(path)
|
|
except SystemExit as exc:
|
|
failures += 1
|
|
end = chunks[index + 1][1] if index + 1 < len(chunks) else None
|
|
segments.gaps.append({"start": offset, "end": end})
|
|
print(
|
|
f"[watch] chunk {index + 1}/{len(chunks)} failed — skipping ({exc})",
|
|
file=sys.stderr,
|
|
)
|
|
continue
|
|
segments.extend(shift_segments(chunk_segments, offset))
|
|
print(
|
|
f"[watch] chunk {index + 1}/{len(chunks)} → {len(chunk_segments)} segments",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
if failures == len(chunks):
|
|
raise SystemExit("Whisper failed on every audio chunk")
|
|
segments.no_speech = not segments and not failures
|
|
return segments
|
|
|
|
|
|
def _transcribe_file(backend: str, api_key: str, audio_path: Path) -> list[dict]:
|
|
"""Upload one audio file and return its 0-based segments."""
|
|
if backend == "groq":
|
|
response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path)
|
|
elif backend == "openai":
|
|
response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path)
|
|
else:
|
|
raise SystemExit(f"Unknown whisper backend: {backend}")
|
|
return _segments_from_response(response)
|
|
|
|
|
|
def transcribe_video(
|
|
video_path: str,
|
|
audio_out: Path,
|
|
backend: str | None = None,
|
|
api_key: str | None = None,
|
|
) -> tuple[list[dict], str]:
|
|
"""Run the full flow: extract audio → upload → parse segments.
|
|
|
|
Returns (segments, backend_used). Raises SystemExit on any failure.
|
|
"""
|
|
if api_key is not None and backend is None:
|
|
raise SystemExit("An API key requires an explicit groq or openai backend.")
|
|
if backend == "whisperx":
|
|
if api_key is not None:
|
|
raise SystemExit("WhisperX does not accept an API key.")
|
|
from local_whisperx import transcribe_audio
|
|
segments = transcribe_audio(extract_audio(video_path, audio_out))
|
|
return segments, "whisperx"
|
|
if backend not in (None, "groq", "openai"):
|
|
raise SystemExit("Unknown transcription backend.")
|
|
if backend is None or api_key is None:
|
|
detected_backend, detected_key = load_api_key(preferred=backend)
|
|
backend = backend or detected_backend
|
|
api_key = api_key or detected_key
|
|
|
|
if not backend or not api_key:
|
|
setup_py = Path(__file__).resolve().parent / "setup.py"
|
|
raise SystemExit(
|
|
"No Whisper API key available. Set GROQ_API_KEY (preferred) or OPENAI_API_KEY "
|
|
"in the environment or in ~/.config/watch/.env. "
|
|
f"Run `python3 {setup_py}` to configure."
|
|
)
|
|
|
|
print(f"[watch] extracting audio for Whisper ({backend})…", file=sys.stderr)
|
|
audio_path = extract_audio(video_path, audio_out)
|
|
audio_bytes = audio_path.stat().st_size
|
|
|
|
def transcribe_one(path: Path) -> list[dict]:
|
|
return _transcribe_file(backend, api_key, path)
|
|
|
|
if audio_bytes <= MAX_UPLOAD_BYTES:
|
|
print(
|
|
f"[watch] audio: {audio_bytes / 1024:.0f} kB — uploading to {backend} Whisper…",
|
|
file=sys.stderr,
|
|
)
|
|
segments = transcribe_one(audio_path)
|
|
else:
|
|
duration = audio_duration(audio_path)
|
|
plan = plan_chunks(duration, audio_bytes, MAX_UPLOAD_BYTES - 64_000)
|
|
print(
|
|
f"[watch] audio: {audio_bytes / (1024 * 1024):.0f} MB exceeds "
|
|
f"{MAX_UPLOAD_BYTES // (1024 * 1024)} MB — splitting into {len(plan)} chunks…",
|
|
file=sys.stderr,
|
|
)
|
|
chunks = split_audio(audio_path, audio_out.parent / "chunks", plan)
|
|
if any(path.stat().st_size > MAX_UPLOAD_BYTES for path, _ in chunks):
|
|
raise SystemExit("A split audio chunk exceeds the upload budget; no chunks were uploaded.")
|
|
segments = transcribe_chunks(chunks, transcribe_one)
|
|
|
|
if not segments:
|
|
print("[watch] Whisper detected no speech", file=sys.stderr)
|
|
|
|
print(f"[watch] transcribed {len(segments)} segments via {backend}", file=sys.stderr)
|
|
return segments, backend
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
configure_stdio()
|
|
parser = argparse.ArgumentParser(description="Transcribe local media with the selected Whisper backend.")
|
|
parser.add_argument("video")
|
|
parser.add_argument("audio_out", nargs="?", default="audio.mp3", type=Path)
|
|
parser.add_argument("--backend", choices=["groq", "openai", "whisperx"])
|
|
args = parser.parse_args()
|
|
try:
|
|
segments, backend = transcribe_video(args.video, Path(args.audio_out), backend=args.backend)
|
|
except ConfigError as exc:
|
|
raise SystemExit(str(exc)) from None
|
|
print(json.dumps({"backend": backend, "segments": segments}, indent=2))
|