#!/usr/bin/env python3
"""Download YouTube transcripts as files: one video, a list, a playlist or a channel.

Uses YouTubeTranscriptAPI (https://youtubetranscriptapi.com). Standard library
only, Python 3.9+. Set YT_API_KEY first (free key, 100 credits, no card).

  python download_transcripts.py dQw4w9WgXcQ
  python download_transcripts.py https://youtu.be/dQw4w9WgXcQ --formats txt,srt
  python download_transcripts.py --file videos.txt --formats json
  python download_transcripts.py --playlist PLxxxxxxxx --formats txt,csv --combine
  python download_transcripts.py --channel @TED --dry-run

Formats (--formats, comma-separated, default txt):
  txt    plain text, one caption line per line (--timestamps adds [m:ss])
  md     Markdown with a title, a link and timestamped paragraphs
  srt    SubRip subtitles
  vtt    WebVTT subtitles
  json   the API response as returned: segments with start and duration
  csv    one row per caption line: video_id, start, end, text

Costs (successful calls only; errors are never charged):
  transcript                    1 credit per video (no captions = 404, free)
  playlist or channel listing   20 credits (up to 100 videos)
  balance check (whoami)        free

Output in ./transcripts/:
  <videoId>.<format>     one file per video and format
  all_transcripts.txt    with --combine: every transcript in one file, for NotebookLM or an LLM
  manifest.csv           one row per video: status, title, language, words, files
Re-running skips videos whose files already exist, so a stopped run resumes.
"""

from __future__ import annotations

import argparse
import csv
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path

BASE = os.environ.get("YT_API_BASE", "https://youtubetranscriptapi.com/api/v2")
TIMEOUT = 60
MAX_ATTEMPTS = 4
LISTING_COST = 20
FORMATS = ("txt", "md", "srt", "vtt", "json", "csv")
ID_RE = re.compile(r"(?:v=|youtu\.be/|shorts/|embed/|live/)([A-Za-z0-9_-]{11})|^([A-Za-z0-9_-]{11})$")
MANIFEST_FIELDS = ["videoId", "status", "title", "language", "segments", "words", "files"]


class ApiError(Exception):
    def __init__(self, status: int, code: str, detail):
        super().__init__(f"{status} {code}: {detail}")
        self.status, self.code, self.detail = status, code, detail


def call(path: str, params: dict, key: str) -> dict:
    """GET one endpoint. Retries 408/429/503 and network errors, honoring Retry-After."""
    url = f"{BASE}{path}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}", "Accept": "application/json"})
    for attempt in range(1, MAX_ATTEMPTS + 1):
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except urllib.error.HTTPError as e:
            try:
                body = json.loads(e.read().decode("utf-8") or "{}")
            except ValueError:
                body = {}
            if e.code in (408, 429, 503) and attempt < MAX_ATTEMPTS:
                time.sleep(_retry_after(e.headers.get("Retry-After"), attempt))
                continue
            raise ApiError(e.code, body.get("code") or body.get("error") or "error", body.get("detail")) from None
        except (urllib.error.URLError, TimeoutError):
            if attempt < MAX_ATTEMPTS:
                time.sleep(2 ** attempt)
                continue
            raise
    raise RuntimeError("unreachable")


def _retry_after(value: str | None, attempt: int) -> float:
    try:
        return max(1.0, min(float(value), 120.0))
    except (TypeError, ValueError):
        return float(2 ** attempt)


def video_id(text: str) -> str | None:
    m = ID_RE.search(text.strip())
    return (m.group(1) or m.group(2)) if m else None


def segments_of(data: dict) -> list[dict]:
    t = data.get("transcript")
    if not isinstance(t, list):
        return []
    out = []
    for s in t:
        text = " ".join((s.get("text") or "").split())
        if text:
            out.append({"text": text, "start": float(s.get("start") or 0), "duration": float(s.get("duration") or 0)})
    return out


def timed(segs: list[dict]) -> list[tuple[float, float, str]]:
    """(start, end, text). Automatic captions overlap; each line ends where the next one starts."""
    rows = []
    for i, s in enumerate(segs):
        end = s["start"] + s["duration"]
        if i + 1 < len(segs):
            end = min(end, segs[i + 1]["start"])
        rows.append((s["start"], max(end, s["start"] + 0.01), s["text"]))
    return rows


def stamp(seconds: float) -> str:
    s = int(seconds)
    h, m, s = s // 3600, (s % 3600) // 60, s % 60
    return f"{h}:{m:02d}:{s:02d}" if h else f"{m}:{s:02d}"


def clock(seconds: float, sep: str) -> str:
    ms = int(round(seconds * 1000))
    h, ms = divmod(ms, 3_600_000)
    m, ms = divmod(ms, 60_000)
    s, ms = divmod(ms, 1000)
    return f"{h:02d}:{m:02d}:{s:02d}{sep}{ms:03d}"


def render(fmt: str, vid: str, title: str, data: dict, segs: list[dict], timestamps: bool) -> str:
    if fmt == "txt":
        return "\n".join(f"[{stamp(s['start'])}] {s['text']}" if timestamps else s["text"] for s in segs) + "\n"
    if fmt == "md":
        lines, para, para_start = [f"# {title}", "", f"https://www.youtube.com/watch?v={vid}", ""], [], None
        for s in segs:
            if para_start is None:
                para_start = s["start"]
            para.append(s["text"])
            if s["start"] - para_start >= 60:
                lines += [f"[{stamp(para_start)}](https://www.youtube.com/watch?v={vid}&t={int(para_start)}s) {' '.join(para)}", ""]
                para, para_start = [], None
        if para:
            lines += [f"[{stamp(para_start)}](https://www.youtube.com/watch?v={vid}&t={int(para_start)}s) {' '.join(para)}", ""]
        return "\n".join(lines)
    if fmt == "srt":
        return "\n".join(f"{i}\n{clock(a, ',')} --> {clock(b, ',')}\n{t}\n" for i, (a, b, t) in enumerate(timed(segs), 1))
    if fmt == "vtt":
        return "WEBVTT\n\n" + "\n".join(f"{clock(a, '.')} --> {clock(b, '.')}\n{t}\n" for a, b, t in timed(segs))
    if fmt == "json":
        return json.dumps(data, ensure_ascii=False, indent=2) + "\n"
    raise ValueError(fmt)


def write_csv(path: Path, vid: str, segs: list[dict]) -> None:
    with open(path, "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f)
        w.writerow(["video_id", "start", "end", "text"])
        for a, b, t in timed(segs):
            w.writerow([vid, round(a, 3), round(b, 3), t])


def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description="Download YouTube transcripts as TXT, Markdown, SRT, VTT, JSON or CSV.")
    p.add_argument("videos", nargs="*", help="video IDs or URLs")
    p.add_argument("--file", help="text file with one video ID or URL per line")
    p.add_argument("--playlist", help="playlist ID or URL (20 credits, up to 100 videos)")
    p.add_argument("--channel", help="channel @handle, URL or UC ID: its 100 newest uploads (20 credits)")
    p.add_argument("--formats", default="txt", help=f"comma-separated: {','.join(FORMATS)} (default txt)")
    p.add_argument("--timestamps", action="store_true", help="prefix each TXT line with [m:ss]")
    p.add_argument("--language", default="en", help="preferred caption language (default en)")
    p.add_argument("--combine", action="store_true", help="also write all_transcripts.txt with every video")
    p.add_argument("--dry-run", action="store_true", help="list the videos and the credit cost; fetch no transcripts")
    p.add_argument("--out", default="transcripts", help="output folder")
    a = p.parse_args(argv)

    formats = [f.strip().lower() for f in a.formats.split(",") if f.strip()]
    bad = [f for f in formats if f not in FORMATS]
    if bad or not formats:
        print(f"Unknown format(s): {', '.join(bad) or '(none)'}. Choose from {', '.join(FORMATS)}.", file=sys.stderr)
        return 2
    key = os.environ.get("YT_API_KEY")
    if not key:
        print("Set YT_API_KEY first (free key, 100 credits: https://youtubetranscriptapi.com/signup)", file=sys.stderr)
        return 2

    videos: list[tuple[str, str]] = []
    raw = list(a.videos)
    if a.file:
        raw += [ln for ln in Path(a.file).read_text(encoding="utf-8").splitlines() if ln.strip() and not ln.startswith("#")]
    for item in raw:
        vid = video_id(item)
        if vid:
            videos.append((vid, ""))
        else:
            print(f"Skipping, not a video ID or URL: {item}", file=sys.stderr)

    credits = 0
    try:
        balance = call("/account/whoami", {}, key)["credits"]["spendable"]
        listings = [(k, v) for k, v in (("playlist", a.playlist), ("channel", a.channel)) if v]
        if a.dry_run and listings:
            print(f"Listing {len(listings)} source(s) costs {LISTING_COST * len(listings)} credits; the dry run lists them anyway.")
        for kind, value in listings:
            listing = call("/youtube/playlist/videos" if kind == "playlist" else "/youtube/channel/videos", {kind: value}, key)
            credits += LISTING_COST
            for r in listing.get("results", []):
                if r.get("videoId"):
                    videos.append((r["videoId"], r.get("title") or ""))
            if listing.get("has_more"):
                print(f"Note: the {kind} has more than the {len(listing.get('results', []))} videos listed here.")
    except ApiError as e:
        print(f"Request failed (not charged): {e}", file=sys.stderr)
        return 1

    seen, unique = set(), []
    for v in videos:
        if v[0] not in seen:
            seen.add(v[0])
            unique.append(v)
    if not unique:
        print("No videos to process.", file=sys.stderr)
        return 2

    out = Path(a.out)
    out.mkdir(parents=True, exist_ok=True)
    todo = [v for v in unique if not all((out / f"{v[0]}.{f}").exists() for f in formats)]
    print(f"{len(unique)} videos, {len(unique) - len(todo)} already downloaded. "
          f"Balance {balance} credits; this run costs at most {len(todo)} more.")
    if a.dry_run:
        for vid, title in todo:
            print(f"  {vid}  {title}")
        return 0

    manifest_path = out / "manifest.csv"
    manifest: dict[str, dict] = {}
    if manifest_path.exists():
        with open(manifest_path, newline="", encoding="utf-8") as f:
            manifest = {r["videoId"]: r for r in csv.DictReader(f)}

    saved = no_captions = failed = 0
    for n, (vid, title) in enumerate(todo, 1):
        try:
            data = call("/youtube/transcript", {"video_url": vid, "include_timestamp": "true",
                                                "send_metadata": "true", "language": a.language}, key)
        except ApiError as e:
            if e.status == 404:
                no_captions += 1
                manifest[vid] = {"videoId": vid, "status": "no captions", "title": title}
                print(f"[{n}/{len(todo)}] {vid}: no captions (free)")
                continue
            if e.status == 402:
                print(f"Out of credits after {saved} videos; stopping. Re-run after topping up to resume.", file=sys.stderr)
                break
            failed += 1
            manifest[vid] = {"videoId": vid, "status": f"error {e.status}", "title": title}
            print(f"[{n}/{len(todo)}] {vid}: {e}", file=sys.stderr)
            continue
        credits += 1
        segs = segments_of(data)
        title = title or (data.get("metadata") or {}).get("title") or vid
        files = []
        for fmt in formats:
            path = out / f"{vid}.{fmt}"
            if fmt == "csv":
                write_csv(path, vid, segs)
            else:
                path.write_text(render(fmt, vid, title, data, segs, a.timestamps), encoding="utf-8")
            files.append(path.name)
        words = sum(len(s["text"].split()) for s in segs)
        manifest[vid] = {"videoId": vid, "status": "saved", "title": title, "language": data.get("language", ""),
                         "segments": len(segs), "words": words, "files": " ".join(files)}
        saved += 1
        print(f"[{n}/{len(todo)}] {vid}: {words:,} words -> {', '.join(files)}")

    with open(manifest_path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=MANIFEST_FIELDS, extrasaction="ignore")
        w.writeheader()
        for row in manifest.values():
            w.writerow(row)

    if a.combine:
        parts = []
        for vid, title in unique:
            row = manifest.get(vid, {})
            txt = out / f"{vid}.txt"
            body = txt.read_text(encoding="utf-8") if txt.exists() else None
            if body is None and (out / f"{vid}.json").exists():
                segs = segments_of(json.loads((out / f"{vid}.json").read_text(encoding="utf-8")))
                body = render("txt", vid, title, {}, segs, False)
            if body:
                name = row.get("title") or title or vid
                parts.append(f"=== {name}\nhttps://www.youtube.com/watch?v={vid}\n\n{body.strip()}\n")
        (out / "all_transcripts.txt").write_text("\n".join(parts), encoding="utf-8")
        print(f"Combined {len(parts)} transcripts into {out / 'all_transcripts.txt'}")

    print(f"\nSaved {saved}, no captions {no_captions}, failed {failed}. Credits used: {credits}. "
          f"Details: {manifest_path}")
    return 0


if __name__ == "__main__":
    sys.exit(main())
