#!/usr/bin/env python3
"""Fetch YouTube transcripts at scale, or price a DIY scraper first.

Standard library only (Python 3.9+). Two commands:

  estimate  Compare a monthly bill for scraping YouTube yourself through
            residential proxies with the same volume on YouTubeTranscriptAPI.
            No network calls.
  run       Fetch transcripts for a list of video IDs or URLs from
            YouTubeTranscriptAPI with parallel workers, a shared rate limit
            under 300 requests a minute, retries, a credit check, and a
            SQLite queue you can stop and resume.

Examples:
  python transcripts_at_scale.py estimate --per-month 10000
  export YT_API_KEY=...        # youtubetranscriptapi.com, 100 free credits
  python transcripts_at_scale.py run ids.txt --out transcripts --workers 24
  python transcripts_at_scale.py run ids.txt --out transcripts --retry-failed

Each video ends in one state: done, no_captions (404, not charged) or failed.
Transcripts are written to OUT/<video_id>.json; the queue lives in OUT/queue.db.

The estimate uses bandwidth we measured on November 19, 2026 (a YouTube watch
page was 138-158 KB compressed and 1.37-1.49 MB uncompressed) and residential
proxy prices from $1 to $8 per GB. Change any input with a flag.
"""
import argparse
import json
import math
import os
import re
import sqlite3
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

BASE = "https://youtubetranscriptapi.com/api/v2"
TIMEOUT = 60          # a transcript not fetched before can take 10-30 s
RATE_PER_MIN = 280    # stay under the 300 requests/minute limit per key
MAX_ATTEMPTS = 5
VIDEO_ID = re.compile(r"(?:v=|youtu\.be/|shorts/|embed/|live/)([A-Za-z0-9_-]{11})|^([A-Za-z0-9_-]{11})$")

# DIY scraping profiles: megabytes transferred per attempt (page + caption file).
PROFILES = {
    "http-compressed": 0.2,    # watch page ~0.15 MB gzipped + caption allowance
    "http-uncompressed": 1.5,  # watch page ~1.4 MB when the client skips gzip
    "headless-browser": 20.0,  # full browser load, per Bright Data's docs
}
YTT_MONTHLY = (3.0, 1000, 2.50)   # $ a month, credits a month, top-up $ per 1K
YTT_ANNUAL = (30.0, 15000, 2.00)  # $ a year, credits up front, top-up $ per 1K


# ---------------------------------------------------------------- estimate

def ytt_month(n):
    """Monthly cost of n transcripts on each YouTubeTranscriptAPI plan."""
    m_price, m_incl, m_topup = YTT_MONTHLY
    monthly = m_price + math.ceil(max(0, n - m_incl) / 1000) * m_topup
    a_price, a_incl, a_topup = YTT_ANNUAL
    annual = (a_price + math.ceil(max(0, n * 12 - a_incl) / 1000) * a_topup) / 12
    return round(monthly, 2), round(annual, 2)


def diy_month(n, mb, usd_per_gb, retry_rate, hours, hourly):
    """Monthly cost of scraping n transcripts yourself: proxy bandwidth plus upkeep."""
    attempts = n * (1 + retry_rate)
    gb = attempts * mb / 1000
    return round(gb, 2), round(gb * usd_per_gb, 2), round(hours * hourly, 2)


def estimate(args):
    n = args.per_month
    prices = args.proxy_usd_per_gb or [1.0, 3.5, 8.0]
    profiles = {args.profile: PROFILES[args.profile]} if args.profile else PROFILES
    if args.mb_per_attempt:
        profiles = {"custom": args.mb_per_attempt}
    upkeep = round(args.hours_per_month * args.hourly_rate, 2)
    print(f"{n:,} transcripts a month, {args.retry_rate:.0%} of attempts retried, "
          f"{args.hours_per_month:g} h/month upkeep at ${args.hourly_rate:.2f}/h = ${upkeep:,.2f}.\n")
    head = f"{'DIY profile':20} {'MB/try':>7} {'GB/mo':>8}" + "".join(f" {'$' + format(p, 'g') + '/GB':>11}" for p in prices)
    print(head)
    for name, mb in profiles.items():
        gb, _, _ = diy_month(n, mb, 1, args.retry_rate, 0, 0)
        cells = []
        for p in prices:
            _, bw, _ = diy_month(n, mb, p, args.retry_rate, 0, 0)
            cells.append(f" {'$' + format(bw, ',.2f'):>11}")
        print(f"{name:20} {mb:>7g} {gb:>8,.1f}" + "".join(cells))
    print(f"\nAdd ${upkeep:,.2f} upkeep to every DIY cell above (proxies only, no servers or PO token tooling).")
    monthly, annual = ytt_month(n)
    print(f"YouTubeTranscriptAPI: ${monthly:,.2f} a month on Monthly, ${annual:,.2f} a month on Annual. "
          "Videos without captions and errors aren't charged.")
    return 0


# --------------------------------------------------------------------- run

class RateLimiter:
    """Shared limit across threads: at most `per_min` request starts a minute."""

    def __init__(self, per_min):
        self.interval = 60.0 / per_min
        self.lock = threading.Lock()
        self.next_at = time.monotonic()

    def wait(self):
        with self.lock:
            now = time.monotonic()
            start = max(now, self.next_at)
            self.next_at = start + self.interval
        time.sleep(max(0.0, start - now))


class Stop(Exception):
    """Out of credits or bad key: stop every worker."""


def video_id(line):
    m = VIDEO_ID.search(line.strip())
    return (m.group(1) or m.group(2)) if m else None


def api_get(path, params, key):
    url = f"{BASE}{path}?{urllib.parse.urlencode(params)}" if params else f"{BASE}{path}"
    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}", "Accept": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
            return r.status, dict(r.headers), r.read()
    except urllib.error.HTTPError as e:
        return e.code, dict(e.headers), e.read()


def open_queue(out):
    db = sqlite3.connect(out / "queue.db", check_same_thread=False, isolation_level=None)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("CREATE TABLE IF NOT EXISTS videos (id TEXT PRIMARY KEY, status TEXT NOT NULL DEFAULT 'pending', "
               "attempts INTEGER NOT NULL DEFAULT 0, error TEXT, updated REAL)")
    return db


def fetch_one(vid, key, limiter, out, language):
    """Fetch one transcript. Returns (status, error). Raises Stop on 401/402."""
    error = ""
    for attempt in range(1, MAX_ATTEMPTS + 1):
        limiter.wait()
        try:
            status, headers, body = api_get("/youtube/transcript", {"video_url": vid, "language": language}, key)
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            status, headers, body, error = 0, {}, b"", str(e)
        if status == 200:
            data = json.loads(body)
            if not data.get("transcript"):
                return "failed", "empty transcript in a 200 response"
            tmp = out / f"{vid}.json.tmp"
            tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
            tmp.replace(out / f"{vid}.json")
            return "done", ""
        if status == 404:
            return "no_captions", ""
        if status == 402:
            raise Stop("Out of credits (402). Top up, then rerun; finished videos are skipped.")
        if status == 401:
            raise Stop("The API key was rejected (401). Check YT_API_KEY.")
        if status in (400, 422):
            return "failed", f"HTTP {status}: not a valid video"
        if status in (0, 408, 429, 500, 502, 503, 504):
            retry_after = {k.lower(): v for k, v in headers.items()}.get("retry-after", "")
            wait = float(retry_after) if retry_after.replace(".", "", 1).isdigit() else min(60, 2 ** attempt)
            error = error or f"HTTP {status}"
            time.sleep(wait)
            continue
        return "failed", f"HTTP {status}"
    return "failed", f"gave up after {MAX_ATTEMPTS} attempts ({error})"


def run(args, key=None):
    key = key or os.environ.get("YT_API_KEY")
    if not key:
        print("Set YT_API_KEY (get one at youtubetranscriptapi.com, 100 free credits).", file=sys.stderr)
        return 2
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)
    db = open_queue(out)
    lines = sys.stdin if args.file == "-" else open(args.file, encoding="utf-8")
    ids, bad = [], 0
    for line in lines:
        if not line.strip() or line.startswith("#"):
            continue
        vid = video_id(line)
        if vid:
            ids.append(vid)
        else:
            bad += 1
    db.executemany("INSERT OR IGNORE INTO videos (id) VALUES (?)", [(v,) for v in dict.fromkeys(ids)])
    if args.retry_failed:
        db.execute("UPDATE videos SET status='pending' WHERE status='failed'")
    todo = [r[0] for r in db.execute("SELECT id FROM videos WHERE status='pending' ORDER BY rowid")]
    if bad:
        print(f"Skipped {bad} line(s) that aren't YouTube URLs or 11-character IDs.")

    status, _, body = api_get("/account/whoami", None, key)
    if status == 401:
        print("The API key was rejected (401). Check YT_API_KEY.", file=sys.stderr)
        return 2
    spendable = (json.loads(body).get("credits") or {}).get("spendable") if status == 200 else None
    cap = len(todo) if args.max_credits is None else min(len(todo), args.max_credits)
    if spendable is not None and spendable < cap:
        print(f"{spendable} credits available for {cap} videos: the run stops when they run out.")
    todo = todo[:cap]
    print(f"{len(todo)} to fetch with {args.workers} workers at up to {args.rate} requests a minute.")

    limiter = RateLimiter(args.rate)
    counts = {"done": 0, "no_captions": 0, "failed": 0}
    started = time.monotonic()
    stop = threading.Event()
    db_lock = threading.Lock()

    def work(vid):
        if stop.is_set():
            return vid, None, ""
        st, err = fetch_one(vid, key, limiter, out, args.language)
        with db_lock:
            db.execute("UPDATE videos SET status=?, attempts=attempts+1, error=?, updated=? WHERE id=?",
                       (st, err, time.time(), vid))
        return vid, st, err

    message = ""
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        futures = [pool.submit(work, v) for v in todo]
        for i, fut in enumerate(as_completed(futures), 1):
            try:
                vid, st, err = fut.result()
            except Stop as e:
                message = str(e)
                stop.set()
                continue
            if st:
                counts[st] += 1
                if st == "failed":
                    print(f"  {vid}: {err}")
            if i % args.progress_every == 0 or i == len(futures):
                mins = max((time.monotonic() - started) / 60, 1e-9)
                print(f"[{i}/{len(todo)}] {counts['done']} done, {counts['no_captions']} no captions, "
                      f"{counts['failed']} failed, {i / mins:.0f}/min")
    if message:
        print(message, file=sys.stderr)
    totals = dict(db.execute("SELECT status, COUNT(*) FROM videos GROUP BY status").fetchall())
    print(f"Queue: {totals}. Credits used this run: {counts['done']} (no-caption videos are free).")
    return 1 if message else 0


def main(argv=None, key=None):
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = p.add_subparsers(dest="cmd", required=True)
    e = sub.add_parser("estimate", help="price a DIY scraper against the API")
    e.add_argument("--per-month", type=int, required=True, help="transcripts a month")
    e.add_argument("--retry-rate", type=float, default=0.2, help="share of attempts retried after a block (0-1)")
    e.add_argument("--profile", choices=sorted(PROFILES), help="show one scraping profile")
    e.add_argument("--mb-per-attempt", type=float, help="your own measured MB per attempt")
    e.add_argument("--proxy-usd-per-gb", type=float, nargs="+", help="proxy prices to compare, such as 1 3.5 8")
    e.add_argument("--hours-per-month", type=float, default=4, help="engineering time to keep the scraper working")
    e.add_argument("--hourly-rate", type=float, default=64.44, help="US median for software developers, BLS 2025")
    r = sub.add_parser("run", help="fetch transcripts through YouTubeTranscriptAPI")
    r.add_argument("file", help="file with one YouTube URL or video ID per line, or - for stdin")
    r.add_argument("--out", default="transcripts")
    r.add_argument("--workers", type=int, default=24, help="parallel requests; new videos take 10-30 s each")
    r.add_argument("--rate", type=int, default=RATE_PER_MIN, help="request starts a minute, under 300")
    r.add_argument("--language", default="en")
    r.add_argument("--max-credits", type=int, help="stop after this many videos")
    r.add_argument("--retry-failed", action="store_true", help="put failed videos back in the queue")
    r.add_argument("--progress-every", type=int, default=100)
    args = p.parse_args(argv)
    if args.cmd == "estimate":
        if not 0 <= args.retry_rate <= 5:
            p.error("--retry-rate is a share such as 0.2")
        return estimate(args)
    if not 1 <= args.rate <= 300:
        p.error("--rate must be between 1 and 300 requests a minute")
    return run(args, key=key)


if __name__ == "__main__":
    sys.exit(main())
