#!/usr/bin/env python3
"""Switch YouTube transcript calls from Supadata to YouTubeTranscriptAPI, and test first.

Three commands, standard library only (Python 3.9+):

  compare  Fetch the same videos from both APIs and write compare.csv:
           status, credits charged, language, word count and text overlap.
           Supadata is called with mode=native, so it never starts AI transcription.
  cost     Estimate a monthly bill on each API for your volume and the share
           of videos without captions. No API calls.
  fetch    Fetch one transcript from YouTubeTranscriptAPI in Supadata's
           response shape, so existing code can switch with a one-line change.

Examples:
  export YT_API_KEY=...            # youtubetranscriptapi.com, 100 free credits
  export SUPADATA_API_KEY=...      # optional, only for compare
  python switch_from_supadata.py compare urls.txt --out compare.csv
  python switch_from_supadata.py cost --per-month 5000 --no-captions 0.12 --minutes 20
  python switch_from_supadata.py fetch https://www.youtube.com/watch?v=dQw4w9WgXcQ

In your own code, replace the Supadata call with get_transcript(url, lang, text).
It returns {"content", "lang", "availableLangs"} like Supadata's /v1/transcript,
with offset and duration in milliseconds, and raises TranscriptUnavailable where
Supadata would answer 206.

Prices in cost are list prices checked on November 18, 2026. Check both pricing
pages before you decide.
"""
import argparse
import csv
import difflib
import json
import math
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

YTT_BASE = "https://youtubetranscriptapi.com/api/v2"
SUPADATA_BASE = "https://api.supadata.ai/v1"
TIMEOUT = 60  # seconds; a transcript not fetched before can take 10-30 s
MAX_TRIES = 4

# Supadata plans as listed on supadata.ai/pricing (Nov 18, 2026):
# name, $ per month, credits per month, auto recharge ($, credits) or None.
# Basic is billed annually ($60/year). Yearly discounts on other plans aren't
# published, so monthly prices are used.
SUPADATA_PLANS = [
    ("Free", 0, 100, None),
    ("Basic", 5, 300, (10, 1000)),
    ("Pro", 17, 3000, (10, 1000)),
    ("Mega", 47, 30000, (10, 5000)),
    ("Giga", 297, 300000, (20, 20000)),
    ("Supa", 897, 1000000, (20, 20000)),
]
SUPADATA_GENERATED_PER_MIN = 2  # credits per minute of AI transcription
SUPADATA_UNAVAILABLE = 1        # credits for a 206 "transcript unavailable"

# YouTubeTranscriptAPI (youtubetranscriptapi.com/pricing, Nov 18, 2026).
YTT_MONTHLY = (3, 1000, 2.50)   # $ per month, credits per month, top-up $ per 1K
YTT_ANNUAL = (30, 15000, 2.00)  # $ per year, credits up front, top-up $ per 1K
YTT_TOPUP_MIN = 5000            # top-ups come in whole thousands, 5K minimum


class TranscriptUnavailable(Exception):
    """No captions, private or removed video (Supadata: 206, us: 404). Not charged here."""


class OutOfCredits(Exception):
    pass


def _get(url, headers):
    """GET with retries on 408, 429 and 5xx. Returns (status, headers, body)."""
    for attempt in range(1, MAX_TRIES + 1):
        req = urllib.request.Request(url, headers=headers)
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
                return r.status, dict(r.headers), r.read()
        except urllib.error.HTTPError as e:
            body = e.read()
            if e.code in (408, 429, 500, 502, 503, 504) and attempt < MAX_TRIES:
                wait = e.headers.get("Retry-After") or str(2 ** attempt)
                time.sleep(float(wait) if wait.replace(".", "", 1).isdigit() else 2 ** attempt)
                continue
            return e.code, dict(e.headers), body
        except (urllib.error.URLError, TimeoutError):
            if attempt == MAX_TRIES:
                raise
            time.sleep(2 ** attempt)
    raise RuntimeError("unreachable")


def _header(headers, name):
    for k, v in headers.items():
        if k.lower() == name.lower():
            return v
    return None


def ytt_raw(url, lang=None):
    """Call YouTubeTranscriptAPI. Returns (status, credits_charged, json_or_None)."""
    key = os.environ.get("YT_API_KEY")
    if not key:
        sys.exit("Set YT_API_KEY (get one at youtubetranscriptapi.com, 100 free credits).")
    params = {"video_url": url, "format": "json", "include_timestamp": "true"}
    if lang:
        params["language"] = lang
    status, headers, body = _get(f"{YTT_BASE}/youtube/transcript?{urllib.parse.urlencode(params)}",
                                 {"Authorization": f"Bearer {key}", "Accept": "application/json"})
    if status == 402:
        raise OutOfCredits("YouTubeTranscriptAPI: out of credits")
    charged = int(_header(headers, "X-Credits-Charged") or (1 if status == 200 else 0))
    data = json.loads(body) if status == 200 else None
    return status, charged, data


def supadata_raw(url, lang=None):
    """Call Supadata /v1/transcript with mode=native. Returns (status, credits, json_or_None)."""
    key = os.environ.get("SUPADATA_API_KEY")
    if not key:
        sys.exit("Set SUPADATA_API_KEY to compare, or run compare with --ours-only.")
    params = {"url": url, "mode": "native"}
    if lang:
        params["lang"] = lang
    status, headers, body = _get(f"{SUPADATA_BASE}/transcript?{urllib.parse.urlencode(params)}",
                                 {"x-api-key": key, "Accept": "application/json"})
    credits = _header(headers, "x-billable-requests")
    data = json.loads(body) if status == 200 and body else None
    return status, (int(credits) if credits and credits.isdigit() else None), data


def get_transcript(url, lang=None, text=False):
    """Drop-in for Supadata's transcript call, served by YouTubeTranscriptAPI.

    Returns {"content": [{"text", "offset", "duration", "lang"}], "lang", "availableLangs"}
    with offset and duration in milliseconds, or {"content": str, ...} when text=True.
    availableLangs only holds the language returned: this API doesn't list other tracks.
    """
    status, _, data = ytt_raw(url, lang)
    if status == 404:
        raise TranscriptUnavailable(url)
    if status != 200:
        raise RuntimeError(f"YouTubeTranscriptAPI returned HTTP {status} for {url}")
    language = data.get("language") or lang or ""
    segments = data.get("transcript") or []
    if text:
        content = " ".join(s["text"].strip() for s in segments if s.get("text"))
    else:
        content = [{"text": s.get("text", ""),
                    "offset": int(round(float(s.get("start", 0)) * 1000)),
                    "duration": int(round(float(s.get("duration", 0)) * 1000)),
                    "lang": language} for s in segments]
    return {"content": content, "lang": language, "availableLangs": [language] if language else []}


def _words(segments_or_text):
    if isinstance(segments_or_text, str):
        text = segments_or_text
    else:
        text = " ".join(s.get("text", "") for s in segments_or_text or [])
    text = re.sub(r"\[[^\]]*\]", " ", text.lower())  # [Music], [Applause]
    return re.findall(r"[a-z0-9']+", text)


def overlap(a, b, limit=3000):
    """Share of matching words between two transcripts (0-1), on the first `limit` words."""
    a, b = a[:limit], b[:limit]
    if not a or not b:
        return None
    return round(difflib.SequenceMatcher(None, a, b, autojunk=False).ratio(), 3)


def compare(args):
    if not os.environ.get("YT_API_KEY"):
        sys.exit("Set YT_API_KEY (get one at youtubetranscriptapi.com, 100 free credits).")
    if not args.ours_only and not os.environ.get("SUPADATA_API_KEY"):
        sys.exit("Set SUPADATA_API_KEY to compare, or run compare with --ours-only.")
    urls = [u.strip() for u in open(args.file, encoding="utf-8") if u.strip() and not u.startswith("#")]
    rows = []
    for i, url in enumerate(urls, 1):
        row = {"url": url}
        try:
            st, cr, data = ytt_raw(url, args.lang)
        except OutOfCredits as e:
            print(e, file=sys.stderr)
            break
        ours = _words((data or {}).get("transcript"))
        row.update(ours_status=st, ours_credits=cr, ours_lang=(data or {}).get("language", ""), ours_words=len(ours))
        if not args.ours_only:
            st2, cr2, data2 = supadata_raw(url, args.lang)
            theirs = _words((data2 or {}).get("content"))
            row.update(supadata_status=st2, supadata_credits="" if cr2 is None else cr2,
                       supadata_lang=(data2 or {}).get("lang", ""), supadata_words=len(theirs),
                       overlap=overlap(ours, theirs) if ours and theirs else "")
        rows.append(row)
        print(f"[{i}/{len(urls)}] {url}: ours {row['ours_status']}"
              + ("" if args.ours_only else f", supadata {row['supadata_status']}, overlap {row['overlap']}"))
    if not rows:
        return 1
    fields = list(rows[0].keys())
    for r in rows[1:]:
        fields += [k for k in r if k not in fields]
    with open(args.out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        w.writerows(rows)
    ok = sum(1 for r in rows if r["ours_status"] == 200)
    print(f"\nYouTubeTranscriptAPI: {ok}/{len(rows)} transcripts, "
          f"{sum(r['ours_credits'] for r in rows)} credits charged.")
    if not args.ours_only:
        ok2 = sum(1 for r in rows if r.get("supadata_status") == 200)
        billed = [r["supadata_credits"] for r in rows if r.get("supadata_credits") != ""]
        print(f"Supadata (native): {ok2}/{len(rows)} transcripts, "
              f"{sum(billed) if billed else 'unknown'} credits billed.")
        both = [r["overlap"] for r in rows if r.get("overlap") not in ("", None)]
        if both:
            both.sort()
            print(f"Text overlap where both returned text: median {both[len(both) // 2]:.2f}, lowest {both[0]:.2f}.")
    print(f"Wrote {args.out}. Read the rows with low overlap or a different language by hand.")
    return 0


def supadata_month(credits):
    """Cheapest Supadata plan (plus auto recharge) for a month needing `credits`. Returns ($, plan)."""
    best = None
    for name, price, included, recharge in SUPADATA_PLANS:
        extra = max(0, credits - included)
        if extra and not recharge:
            continue
        packs = -(-extra // recharge[1]) if extra else 0
        cost = price + packs * (recharge[0] if recharge else 0)
        label = name + (f" + {packs} recharge" + ("s" if packs > 1 else "") if packs else "")
        if best is None or cost < best[0]:
            best = (cost, label)
    return best


def ytt_month(transcripts):
    """Monthly cost on each YouTubeTranscriptAPI plan for `transcripts` a month. Returns dict."""
    m_price, m_incl, m_topup = YTT_MONTHLY
    # Top-ups never expire, so a 5K minimum pack spreads over later months:
    # count the thousands this month actually uses.
    thousands = -(-max(0, transcripts - m_incl) // 1000)
    monthly = m_price + thousands * m_topup
    a_price, a_incl, a_topup = YTT_ANNUAL
    year_extra = max(0, transcripts * 12 - a_incl)
    annual = (a_price + -(-year_extra // 1000) * a_topup) / 12
    return {"Monthly": round(monthly, 2), "Annual": round(annual, 2)}


def cost(args):
    n = args.per_month
    missing = round(n * args.no_captions)
    captioned = n - missing
    # Supadata default mode=auto: captionless videos are transcribed at 2 credits a minute.
    auto = captioned + math.ceil(missing * SUPADATA_GENERATED_PER_MIN * args.minutes)
    # mode=native: captionless videos return 206, which still costs 1 credit.
    native = captioned + missing * SUPADATA_UNAVAILABLE
    ours = ytt_month(captioned)  # 404s for missing captions are free
    rows = [
        ("Supadata, mode=auto (default)", auto, *supadata_month(auto)),
        ("Supadata, mode=native", native, *supadata_month(native)),
        ("YouTubeTranscriptAPI, Monthly", captioned, ours["Monthly"], "Monthly + top-ups" if captioned > 1000 else "Monthly"),
        ("YouTubeTranscriptAPI, Annual", captioned, ours["Annual"], "Annual, per month" + (" + top-ups" if captioned * 12 > 15000 else "")),
    ]
    print(f"{n:,} videos a month, {missing:,} without captions (avg {args.minutes} min).\n")
    print(f"{'Route':34} {'Credits':>10} {'$/month':>9}  Plan")
    for route, credits, dollars, plan in rows:
        print(f"{route:34} {credits:>10,} {dollars:>9.2f}  {plan}")
    print("\nThe auto row includes AI transcripts for the captionless videos; the other rows skip them.")
    print("YouTubeTranscriptAPI top-ups are sold 5K at a time and never expire. Supadata credits don't roll over.")
    return 0


def fetch(args):
    try:
        result = get_transcript(args.url, args.lang, args.text)
    except TranscriptUnavailable:
        print("No transcript for this video (no captions, private or removed). Not charged.", file=sys.stderr)
        return 3
    json.dump(result, sys.stdout, ensure_ascii=False, indent=2)
    print()
    return 0


def main(argv=None):
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = p.add_subparsers(dest="cmd", required=True)
    c = sub.add_parser("compare", help="fetch the same videos from both APIs")
    c.add_argument("file", help="text file, one YouTube URL or video ID per line")
    c.add_argument("--out", default="compare.csv")
    c.add_argument("--lang", help="preferred language code, such as en")
    c.add_argument("--ours-only", action="store_true", help="skip Supadata (no key needed)")
    c.set_defaults(func=compare)
    k = sub.add_parser("cost", help="estimate a monthly bill on each API")
    k.add_argument("--per-month", type=int, required=True, help="videos a month")
    k.add_argument("--no-captions", type=float, default=0.12, help="share without captions (0-1)")
    k.add_argument("--minutes", type=float, default=20, help="average length of a captionless video")
    k.set_defaults(func=cost)
    f = sub.add_parser("fetch", help="one transcript in Supadata's response shape")
    f.add_argument("url")
    f.add_argument("--lang")
    f.add_argument("--text", action="store_true", help="plain text instead of timed chunks")
    f.set_defaults(func=fetch)
    args = p.parse_args(argv)
    if getattr(args, "no_captions", 0) and not 0 <= args.no_captions <= 1:
        p.error("--no-captions is a share between 0 and 1, such as 0.12")
    return args.func(args)


if __name__ == "__main__":
    sys.exit(main())
