#!/usr/bin/env python3
"""Analyze what YouTube videos say: topics, claims, mentions and questions, as data.

Transcripts come from YouTubeTranscriptAPI (https://youtubetranscriptapi.com);
the analysis comes from the Claude API with a fixed JSON schema, so every video
returns the same fields. Claims and mentions carry the [m:ss] moment they come
from, turned into links that open the video there.

  pip install anthropic
  export YT_API_KEY=...            # free key, 100 credits
  export ANTHROPIC_API_KEY=...

  python analyze_videos.py dQw4w9WgXcQ
  python analyze_videos.py --channel @competitor --model claude-haiku-5-5
  python analyze_videos.py --playlist PLxxxxxxxx --question "What pricing does the speaker mention?"
  python analyze_videos.py --channel @competitor --estimate      # transcripts only, no LLM calls

Costs:
  transcript                    1 YouTubeTranscriptAPI credit per video (no captions = 404, free)
  playlist or channel listing   20 credits (up to 100 videos)
  analysis                      Claude API tokens, priced in PRICES below (list prices, Nov 2026)

Output in ./analysis/:
  json/<videoId>.json     the full analysis for one video
  videos.csv              one row per video: title, views text, sentiment, topics, mentions, summary
  claims.csv              one row per claim, with a link to the moment
  mentions.csv            one row per brand, product, person or organization mentioned
  topics.csv              every topic with the number of videos that cover it
  report.md               the channel picture: top topics, top mentions, sentiment, calls to action
  transcripts/<id>.json   cached transcripts, so re-runs never pay for them twice
Re-running skips videos that already have an analysis.
"""

from __future__ import annotations

import argparse
import collections
import csv
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path

BASE = os.environ.get("YT_API_BASE", "https://youtubetranscriptapi.com/api/v2")
TIMEOUT = 60
MAX_ATTEMPTS = 4
LISTING_COST = 20
ID_RE = re.compile(r"(?:v=|youtu\.be/|shorts/|embed/|live/)([A-Za-z0-9_-]{11})|^([A-Za-z0-9_-]{11})$")

# USD per million tokens (input, output), Claude API list prices checked November 2026:
# https://platform.claude.com/docs/en/about-claude/pricing
PRICES = {
    "claude-haiku-5-5": ((0.10, 0.50), (0.50, 2.50)),
    "claude-sonnet-5-5": ((2.00, 10.00), (2.00, 10.00)),
    "claude-opus-5-5": ((4.00, 20.00), (4.00, 20.00)),
}
LONG_PROMPT = 100_000  # Claude Haiku 5.5 costs more above this many prompt tokens
FALLBACK_MODELS = {"claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-2"}
FALLBACK_BETA = "server-side-fallback-2026-07-01"

STAMP = {"type": "string", "description": "Timestamp from the transcript, as m:ss or h:mm:ss"}
SCHEMA = {
    "type": "object",
    "additionalProperties": False,
    "required": ["summary", "topics", "claims", "mentions", "questions", "call_to_action", "sentiment", "audience", "answer"],
    "properties": {
        "summary": {"type": "string", "description": "Two or three sentences: who speaks and what the video is about"},
        "topics": {"type": "array", "items": {"type": "string"}, "description": "3-8 short topic tags, lowercase, 1-3 words each"},
        "claims": {"type": "array", "items": {
            "type": "object", "additionalProperties": False, "required": ["claim", "timestamp"],
            "properties": {"claim": {"type": "string"}, "timestamp": STAMP}}},
        "mentions": {"type": "array", "items": {
            "type": "object", "additionalProperties": False, "required": ["name", "kind", "context", "timestamp"],
            "properties": {"name": {"type": "string"},
                           "kind": {"type": "string", "enum": ["brand", "product", "person", "organization"]},
                           "context": {"type": "string"}, "timestamp": STAMP}}},
        "questions": {"type": "array", "items": {"type": "string"}, "description": "Questions the video answers"},
        "call_to_action": {"type": "string", "description": "What the speaker asks viewers to do, or empty"},
        "sentiment": {"type": "string", "enum": ["positive", "neutral", "negative", "mixed"]},
        "audience": {"type": "string", "description": "Who the video is for, in a few words"},
        "answer": {"type": "string", "description": "Answer to the extra question, or empty if there is none"},
    },
}

SYSTEM = """You analyze YouTube videos from their transcripts and return structured data.

Rules:
- Use only what the transcript says. Never add facts, names or numbers from elsewhere.
- Claims are specific statements the speaker presents as true: numbers, results, comparisons, recommendations. Up to 10, most important first.
- Mentions are brands, products, people and organizations named in the video. Up to 15. Keep the spelling from the transcript.
- Every claim and mention carries the timestamp where it is said, copied from the transcript line it comes from.
- Captions are automatic and can misspell names. Do not correct them silently; keep the transcript's wording.
- If the transcript is too short or garbled to analyze, say so in the summary and leave the lists empty."""

PROMPT = """Video: {title}
{question}
<transcript>
{text}
</transcript>"""


class ApiError(Exception):
    def __init__(self, status: int, code: str, detail):
        super().__init__(f"{status} {code}: {detail}")
        self.status, self.code, self.detail = status, code, detail


def call(path: str, params: dict, key: str) -> dict:
    """GET one YouTubeTranscriptAPI endpoint. Retries 408/429/503 and network errors, honoring Retry-After."""
    url = f"{BASE}{path}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}", "Accept": "application/json"})
    for attempt in range(1, MAX_ATTEMPTS + 1):
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except urllib.error.HTTPError as e:
            try:
                body = json.loads(e.read().decode("utf-8") or "{}")
            except ValueError:
                body = {}
            if e.code in (408, 429, 503) and attempt < MAX_ATTEMPTS:
                time.sleep(_retry_after(e.headers.get("Retry-After"), attempt))
                continue
            raise ApiError(e.code, body.get("code") or body.get("error") or "error", body.get("detail")) from None
        except (urllib.error.URLError, TimeoutError):
            if attempt < MAX_ATTEMPTS:
                time.sleep(2 ** attempt)
                continue
            raise
    raise RuntimeError("unreachable")


def _retry_after(value: str | None, attempt: int) -> float:
    try:
        return max(1.0, min(float(value), 120.0))
    except (TypeError, ValueError):
        return float(2 ** attempt)


def video_id(text: str) -> str | None:
    m = ID_RE.search(text.strip())
    return (m.group(1) or m.group(2)) if m else None


def stamp(seconds: float) -> str:
    s = int(seconds)
    h, m, s = s // 3600, (s % 3600) // 60, s % 60
    return f"{h}:{m:02d}:{s:02d}" if h else f"{m}:{s:02d}"


def to_seconds(text: str) -> int | None:
    text = (text or "").strip().strip("[]")
    if not re.fullmatch(r"\d{1,2}(:\d{1,2}){1,2}", text):
        return None
    secs = 0
    for part in text.split(":"):
        secs = secs * 60 + int(part)
    return secs


def segments_of(data: dict) -> list[dict]:
    t = data.get("transcript")
    return [s for s in t if (s.get("text") or "").strip()] if isinstance(t, list) else []


def render(segments: list[dict], every: float = 30.0) -> str:
    """Transcript as lines that each start with the [m:ss] of their first caption, about every 30 seconds."""
    lines, current, line_start = [], [], None
    for seg in segments:
        start = float(seg.get("start") or 0)
        if line_start is None:
            line_start = start
        if current and start - line_start >= every:
            lines.append(f"[{stamp(line_start)}] {' '.join(current)}")
            current, line_start = [], start
        current.append(" ".join(seg["text"].split()))
    if current:
        lines.append(f"[{stamp(line_start)}] {' '.join(current)}")
    return "\n".join(lines)


def duration_of(segments: list[dict]) -> float:
    if not segments:
        return 0.0
    last = segments[-1]
    return float(last.get("start") or 0) + float(last.get("duration") or 0)


def est_tokens(text: str) -> int:
    return max(1, len(text) // 4)  # rough: about 4 characters per English token


def price(model: str, input_tokens: int, output_tokens: int) -> float | None:
    if model not in PRICES:
        return None
    short, long_ = PRICES[model]
    p_in, p_out = long_ if input_tokens > LONG_PROMPT else short
    return (input_tokens * p_in + output_tokens * p_out) / 1_000_000


def link(vid: str, ts: str, limit: float) -> tuple[str, bool]:
    """Link to the moment, and whether the timestamp is valid for this video."""
    secs = to_seconds(ts)
    if secs is None or (limit and secs > limit + 5):
        return "", False
    return f"https://www.youtube.com/watch?v={vid}&t={secs}s", True


class Analyzer:
    def __init__(self, client, model: str, effort: str):
        self.client, self.model, self.effort = client, model, effort
        self.input_tokens = self.output_tokens = self.calls = 0
        self.cost = 0.0

    def analyze(self, title: str, text: str, question: str) -> dict:
        q = f"Extra question to answer in the 'answer' field: {question}\n" if question else "No extra question; leave 'answer' empty.\n"
        kwargs = dict(
            model=self.model,
            max_tokens=16000,
            system=SYSTEM,
            messages=[{"role": "user", "content": PROMPT.format(title=title, question=q, text=text)}],
            output_config={"effort": self.effort, "format": {"type": "json_schema", "schema": SCHEMA}},
        )
        if self.model in FALLBACK_MODELS:
            kwargs.update(betas=[FALLBACK_BETA], fallbacks="default")
        resp = self.client.beta.messages.create(**kwargs)
        self.calls += 1
        self.input_tokens += resp.usage.input_tokens
        self.output_tokens += resp.usage.output_tokens
        self.cost += price(getattr(resp, "model", None) or self.model, resp.usage.input_tokens, resp.usage.output_tokens) or 0.0
        if resp.stop_reason == "refusal":
            raise RuntimeError("the model declined to analyze this transcript")
        if resp.stop_reason == "max_tokens":
            raise RuntimeError("the analysis hit the output limit")
        text_out = next(b.text for b in resp.content if getattr(b, "type", "") == "text")
        return json.loads(text_out)


def views_number(text: str | None) -> int | None:
    """'3.2M views 2 weeks ago' -> 3200000; None when unknown."""
    m = re.search(r"([\d.,]+)\s*([KMB]?)\s*views", text or "", re.I)
    if not m:
        return None
    n = float(m.group(1).replace(",", ""))
    return int(n * {"": 1, "K": 1e3, "M": 1e6, "B": 1e9}[m.group(2).upper()])


def write_outputs(out: Path, rows: list[dict]) -> None:
    def w(name, header, items):
        with open(out / name, "w", newline="", encoding="utf-8") as f:
            wr = csv.writer(f)
            wr.writerow(header)
            wr.writerows(items)

    w("videos.csv", ["video_id", "title", "views_text", "views", "length", "sentiment", "audience", "topics", "mentions", "call_to_action", "answer", "summary"],
      [[r["video_id"], r["title"], r.get("views_text", ""), r.get("views") or "", r["length"], r["sentiment"], r["audience"],
        "; ".join(r["topics"]), "; ".join(sorted({m["name"] for m in r["mentions"]})), r["call_to_action"], r["answer"], r["summary"]] for r in rows])
    w("claims.csv", ["video_id", "title", "timestamp", "link", "claim"],
      [[r["video_id"], r["title"], c["timestamp"], c["link"], c["claim"]] for r in rows for c in r["claims"]])
    w("mentions.csv", ["video_id", "title", "name", "kind", "timestamp", "link", "context"],
      [[r["video_id"], r["title"], m["name"], m["kind"], m["timestamp"], m["link"], m["context"]] for r in rows for m in r["mentions"]])

    topic_videos = collections.defaultdict(set)
    for r in rows:
        for t in r["topics"]:
            topic_videos[t.strip().lower()].add(r["video_id"])
    topics = sorted(topic_videos.items(), key=lambda kv: (-len(kv[1]), kv[0]))
    w("topics.csv", ["topic", "videos", "video_ids"], [[t, len(v), " ".join(sorted(v))] for t, v in topics])

    mention_videos = collections.defaultdict(set)
    first_link = {}
    for r in rows:
        for m in r["mentions"]:
            k = m["name"].strip()
            mention_videos[k.lower()].add(r["video_id"])
            first_link.setdefault(k.lower(), (k, m["link"]))
    mentions = sorted(mention_videos.items(), key=lambda kv: (-len(kv[1]), kv[0]))
    sentiments = collections.Counter(r["sentiment"] for r in rows)
    flagged = sum(r["bad_timestamps"] for r in rows)

    lines = [f"# Analysis of {len(rows)} videos", "",
             "Sentiment: " + ", ".join(f"{k} {v}" for k, v in sentiments.most_common()), "",
             "## Top topics", "", "| Topic | Videos |", "|---|---|"]
    lines += [f"| {t} | {len(v)} |" for t, v in topics[:15]]
    lines += ["", "## Most mentioned", "", "| Name | Videos | First mention |", "|---|---|---|"]
    for k, v in mentions[:15]:
        name, url = first_link[k]
        lines.append(f"| {name} | {len(v)} | {url or 'n/a'} |")
    ctas = [r for r in rows if r["call_to_action"]]
    lines += ["", "## Calls to action", ""] + [f"- {r['title']}: {r['call_to_action']}" for r in ctas[:20]]
    if any(r["answer"] for r in rows):
        lines += ["", "## Answers to your question", ""] + [f"- {r['title']}: {r['answer']}" for r in rows if r["answer"]]
    lines += ["", f"Timestamps that didn't match the video (no link, check by hand): {flagged}", ""]
    (out / "report.md").write_text("\n".join(lines), encoding="utf-8")


def main(argv: list[str] | None = None, client=None) -> int:
    p = argparse.ArgumentParser(description="Analyze what YouTube videos say, as structured data.")
    p.add_argument("videos", nargs="*", help="video IDs or URLs")
    p.add_argument("--file", help="text file with one video ID or URL per line")
    p.add_argument("--playlist", help="playlist ID or URL (20 credits, up to 100 videos)")
    p.add_argument("--channel", help="channel @handle, URL or UC ID: its 100 newest uploads (20 credits)")
    p.add_argument("--model", default="claude-opus-5-5", help="Claude model (default claude-opus-5-5; claude-haiku-5-5 is the cheap bulk option)")
    p.add_argument("--effort", default="low", choices=["low", "medium", "high"], help="model effort (default low)")
    p.add_argument("--question", default="", help="an extra question answered for every video")
    p.add_argument("--language", default="en", help="preferred caption language (default en)")
    p.add_argument("--estimate", action="store_true", help="fetch transcripts and estimate LLM cost; no LLM calls")
    p.add_argument("--out", default="analysis", help="output folder")
    a = p.parse_args(argv)

    key = os.environ.get("YT_API_KEY")
    if not key:
        print("Set YT_API_KEY first (free key, 100 credits: https://youtubetranscriptapi.com/signup)", file=sys.stderr)
        return 2
    if not a.estimate and client is None:
        if not os.environ.get("ANTHROPIC_API_KEY"):
            print("Set ANTHROPIC_API_KEY first, or run with --estimate.", file=sys.stderr)
            return 2
        import anthropic  # pip install anthropic
        client = anthropic.Anthropic(max_retries=4)

    videos: list[dict] = []
    raw = list(a.videos)
    if a.file:
        raw += [ln for ln in Path(a.file).read_text(encoding="utf-8").splitlines() if ln.strip() and not ln.startswith("#")]
    for item in raw:
        vid = video_id(item)
        if vid:
            videos.append({"video_id": vid, "title": ""})
        else:
            print(f"Skipping, not a video ID or URL: {item}", file=sys.stderr)

    credits = 0
    try:
        balance = call("/account/whoami", {}, key)["credits"]["spendable"]
        for kind, value in (("playlist", a.playlist), ("channel", a.channel)):
            if not value:
                continue
            listing = call("/youtube/playlist/videos" if kind == "playlist" else "/youtube/channel/videos", {kind: value}, key)
            credits += LISTING_COST
            for r in listing.get("results", []):
                if r.get("videoId"):
                    videos.append({"video_id": r["videoId"], "title": r.get("title") or "", "views_text": r.get("viewCountText") or "",
                                   "views": views_number(r.get("viewCountText"))})
            if listing.get("has_more"):
                print(f"Note: the {kind} has more than the {len(listing.get('results', []))} videos listed here.")
    except ApiError as e:
        print(f"Request failed (not charged): {e}", file=sys.stderr)
        return 1

    seen, unique = set(), []
    for v in videos:
        if v["video_id"] not in seen:
            seen.add(v["video_id"])
            unique.append(v)
    if not unique:
        print("No videos to process.", file=sys.stderr)
        return 2
    print(f"{len(unique)} videos. Balance {balance} credits; transcripts cost at most {len(unique)}.")

    out = Path(a.out)
    (out / "transcripts").mkdir(parents=True, exist_ok=True)
    (out / "json").mkdir(parents=True, exist_ok=True)
    analyzer = Analyzer(client, a.model, a.effort)
    rows, no_captions, failed = [], [], []
    est_in, est_cost = 0, {}
    for n, v in enumerate(unique, 1):
        vid = v["video_id"]
        done_path = out / "json" / f"{vid}.json"
        if done_path.exists() and not a.estimate:
            rows.append(json.loads(done_path.read_text(encoding="utf-8")))
            continue
        cached = out / "transcripts" / f"{vid}.json"
        if cached.exists():
            data = json.loads(cached.read_text(encoding="utf-8"))
        else:
            try:
                data = call("/youtube/transcript", {"video_url": vid, "include_timestamp": "true",
                                                    "send_metadata": "true", "language": a.language}, key)
            except ApiError as e:
                if e.status == 404:
                    no_captions.append(vid)
                    print(f"[{n}/{len(unique)}] {vid}: no captions (free)")
                    continue
                if e.status == 402:
                    print(f"Out of credits after {n - 1} videos; stopping. Re-run after topping up to resume.", file=sys.stderr)
                    break
                failed.append({"videoId": vid, "error": f"{e.status} {e.code}"})
                print(f"[{n}/{len(unique)}] {vid}: {e}", file=sys.stderr)
                continue
            credits += 1
            cached.write_text(json.dumps(data), encoding="utf-8")
        segs = segments_of(data)
        if not segs:
            no_captions.append(vid)
            continue
        title = v["title"] or (data.get("metadata") or {}).get("title") or vid
        text = render(segs)
        if a.estimate:
            tokens = est_tokens(text) + 900  # transcript plus instructions and schema
            est_in += tokens
            for m in PRICES:  # about 1,200 output tokens per analysis, priced per request as the API does
                est_cost[m] = est_cost.get(m, 0.0) + (price(m, tokens, 1200) or 0.0)
            print(f"[{n}/{len(unique)}] {vid}: {stamp(duration_of(segs))}, ~{tokens:,} tokens")
            continue
        print(f"[{n}/{len(unique)}] {vid}: analyzing {title[:60]}")
        try:
            result = analyzer.analyze(title, text, a.question)
        except Exception as e:  # keep going; the SDK already retried transient errors
            failed.append({"videoId": vid, "error": str(e)[:200]})
            print(f"    failed: {e}", file=sys.stderr)
            continue
        limit = duration_of(segs)
        bad = 0
        for item in result.get("claims", []) + result.get("mentions", []):
            item["link"], ok = link(vid, item.get("timestamp", ""), limit)
            bad += not ok
        row = {"video_id": vid, "title": title, "views_text": v.get("views_text", ""), "views": v.get("views"),
               "length": stamp(limit), "caption_language": data.get("language", ""), "model": a.model,
               "bad_timestamps": bad, **result}
        if bad:
            print(f"    check: {bad} timestamp(s) don't match the video; they have no link")
        done_path.write_text(json.dumps(row, ensure_ascii=False, indent=2), encoding="utf-8")
        rows.append(row)

    report = {"videos": len(unique), "analyzed": len(rows), "no_captions": no_captions, "failed": failed,
              "transcript_credits": credits, "model": a.model, "llm_calls": analyzer.calls,
              "input_tokens": analyzer.input_tokens, "output_tokens": analyzer.output_tokens,
              "llm_cost_usd": round(analyzer.cost, 4)}
    if a.estimate:
        report["estimate"] = {"input_tokens": est_in, "llm_cost_usd_by_model": {m: round(c, 4) for m, c in est_cost.items()}}
        print(f"\nEstimated input: ~{est_in:,} tokens. LLM cost by model: " +
              ", ".join(f"{m} ${c:.2f}" for m, c in est_cost.items()) + f". Transcript credits used: {credits}.")
    else:
        if rows:
            write_outputs(out, rows)
        print(f"\nAnalyzed {len(rows)}, no captions {len(no_captions)}, failed {len(failed)}. Transcript credits: {credits}. "
              f"Claude: {analyzer.input_tokens:,} in / {analyzer.output_tokens:,} out, about ${analyzer.cost:.2f}. "
              f"Report: {out / 'report.md'}")
    (out / "run.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
    return 0


if __name__ == "__main__":
    sys.exit(main())
