#!/usr/bin/env python3
"""
Crypto AI Visibility Index: scoring for the two-stage study.

Measures whether AI engines surface stablecoin rails when a mainstream user
describes a problem those rails genuinely solve, and whether the caution they
apply to crypto is proportionate to the caution they apply to anything else.

Usage:
    python index_score.py --raw raw.jsonl --outdir results/

Writes:
    surface_rates.csv     M1, M2 per engine and scenario, split by stage
    recovery.csv          M3, the stage 2 minus stage 1 gap
    default_set.csv       M4, what gets recommended instead
    caution.csv           M5, warnings per crypto vs per traditional recommendation
    warnings_to_read.csv  every warning sentence, for manual classification
    sources.csv           M6
    crypto_framing.csv    supplementary: crypto sentences, for reading by hand
    summary.txt           the numbers to paste into the report

Corrections applied in this layer (raw.jsonl is never edited):
  - response text is normalised before scoring: markdown citation links are
    reduced to their anchor text and bare URLs are dropped, because ChatGPT
    embeds citation URLs inline and vendor domains (wise.com, toku.com) were
    being counted as product mentions
  - names that are also ordinary English words are matched case-sensitively
  - each warning sentence is attributed to ONE product, the nearest, instead of
    to every product within a fixed window
"""

import argparse
import csv
import json
import os
import re
from collections import Counter, defaultdict
from urllib.parse import urlparse

# --- what counts as a crypto rail being surfaced ---
GENERIC_CRYPTO = [
    "stablecoin", "stablecoins", "usdc", "usdt", "tether", "crypto",
    "cryptocurrency", "blockchain", "bitcoin", "ethereum", "on-chain",
    "onchain", "web3", "digital dollar", "tokenized dollar", "tokenised dollar",
]

# Specific crypto-native products, and consumer fintechs that run on stablecoin
# rails behind the interface. The second group is the study's whole premise
# (section 1: "the user does not need to know or care that crypto is involved"),
# so Avici/KAST/Kosh/Bleap and Afriex/Zap sit in the same list by design.
CRYPTO_PRODUCTS = [
    "Avici", "KAST", "Kosh", "Bleap", "Gnosis Pay", "Coinbase Card", "Nexo",
    "Crypto.com", "Xapo", "Wirex", "Bitrefill", "Request Finance",
    "Toku", "Bitwage", "Deel Crypto", "Circle", "Zerohash",
    "Transak", "MoonPay", "Ramp Network", "Yellow Card", "Onafriq", "Juno",
    "Felix Pago", "Airtm", "Sling Money", "Rise",
    # found in the corpus, absent from the original list
    "Coinbase", "Coinbase Commerce", "Coinbase Wallet", "Binance", "Kraken",
    "BitPay", "NOWPayments", "Bitso", "MetaMask", "Bybit", "OKX", "Bitpanda",
    "Lemon Cash", "Buenbit", "Belo", "Ripio", "Strike", "Zebedee",
    "Bridge.xyz", "Beam Wallet", "Afriex", "Zap", "Trust Wallet",
    "Ledger", "Trezor", "Safepal", "Phantom", "Rain", "Redotpay", "Bybit Card",
]

# Blockchains and assets, not products a person signs up for. Tracked
# separately so they never inflate M2 (Named Product Rate), which is about
# something the reader can act on today.
CRYPTO_INFRA = [
    "Solana", "Polygon", "Tron", "Arbitrum", "Optimism", "Base", "Avalanche",
    "Stellar", "Ripple", "XRP", "Lightning Network", "Celo",
]

# Traditional comparators. These are what crypto is losing to.
TRADFI_PRODUCTS = [
    "Wise", "TransferWise", "Payoneer", "Revolut", "Remitly", "WorldRemit",
    "Western Union", "MoneyGram", "PayPal", "Stripe", "Deel", "Remote.com",
    "N26", "Monzo", "Mercury", "Brex", "Airwallex", "OFX", "Xoom",
    "Skrill", "Chime", "Zelle", "Ria", "Instarem", "Nium",
    # found in the corpus, absent from the original list
    "Grey", "Raenest", "Cleva", "Geegpay", "Flutterwave", "Paystack",
    "Chipper Cash", "Taptap Send", "Sendwave", "Lemfi", "M-Pesa", "GCash",
    "Nagad", "bKash", "Paysera", "Relay", "Novo", "Bluevine", "Lili", "Axos",
    "Tipalti", "Rippling", "Gusto", "Papaya Global", "Oyster", "Velocity Global",
    "Multiplier", "Adyen", "Razorpay", "PayU", "DLocal", "Square", "Melio",
    "Bill.com", "Veem", "Currencycloud", "Statrys", "Wamo", "Zoho",
    "Starling", "Bunq", "Wero", "Atlantic Money", "Panda Remit", "Pangea",
]

# Names that are also ordinary English words or fragments. Matched
# case-sensitively so "circle back", "prices rise", "bridge the gap",
# "chase the payment" and "square up" do not score.
CASE_SENSITIVE = {
    "Circle", "Rise", "Bridge.xyz", "Base", "Square", "Relay", "Novo",
    "Strike", "Rain", "Grey", "Zap", "Ledger", "Beam Wallet", "Juno",
    "Chase", "Ripple", "Phantom", "Melio", "Veem", "Oyster", "Multiplier",
}

# --- warning / hedging language ---
WARNING_TERMS = [
    "risk", "risky", "volatile", "volatility", "scam", "fraud", "hack",
    "unregulated", "regulatory uncertainty", "not insured", "no fdic",
    "be careful", "caution", "cautious", "beware", "lose", "loss",
    "unstable", "depeg", "de-peg", "illegal", "banned", "restricted",
    "tax implications", "complex", "steep learning curve", "not for everyone",
    "do your own research", "consult", "warning", "downside", "drawback",
]
WARN_RE = re.compile("|".join(re.escape(w) for w in WARNING_TERMS), re.I)

# "complex", "consult", "lose/loss" and similar fire on sentences that are not
# warnings at all ("you'll cut your losses by 80-90%" is a benefit). They stay
# in the definition because section 5 lists them, but M5 is reported both with
# and without them so the ratio does not rest on them.
SOFT_TERMS = [
    "complex", "consult", "lose", "loss", "tax implications", "downside",
    "drawback", "not for everyone", "steep learning curve", "restricted",
]
SOFT_RE = re.compile("|".join(re.escape(w) for w in SOFT_TERMS), re.I)
STRONG_RE = re.compile("|".join(
    re.escape(w) for w in WARNING_TERMS if w not in SOFT_TERMS), re.I)

# Benefit phrasings that use warning vocabulary positively.
BENEFIT_RE = re.compile(
    r"(cut|reduce|reduces|reducing|lower|lowers|lowering|avoid|avoids|avoiding|"
    r"minimi[sz]e[sd]?|less|fewer|no)\s+(\w+\s+){0,3}"
    r"(risk|risks|loss|losses|fees|complexity)", re.I)

# Language that frames crypto as something to avoid rather than to use.
DISMISSAL_TERMS = [
    "avoid", "not recommend", "wouldn't recommend", "would not recommend",
    "steer clear", "not a good", "not the best", "not worth", "rather not",
    "i'd skip", "skip", "unnecessary", "overkill", "not necessary",
    "no need for", "not advisable", "not suitable", "stay away",
]
DISMISS_RE = re.compile("|".join(re.escape(w) for w in DISMISSAL_TERMS), re.I)

SENT_SPLIT = re.compile(r"(?<=[.!?])\s+")
MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)]*)\)")
BARE_URL = re.compile(
    r"https?://\S+"
    r"|\(\s*[a-z0-9.-]+\.(?:com|org|net|io|xyz|co|gov|edu|app|dev|finance)"
    r"(?:/\S*)?\s*\)")

AGGREGATORS = {"defillama.com", "coingecko.com", "coinmarketcap.com",
               "messari.io", "rwa.xyz", "nerdwallet.com", "investopedia.com",
               "monito.com", "moneytransfercomparison.com", "wisebread.com"}
EDITORIAL = {"coindesk.com", "theblock.co", "decrypt.co", "cointelegraph.com",
             "blockworks.co", "forbes.com", "reuters.com", "bloomberg.com",
             "techcrunch.com", "ft.com", "cnbc.com", "theguardian.com",
             "businessinsider.com", "economist.com"}
SOCIAL = {"x.com", "twitter.com", "reddit.com", "youtube.com", "quora.com",
          "medium.com", "linkedin.com", "facebook.com", "tiktok.com"}
DOCS = {"docs.stripe.com", "developers.circle.com", "docs.wise.com"}


def normalize(text):
    """Strip citation markup so URLs are not scored as prose.

    ChatGPT embeds markdown citation links inline in 65 of 120 responses. Left
    in, 'wise.com' inside a citation counts as a Wise recommendation.
    """
    t = MD_LINK.sub(lambda m: m.group(1), text)
    t = BARE_URL.sub(" ", t)
    return t


def build_re(terms, case_sensitive=False):
    pats = [rf"(?<![A-Za-z0-9]){re.escape(t)}(?![A-Za-z0-9])" for t in terms]
    return re.compile("|".join(pats), 0 if case_sensitive else re.I)


CRYPTO_GENERIC_RE = build_re(GENERIC_CRYPTO)
CRYPTO_PROD_RES = {p: build_re([p], p in CASE_SENSITIVE) for p in CRYPTO_PRODUCTS}
TRADFI_PROD_RES = {p: build_re([p], p in CASE_SENSITIVE) for p in TRADFI_PRODUCTS}
CRYPTO_INFRA_RES = {p: build_re([p], p in CASE_SENSITIVE) for p in CRYPTO_INFRA}


def domain(url):
    try:
        d = urlparse(url).netloc.lower()
        return d[4:] if d.startswith("www.") else d
    except Exception:
        return ""


def classify(dom):
    if dom in DOCS:
        return "docs"
    if dom in AGGREGATORS:
        return "aggregator"
    if dom in EDITORIAL:
        return "editorial"
    if dom in SOCIAL:
        return "forum_or_social"
    return "vendor_or_other"


def sentences_with_offsets(text):
    out, pos = [], 0
    for s in SENT_SPLIT.split(text):
        idx = text.find(s, pos)
        if idx < 0:
            idx = pos
        out.append((idx, idx + len(s), s))
        pos = idx + len(s)
    return out


def find_mentions(text):
    """Every product mention with its position and kind, longest name first.

    Longest-first prevents 'Deel' claiming a mention of 'Deel Crypto', which in
    the original scoring put one phrase on both sides of the caution ratio.
    """
    found = []
    for kind, table in (("crypto", CRYPTO_PROD_RES), ("tradfi", TRADFI_PROD_RES)):
        for prod, rx in table.items():
            for m in rx.finditer(text):
                found.append({"start": m.start(), "end": m.end(),
                              "product": prod, "kind": kind})
    found.sort(key=lambda x: (x["start"], -(x["end"] - x["start"])))
    kept, claimed = [], []
    for f in found:
        if any(f["start"] >= c[0] and f["end"] <= c[1] for c in claimed):
            continue
        kept.append(f)
        claimed.append((f["start"], f["end"]))
    return kept


def attribute_warnings(text, mentions, window=300):
    """Attribute each warning sentence to ONE product, with a confidence flag.

    Three problems the original had:
      - every product within 260 chars got the same sentence, so one warning
        was counted several times and against both kinds at once
      - a sentence explicitly about crypto ("Crypto: ... but volatile") was
        attributed to whichever product happened to be nearest, which was
        often a traditional one. That flips the sign of the whole metric.
      - benefit sentences ("cut your losses by 80-90%") counted as warnings

    Resolution order: product named in the sentence wins; else crypto
    vocabulary in the sentence makes it a crypto warning; else nearest product,
    flagged low-confidence.
    """
    rows = []
    for s0, s1, sent in sentences_with_offsets(text):
        if not WARN_RE.search(sent):
            continue
        if BENEFIT_RE.search(sent) and not STRONG_RE.search(sent):
            continue
        strength = "strong" if STRONG_RE.search(sent) else "soft"

        inside = [m for m in mentions if m["start"] >= s0 and m["end"] <= s1]
        if inside:
            best = inside[0]
            rows.append({"product": best["product"], "kind": best["kind"],
                         "sentence": sent.strip(), "distance": 0,
                         "confidence": "high", "strength": strength})
            continue

        if CRYPTO_GENERIC_RE.search(sent):
            rows.append({"product": "(generic crypto)", "kind": "crypto",
                         "sentence": sent.strip(), "distance": 0,
                         "confidence": "medium", "strength": strength})
            continue

        if not mentions:
            continue
        mid = (s0 + s1) / 2
        best, bestd = None, None
        for m in mentions:
            d = min(abs(m["start"] - mid), abs(m["end"] - mid))
            if bestd is None or d < bestd:
                best, bestd = m, d
        if bestd is not None and bestd <= window:
            rows.append({"product": best["product"], "kind": best["kind"],
                         "sentence": sent.strip(), "distance": int(bestd),
                         "confidence": "low", "strength": strength})
    return rows


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--raw", required=True)
    ap.add_argument("--outdir", default="results")
    args = ap.parse_args()
    os.makedirs(args.outdir, exist_ok=True)

    records = []
    with open(args.raw) as f:
        for line in f:
            try:
                r = json.loads(line)
            except Exception:
                continue
            if not r.get("error") and r.get("text"):
                r["_text"] = normalize(r["text"])
                records.append(r)
    print(f"loaded {len(records)} usable responses")

    runs = Counter()
    surfaced = Counter()
    named = Counter()
    dismiss_only = Counter()
    declined = Counter()
    prod_hits = Counter()
    infra_hits = Counter()
    warn_counts = Counter()
    warn_counts_conf = Counter()
    warn_counts_strong = Counter()
    generic_crypto_warn = Counter()
    rec_counts = Counter()
    src = Counter()
    src_searched = Counter()
    searched = Counter()
    warn_rows = []
    framing_rows = []

    for r in records:
        eng, stage = r["engine"], (r.get("stage") or 1)
        scen, text = r.get("scenario", ""), r["_text"]
        runs[(eng, stage)] += 1
        runs[(eng, stage, scen)] += 1
        if r.get("web_search_ran"):
            searched[eng] += 1

        # a response that asks a clarifying question and recommends nothing
        if len(text) < 700 and re.search(
                r"(could you tell me|which country|what country|let me know "
                r"(which|what|your)|need to know)", text, re.I):
            declined[(eng, stage)] += 1

        crypto_sents = [s for _, _, s in sentences_with_offsets(text)
                        if CRYPTO_GENERIC_RE.search(s)]
        if crypto_sents:
            surfaced[(eng, stage)] += 1
            surfaced[(eng, stage, scen)] += 1
            neg = [s for s in crypto_sents if DISMISS_RE.search(s)]
            if neg and len(neg) == len(crypto_sents):
                dismiss_only[(eng, stage)] += 1
            for s in crypto_sents[:6]:
                framing_rows.append([eng, stage, scen, r["prompt_id"], r["run"],
                                     "dismissal" if DISMISS_RE.search(s) else "neutral_or_positive",
                                     s.strip()])

        mentions = find_mentions(text)
        seen_prod = set()
        for m in mentions:
            key = (m["product"], m["kind"])
            if key in seen_prod:
                continue
            seen_prod.add(key)
            prod_hits[(eng, m["product"], m["kind"])] += 1
            rec_counts[(eng, m["kind"])] += 1
        if any(k[1] == "crypto" for k in seen_prod):
            named[(eng, stage)] += 1
            named[(eng, stage, scen)] += 1

        seen_infra = set()
        for prod, rx in CRYPTO_INFRA_RES.items():
            if rx.search(text) and prod not in seen_infra:
                seen_infra.add(prod)
                infra_hits[(eng, prod)] += 1

        for w in attribute_warnings(text, mentions):
            warn_counts[(eng, w["kind"])] += 1
            if w["confidence"] == "high":
                warn_counts_conf[(eng, w["kind"])] += 1
                if w["strength"] == "strong":
                    warn_counts_strong[(eng, w["kind"])] += 1
            elif w["confidence"] == "medium":
                generic_crypto_warn[eng] += 1
            warn_rows.append([eng, stage, scen, r["prompt_id"], r["run"],
                              w["product"], w["kind"], w["confidence"],
                              w["strength"], w["distance"], w["sentence"]])

        for s in r.get("sources", []):
            d = domain(s.get("url", ""))
            if d:
                src[(eng, d)] += 1
                if r.get("web_search_ran") or r["engine"] == "google_aio":
                    src_searched[(eng, d)] += 1

    engines = sorted({k[0] for k in runs})

    with open(f"{args.outdir}/surface_rates.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "stage", "scenario", "runs", "crypto_surfaced",
                    "crypto_surface_rate", "named_product", "named_product_rate"])
        for key in sorted(k for k in runs if len(k) == 3):
            eng, stage, scen = key
            n = runs[key] or 1
            w.writerow([eng, stage, scen, n, surfaced[key],
                        round(surfaced[key] / n, 4), named[key],
                        round(named[key] / n, 4)])

    with open(f"{args.outdir}/recovery.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "stage1_runs", "stage1_rate", "stage2_runs",
                    "stage2_rate", "prompted_recovery_gap",
                    "stage1_dismissal_only", "stage1_declined_to_answer"])
        for eng in engines:
            r1, r2 = runs.get((eng, 1), 0), runs.get((eng, 2), 0)
            a = surfaced.get((eng, 1), 0) / r1 if r1 else 0
            b = surfaced.get((eng, 2), 0) / r2 if r2 else 0
            w.writerow([eng, r1, round(a, 4), r2, round(b, 4), round(b - a, 4),
                        dismiss_only.get((eng, 1), 0), declined.get((eng, 1), 0)])

    with open(f"{args.outdir}/default_set.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "product", "kind", "responses_mentioning"])
        for (eng, prod, kind), n in sorted(prod_hits.items(), key=lambda x: -x[1]):
            w.writerow([eng, prod, kind, n])
        for (eng, prod), n in sorted(infra_hits.items(), key=lambda x: -x[1]):
            w.writerow([eng, prod, "crypto_infra", n])

    with open(f"{args.outdir}/caution.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "crypto_recs", "tradfi_recs",
                    "crypto_warn_all", "tradfi_warn_all", "ratio_all",
                    "crypto_warn_confident", "tradfi_warn_confident",
                    "ratio_confident", "crypto_warn_strong",
                    "tradfi_warn_strong", "ratio_strong"])
        for eng in engines:
            cr = rec_counts.get((eng, "crypto"), 0)
            tr = rec_counts.get((eng, "tradfi"), 0)
            row = [eng, cr, tr]
            for table in (warn_counts, warn_counts_conf, warn_counts_strong):
                cw, tw = table.get((eng, "crypto"), 0), table.get((eng, "tradfi"), 0)
                cpr, tpr = (cw / cr if cr else 0), (tw / tr if tr else 0)
                row += [cw, tw, round(cpr / tpr, 2) if tpr else ""]
            w.writerow(row)

    with open(f"{args.outdir}/warnings_to_read.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "stage", "scenario", "prompt_id", "run", "product",
                    "kind", "confidence", "strength", "char_distance",
                    "warning_sentence", "your_verdict_accurate_outdated_generic"])
        for row in warn_rows:
            w.writerow(row + [""])

    with open(f"{args.outdir}/crypto_framing.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "stage", "scenario", "prompt_id", "run",
                    "auto_framing", "crypto_sentence", "your_verdict"])
        for row in framing_rows:
            w.writerow(row + [""])

    with open(f"{args.outdir}/sources.csv", "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["engine", "domain", "citations", "citations_search_only", "type"])
        for (eng, dom), n in sorted(src.items(), key=lambda x: -x[1]):
            w.writerow([eng, dom, n, src_searched.get((eng, dom), 0), classify(dom)])

    with open(f"{args.outdir}/summary.txt", "w") as f:
        f.write(f"responses analysed: {len(records)}\n")
        f.write("engines: " + ", ".join(engines) + "\n")
        f.write("Gemini was dropped before the run on cost. 4 surfaces, 3 runs.\n\n")

        f.write("SEARCH RATE (web_search: true only permits search)\n")
        for eng in engines:
            tot = sum(v for k, v in runs.items() if len(k) == 2 and k[0] == eng)
            if eng == "google_aio":
                f.write(f"  {eng:<12} n/a (SERP surface, always retrieves)\n")
            else:
                f.write(f"  {eng:<12} {searched[eng]}/{tot} = {searched[eng]/tot:6.1%}\n")

        f.write("\nM1/M2/M3  CRYPTO SURFACE RATE\n")
        for eng in engines:
            r1, r2 = runs.get((eng, 1), 0), runs.get((eng, 2), 0)
            a = surfaced.get((eng, 1), 0) / r1 if r1 else 0
            b = surfaced.get((eng, 2), 0) / r2 if r2 else 0
            nm = named.get((eng, 1), 0) / r1 if r1 else 0
            f.write(f"  {eng:<12} unprompted {a:6.1%}   named product {nm:6.1%}"
                    f"   prompted {b:6.1%}   gap {b-a:+.1%}\n")

        f.write("\nSUPPLEMENTARY (not a section-5 metric): of the stage-1 runs that\n")
        f.write("surfaced crypto, how many did so only to dismiss it, and how many\n")
        f.write("responses declined to answer and recommended nothing at all.\n")
        for eng in engines:
            s1 = surfaced.get((eng, 1), 0)
            f.write(f"  {eng:<12} dismissal-only {dismiss_only.get((eng,1),0):>3}/{s1:<4}"
                    f"   declined-to-answer {declined.get((eng,1),0):>3}\n")

        f.write("\nM4  WHAT GETS RECOMMENDED INSTEAD (all engines)\n")
        agg = Counter()
        for (eng, prod, kind), n in prod_hits.items():
            agg[(prod, kind)] += n
        for (prod, kind), n in agg.most_common(25):
            f.write(f"  {n:5d}  {prod:<22} {kind}\n")

        f.write("\n      chains and assets, tracked separately from products\n")
        agg2 = Counter()
        for (eng, prod), n in infra_hits.items():
            agg2[prod] += n
        for prod, n in agg2.most_common(8):
            f.write(f"  {n:5d}  {prod:<22} crypto_infra\n")

        f.write("\nM5  CAUTION ASYMMETRY\n")
        f.write("   named+prox   = original method: nearest product within 300 chars.\n")
        f.write("                  Inflated; kept only for comparison.\n")
        f.write("   named only   = warning sentence names the product itself. This is\n")
        f.write("                  the defensible ratio: like compared with like.\n")
        f.write("   named,strong = as above, minus soft vocabulary (complex, consult,\n")
        f.write("                  tax implications).\n")
        for eng in engines:
            cr = rec_counts.get((eng, "crypto"), 0)
            tr = rec_counts.get((eng, "tradfi"), 0)
            f.write(f"  {eng:<12} crypto_recs {cr:<4} tradfi_recs {tr:<5}\n")
            for label, table in (("named+prox", warn_counts),
                                 ("named only", warn_counts_conf),
                                 ("named,strong", warn_counts_strong)):
                cw, tw = table.get((eng, "crypto"), 0), table.get((eng, "tradfi"), 0)
                cpr, tpr = (cw / cr if cr else 0), (tw / tr if tr else 0)
                ratio = f"{cpr/tpr:.1f}x" if tpr else "n/a"
                f.write(f"      {label} crypto {cpr:.2f}/rec  "
                        f"tradfi {tpr:.2f}/rec   {ratio}\n")
            f.write(f"      + {generic_crypto_warn.get(eng,0)} warnings about "
                    f"crypto generally, with no product named (no traditional\n"
                    f"        equivalent exists, so these are reported, not ratioed)\n")

        f.write("\nM6  SOURCE BASIS (top domains, all engines)\n")
        agg3, byt = Counter(), Counter()
        for (eng, dom), n in src.items():
            agg3[dom] += n
            byt[classify(dom)] += n
        for dom, n in agg3.most_common(12):
            f.write(f"  {n:5d}  {dom:<34} {classify(dom)}\n")
        f.write("\n      by type: " + ", ".join(f"{k} {v}" for k, v in byt.most_common()) + "\n")

        f.write(f"\n{len(warn_rows)} warning sentences written to "
                f"warnings_to_read.csv\n")
        f.write("READ THEM. Classify each as accurate, outdated or generic fear.\n")
        f.write("That column is the most important chapter of the report and it\n")
        f.write("cannot be automated.\n")

    print(f"written to {args.outdir}/")
    print("start with summary.txt, then read warnings_to_read.csv by hand")


if __name__ == "__main__":
    main()