#!/usr/bin/env python3
"""Offline, standard-library analysis. Rebuilds every observed-data product from raw bytes."""
from pathlib import Path
from collections import defaultdict, Counter
import csv, gzip, hashlib, io, json, math

ROOT = Path(__file__).resolve().parents[1]
START, END = 2010, 2024
CORE = ["electricity_generation", "fossil_electricity", "low_carbon_electricity"]
FIELDS = ["country", "iso_code", "year", *CORE, "electricity_demand", "net_elec_imports",
          "nuclear_electricity", "hydro_electricity", "wind_electricity", "solar_electricity",
          "biofuel_electricity", "other_renewable_exc_biofuel_electricity", "other_renewable_electricity",
          "coal_electricity", "gas_electricity", "oil_electricity", "renewables_electricity", "fossil_share_elec"]

def dump(path, value):
    (ROOT / path).write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")

def write_csv(path, rows, fields=None):
    if not rows:
        return
    with (ROOT / path).open("w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=fields or list(rows[0]), extrasaction="ignore")
        w.writeheader()
        w.writerows(rows)

def delta(a, b, k):
    return None if a[k] is None or b[k] is None else b[k] - a[k]

def change(a, b):
    elapsed = b["year"] - a["year"]
    d = {"country": a["country"], "iso_code": a["iso_code"], "start_year": a["year"], "end_year": b["year"],
         "start_fossil_share_pct": a["fossil_share_pct"], "end_fossil_share_pct": b["fossil_share_pct"]}
    d["share_change_pp"] = delta(a, b, "fossil_share_pct")
    d["share_change_pp_per_year"] = d["share_change_pp"] / elapsed
    for key, out in [("fossil_electricity", "fossil_change_twh"), ("electricity_demand", "demand_change_twh"),
                     ("low_carbon_electricity", "clean_change_twh"), ("net_elec_imports", "imports_change_twh"),
                     ("electricity_generation", "generation_change_twh"), ("nuclear_electricity", "nuclear_change_twh"),
                     ("hydro_electricity", "hydro_change_twh"), ("wind_electricity", "wind_change_twh"),
                     ("solar_electricity", "solar_change_twh"), ("other_clean_twh", "other_clean_change_twh"),
                     ("coal_electricity", "coal_change_twh"), ("gas_electricity", "gas_change_twh")]:
        d[out] = delta(a, b, key)
    d["fossil_change_twh_per_year"] = d["fossil_change_twh"] / elapsed
    d["demand_contribution_twh"] = d["demand_change_twh"]
    d["clean_contribution_twh"] = -d["clean_change_twh"] if d["clean_change_twh"] is not None else None
    d["trade_contribution_twh"] = -d["imports_change_twh"] if d["imports_change_twh"] is not None else None
    parts = [d[k] for k in ["demand_contribution_twh", "clean_contribution_twh", "trade_contribution_twh"]]
    d["reconciliation_twh"] = d["fossil_change_twh"] - sum(parts) if all(x is not None for x in parts) else None
    return d

def main():
    manifest = json.loads((ROOT / "data/source-manifest.json").read_text())
    data = gzip.decompress((ROOT / "data/raw/owid-energy-data.csv.gz").read_bytes())
    assert hashlib.sha256(data).hexdigest() == manifest["sources"][0]["sha256_uncompressed"]
    raw_rows = list(csv.DictReader(io.StringIO(data.decode())))
    rows, lookup, raw_line_map = [], defaultdict(dict), []
    for line_no, r in enumerate(raw_rows, 2):
        if len(r["iso_code"]) != 3 or not START <= int(r["year"]) <= END:
            continue
        c = {k: r[k] if k in ("country", "iso_code") else int(r[k]) if k == "year" else float(r[k]) if r[k] else None for k in FIELDS}
        c["raw_csv_line"] = line_no
        g, f = c["electricity_generation"], c["fossil_electricity"]
        c["fossil_share_pct"] = 100 * f / g if f is not None and g is not None and g > 0 else None
        clean_parts = [c[k] for k in ["low_carbon_electricity", "nuclear_electricity", "hydro_electricity", "wind_electricity", "solar_electricity"]]
        c["other_clean_twh"] = clean_parts[0] - sum(clean_parts[1:]) if all(v is not None for v in clean_parts) else None
        c["generation_balance_residual_twh"] = g - f - c["low_carbon_electricity"] if all(c[k] is not None for k in CORE) else None
        c["demand_balance_residual_twh"] = c["electricity_demand"] - g - c["net_elec_imports"] if all(c[k] is not None for k in ["electricity_generation", "electricity_demand", "net_elec_imports"]) else None
        assert c["year"] not in lookup[c["iso_code"]], "Duplicate country-year"
        lookup[c["iso_code"]][c["year"]] = c
        rows.append(c)
        raw_line_map.append({"country": c["country"], "iso_code": c["iso_code"], "year": c["year"], "raw_csv_line": line_no})
    coverage, eligible = [], []
    for iso, years in sorted(lookup.items(), key=lambda x: next(iter(x[1].values()))["country"]):
        missing_years = [y for y in range(START, END+1) if y not in years or any(years[y][k] is None for k in CORE)]
        baseline = years.get(START, {}).get("electricity_generation")
        ok = not missing_years and baseline is not None and baseline >= 20
        reasons = []
        if missing_years:
            reasons.append("Missing core data in " + ", ".join(map(str, missing_years)))
        if baseline is None:
            reasons.append("No baseline generation")
        elif baseline < 20:
            reasons.append("2010 generation below 20 TWh")
        if ok:
            eligible.append(iso)
        coverage.append({"iso_code": iso, "country": next(iter(years.values()))["country"], "eligible": ok,
                         "generation_2010_twh": baseline, "complete_core_years": 15-len(missing_years),
                         "reason": "; ".join(reasons) or "Included", "missing_core_years": ",".join(map(str, missing_years))})
    selected = [r for r in rows if r["iso_code"] in eligible]
    ranks = sorted([change(lookup[iso][START], lookup[iso][END]) for iso in eligible], key=lambda r: r["share_change_pp_per_year"])
    abs_order = sorted(ranks, key=lambda r: r["fossil_change_twh"])
    for rank, r in enumerate(ranks, 1):
        r["rank_share_speed"] = rank
        r["rank_absolute_decline"] = next(i for i,x in enumerate(abs_order,1) if x["iso_code"] == r["iso_code"])
    by = {r["iso_code"]: r for r in ranks}
    sens = []
    for start in [2010, 2015, 2020]:
        rr = sorted([change(lookup[i][start], lookup[i][END]) for i in eligible], key=lambda x: x["share_change_pp_per_year"])
        for rank, r in enumerate(rr, 1):
            r["rank"] = rank
            sens.append(r)
    n = len(eligible)
    f1 = by["DNK"]; f2 = by["GBR"]; f3 = by["CHN"]; f4 = by["DEU"]; f5 = by["USA"]; f6 = by["NOR"]
    claims = [
        {"id": "pace", "number": "01", "title": "Denmark changes the mix fastest.", "pair": ["DNK", "CHN"],
         "stat": f"{abs(f1['share_change_pp']):.1f} pp", "label": "Denmark's fossil-share decline",
         "text": f"Denmark's fossil share falls from {f1['start_fossil_share_pct']:.1f}% to {f1['end_fossil_share_pct']:.1f}%, or {abs(f1['share_change_pp_per_year']):.2f} percentage points a year. It ranks first among the {n} eligible grids. Wind and solar add {f1['wind_change_twh']+f1['solar_change_twh']:.1f} TWh.",
         "caveat": "This is a screened cohort, not every country. A percentage-point ranking rewards starting with more fossil generation; smaller grids can move faster.", "rows": ["DNK:2010", "DNK:2024"], "metrics": ["share_change_pp", "share_change_pp_per_year", "wind_change_twh", "solar_change_twh"]},
        {"id": "demand", "number": "02", "title": "Britain's decline has three parts.", "pair": ["GBR", "DEU"],
         "stat": f"{f2['fossil_change_twh']:.1f} TWh", "label": "UK fossil-generation change",
         "text": f"The UK's {abs(f2['fossil_change_twh']):.1f} TWh fall balances with {abs(f2['demand_change_twh']):.1f} TWh less demand, {f2['clean_change_twh']:.1f} TWh more clean generation and {f2['imports_change_twh']:.1f} TWh more net imports.",
         "caveat": "These are accounting contributions, not independent causes. Imports have an electricity mix of their own; the chart does not measure consumption emissions.", "rows": ["GBR:2010", "GBR:2024"], "metrics": ["fossil_change_twh", "demand_change_twh", "clean_change_twh", "imports_change_twh"]},
        {"id": "growth", "number": "03", "title": "A cleaner share can hide more fossil output.", "pair": ["CHN", "IND"],
         "stat": f"+{f3['fossil_change_twh']:,.0f} TWh", "label": "China's fossil-generation growth",
         "text": f"China's fossil share falls {abs(f3['share_change_pp']):.1f} points, yet fossil output rises {f3['fossil_change_twh']:,.0f} TWh. Clean generation grows {f3['clean_change_twh']:,.0f} TWh while demand grows {f3['demand_change_twh']:,.0f} TWh. India shows the same directional pattern.",
         "caveat": "Lower fossil share is not evidence of lower absolute emissions. This analysis measures electricity output, not economy-wide energy use or a causal policy effect.", "rows": ["CHN:2010", "CHN:2024", "IND:2010", "IND:2024"], "metrics": ["share_change_pp", "fossil_change_twh", "clean_change_twh", "demand_change_twh"]},
        {"id": "replacement", "number": "04", "title": "New clean supply also replaces old clean supply.", "pair": ["DEU", "FRA"],
         "stat": f"+{f4['clean_change_twh']:.1f} TWh", "label": "Germany's net clean-generation gain",
         "text": f"Germany adds {f4['wind_change_twh']+f4['solar_change_twh']:.1f} TWh of wind and solar, but nuclear generation falls {abs(f4['nuclear_change_twh']):.1f} TWh. The net gain from all clean sources is only {f4['clean_change_twh']:.1f} TWh.",
         "caveat": "This balance does not estimate the counterfactual without nuclear retirements. France is a useful comparison with a very different starting mix, not a controlled experiment.", "rows": ["DEU:2010", "DEU:2024", "FRA:2010", "FRA:2024"], "metrics": ["wind_change_twh", "solar_change_twh", "nuclear_change_twh", "clean_change_twh"]},
        {"id": "scale", "number": "05", "title": "The leader changes with the unit.", "pair": ["USA", "DNK"],
         "stat": f"{f5['fossil_change_twh']:,.1f} TWh", "label": "US fossil-generation change",
         "text": f"The US records the largest absolute fossil decline in the cohort, at {abs(f5['fossil_change_twh']):,.1f} TWh, but ranks {f5['rank_share_speed']} by share-change speed. Coal falls {abs(f5['coal_change_twh']):,.0f} TWh while gas rises {f5['gas_change_twh']:,.0f} TWh.",
         "caveat": "Coal-to-gas substitution stays inside the fossil category. A fossil-output metric cannot by itself quantify the emissions benefit or cost of that substitution.", "rows": ["USA:2010", "USA:2024", "DNK:2010", "DNK:2024"], "metrics": ["fossil_change_twh", "share_change_pp_per_year", "coal_change_twh", "gas_change_twh"]},
        {"id": "starting", "number": "06", "title": "Starting clean leaves less room to fall.", "pair": ["NOR", "BRA"],
         "stat": f"{f6['start_fossil_share_pct']:.1f}%", "label": "Norway's fossil share in 2010",
         "text": f"Norway starts with just {f6['start_fossil_share_pct']:.1f}% fossil electricity and reaches {f6['end_fossil_share_pct']:.1f}%. Brazil starts at {by['BRA']['start_fossil_share_pct']:.1f}% and ends at {by['BRA']['end_fossil_share_pct']:.1f}%, yet adds {by['BRA']['fossil_change_twh']:.1f} TWh of fossil output as its grid grows.",
         "caveat": "A slow share decline can coexist with a very clean grid. Hydro output varies by year; changing the baseline tests sensitivity, not statistical confidence.", "rows": ["NOR:2010", "NOR:2024", "BRA:2010", "BRA:2024"], "metrics": ["start_fossil_share_pct", "end_fossil_share_pct", "fossil_change_twh"]}
    ]
    for c in claims:
        c["source_url"] = manifest["sources"][0]["url"]
        c["evidence_url"] = "evidence.html#" + c["id"]
    missing = Counter(k for r in selected for k in FIELDS[3:] if r[k] is None)
    audit = {
        "eligible_grids": n, "eligible_rows": len(selected), "snapshot_rows_in_interval": len(rows),
        "candidate_grids": len(coverage), "excluded_grids": len(coverage)-n,
        "interval": [START, END], "elapsed_years": END-START,
        "missing_cells_by_field_in_eligible_rows": dict(missing),
        "max_abs_generation_balance_residual_twh": max(abs(r["generation_balance_residual_twh"]) for r in selected),
        "max_abs_demand_balance_residual_twh": max(abs(r["demand_balance_residual_twh"]) for r in selected if r["demand_balance_residual_twh"] is not None),
        "max_abs_endpoint_reconciliation_twh": max(abs(r["reconciliation_twh"]) for r in ranks if r["reconciliation_twh"] is not None),
        "rows_with_negative_other_clean_residual": sum(r["other_clean_twh"] is not None and r["other_clean_twh"] < -0.02 for r in selected),
        "selection": "Country/territory ISO-3 code; complete generation, fossil and low-carbon data for every 2010-2024 year; generation >=20 TWh in 2010. No population weighting. Fixed cohort for alternate baselines.",
    }
    assert len(selected) == n * 15
    assert {"USA", "CHN", "IND", "DEU", "FRA", "GBR", "BRA", "NOR", "AUS"}.issubset(eligible)
    assert audit["rows_with_negative_other_clean_residual"] == 0
    write_csv("data/clean/electricity-all.csv", rows)
    write_csv("data/clean/electricity.csv", selected)
    write_csv("data/evidence/rankings.csv", ranks)
    write_csv("data/evidence/baseline-sensitivity.csv", sens)
    write_csv("data/evidence/coverage.csv", coverage)
    write_csv("data/evidence/raw-row-index.csv", raw_line_map)
    dump("data/evidence/audit.json", audit)
    dump("data/evidence/claims.json", claims)
    dump("data/atlas.json", {"meta": {**audit, "vintage": "May 2025", "snapshot_commit": manifest["snapshot_commit"],
                                     "retrieved_at_utc": manifest["retrieved_at_utc"]},
                             "countries": [{"iso": iso, "name": lookup[iso][START]["country"], "rows": [lookup[iso][y] for y in range(START, END+1)]} for iso in eligible],
                             "rankings": ranks, "coverage": coverage, "sensitivity": sens, "claims": claims})
    print(json.dumps(audit, indent=2))
    print("Top five:", [(r["country"], round(r["share_change_pp_per_year"], 3)) for r in ranks[:5]])

if __name__ == "__main__":
    main()