"""Week 5: compare incoming Wikipedia links with word-like article tokens.

The unchanged course text archive is read in place. No live Wikipedia requests.
All 303 roster nodes are retained, including zero-in-degree and isolated nodes.
"""
from collections import Counter
from pathlib import Path
from urllib.parse import unquote
import csv
import hashlib
import json
import math
import os
import platform
import re
import statistics as st
import tempfile
import zipfile

os.environ.setdefault("MPLCONFIGDIR", str(Path(tempfile.gettempdir()) / "socialgraphs-week5-matplotlib"))
os.environ.setdefault("XDG_CACHE_HOME", str(Path(tempfile.gettempdir()) / "socialgraphs-week5-cache"))
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "data/week5"
FIG = ROOT / "weeks/week5"
ARCHIVE = ROOT / "data/marvel_pages.zip"
# Unicode letters, with internal straight/curly apostrophes. Hyphens split.
WORD = re.compile(r"[^\W\d_]+(?:['’][^\W\d_]+)*")
CASES = ["Brian_Braddock", "Miracleman_(character)", "Quasar_(character)", "Betsy_Braddock"]


def tokenize(text):
    return WORD.findall(text)


def load():
    with (ROOT / "data/week1/week1_nodes.tsv").open(encoding="utf-8") as f:
        nodes = list(csv.DictReader((l for l in f if not l.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE))
    with (ROOT / "data/week1/week1_edges.tsv").open(encoding="utf-8") as f:
        edges = [tuple(l.strip().split("\t")) for l in f if l.strip() and not l.startswith("#")]
    with zipfile.ZipFile(ARCHIVE) as z:
        members = [n for n in z.namelist() if n.endswith(".txt") and "README" not in n]
        pages = {unquote(Path(n).name[:-4]): z.read(n).decode("utf-8") for n in members}
    ids = {r["node_id"] for r in nodes}
    assert len(nodes) == len(ids) == len(members) == len(pages) == 303
    assert set(pages) == ids, "URL-decode titles before joining, including Mark_Hazzard:_Merc"
    assert len(edges) == len(set(edges)) == 1784
    assert all(a in ids and b in ids and a != b for a, b in edges)
    return nodes, edges, pages


def ranks(values):
    """Average ranks for ties; used for descriptive Spearman correlation."""
    ordered = sorted(enumerate(values), key=lambda x: x[1])
    output = [0.] * len(values)
    i = 0
    while i < len(values):
        j = i + 1
        while j < len(values) and ordered[j][1] == ordered[i][1]:
            j += 1
        for index, _ in ordered[i:j]:
            output[index] = (i + 1 + j) / 2
        i = j
    return output


def spearman(a, b):
    return st.correlation(ranks(a), ranks(b))


def analyse():
    nodes, edges, pages = load()
    incoming = Counter(b for a, b in edges)
    outgoing = Counter(a for a, b in edges)
    rows = []
    for node in nodes:
        key = node["node_id"]
        text = pages[key]
        rows.append({"node_id": key, "name": node["name"], "url": node["url"],
                     "in_degree": incoming[key], "out_degree": outgoing[key],
                     "tokens": len(tokenize(text)), "characters": len(text),
                     "whitespace_tokens": len(text.split())})
    x = [math.log1p(r["in_degree"]) for r in rows]
    y = [math.log(r["tokens"]) for r in rows]
    slope = st.covariance(x, y) / st.variance(x)
    intercept = st.mean(y) - slope * st.mean(x)
    for r in rows:
        r["fitted_tokens"] = math.exp(intercept + slope * math.log1p(r["in_degree"]))
        r["observed_fitted_ratio"] = r["tokens"] / r["fitted_tokens"]
        r["log_residual"] = math.log(r["observed_fitted_ratio"])
    isolated = [r for r in rows if r["in_degree"] == r["out_degree"] == 0]
    positive = [r for r in rows if r["in_degree"] > 0]
    correlations = {
        "spearman_words": spearman([r["in_degree"] for r in rows], [r["tokens"] for r in rows]),
        "spearman_characters": spearman([r["in_degree"] for r in rows], [r["characters"] for r in rows]),
        "spearman_whitespace": spearman([r["in_degree"] for r in rows], [r["whitespace_tokens"] for r in rows]),
        "spearman_positive_only": spearman([r["in_degree"] for r in positive], [r["tokens"] for r in positive]),
        "positive_only_n": len(positive),
        "log_pearson": st.correlation(x, y),
        "log_r_squared": st.correlation(x, y) ** 2,
    }
    result = {
        "n": len(rows), "directed_edges": len(edges), "isolates": len(isolated),
        "zero_in_degree": sum(r["in_degree"] == 0 for r in rows),
        "total_tokens": sum(r["tokens"] for r in rows),
        "median_tokens": st.median(r["tokens"] for r in rows),
        "min_tokens": min(r["tokens"] for r in rows), "max_tokens": max(r["tokens"] for r in rows),
        "correlations": correlations,
        "fit": {"intercept": intercept, "slope": slope,
                "equation": "ln(tokens) = intercept + slope * ln(1 + in_degree)",
                "scope": "OLS descriptive guide fitted to all 303 pages; exponentiated predictions describe the log-scale trend, not a conditional arithmetic mean or causal effect."},
        "largest_positive_residuals": sorted(rows, key=lambda r: -r["log_residual"])[:5],
        "largest_negative_residuals": sorted(rows, key=lambda r: r["log_residual"])[:5],
        "cases": {r["node_id"]: r for r in rows if r["node_id"] in CASES},
        "pages": rows,
        "tokenizer": {"pattern": WORD.pattern, "example": "Spider-Man didn't save Queens; Doctor Strange's portal did!",
                      "example_tokens": tokenize("Spider-Man didn't save Queens; Doctor Strange's portal did!"),
                      "choices": "Unicode letter sequences with internal apostrophes; hyphens split; numbers and punctuation excluded. Case, function words and inflected forms retained. No stemming, stopword filtering or subword tokenizer."},
        "scope": "All 303 frozen articles and full directed network; in-degree counts distinct source pages within this roster, not the whole web or reader traffic.",
        "source_url": "https://sunelehmann.com/socialgraphs2026-web/data/marvel_pages.zip",
        "snapshot": "2026-08-26", "checked_on": "2026-10-05",
        "source_sha256": {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in [ARCHIVE, ROOT / "data/week1/week1_nodes.tsv", ROOT / "data/week1/week1_edges.tsv"]},
        "text_sha256": {k: hashlib.sha256(v.encode("utf-8")).hexdigest() for k, v in sorted(pages.items())},
        "versions": {"python": platform.python_version(), "numpy": np.__version__, "matplotlib": matplotlib.__version__},
    }
    return rows, result, pages


def plot(rows, result):
    plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 12, "axes.spines.top": False, "axes.spines.right": False})
    fig, ax = plt.subplots(figsize=(11.5, 6.7), facecolor="#f6f1e7")
    ax.set_facecolor("#f6f1e7")
    ax.scatter([r["in_degree"] + 1 for r in rows], [r["tokens"] for r in rows],
               c="#527e84", alpha=.5, s=30, edgecolors="none", label="One frozen article")
    grid = np.geomspace(1, 107, 200)
    fit = np.exp(result["fit"]["intercept"]) * grid ** result["fit"]["slope"]
    ax.plot(grid, fit, color="#20231f", linewidth=1.5, linestyle="--", label="Descriptive log-scale trend")
    labels = {
        "Brian_Braddock": (12, 12, "Brian Braddock"),
        "Miracleman_(character)": (14, 18, "Miracleman"),
        "Quasar_(character)": (10, -24, "Quasar"),
        "Betsy_Braddock": (12, 10, "Betsy Braddock"),
        "Spider-Man": (-95, -24, "Spider-Man"),
    }
    for r in rows:
        if r["node_id"] in labels:
            dx, dy, name = labels[r["node_id"]]
            point = (r["in_degree"] + 1, r["tokens"])
            ax.scatter(*point, c="#dd624b", s=60, edgecolors="#20231f", linewidth=.6, zorder=5)
            ax.annotate(name, point, xytext=(dx, dy), textcoords="offset points", fontsize=11,
                        arrowprops={"arrowstyle": "-", "color": "#20231f", "linewidth": .65}, zorder=6)
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_xticks([1, 2, 6, 11, 26, 51, 107], labels=["0", "1", "5", "10", "25", "50", "106"])
    ax.set_yticks([250, 500, 1000, 2000, 5000, 10000, 20000], labels=["250", "500", "1,000", "2,000", "5,000", "10,000", "20,000"])
    ax.minorticks_off()
    ax.set_xlim(.85, 130)
    ax.set_ylim(140, 22500)
    ax.set_xlabel("Incoming links from the other Marvel articles (log of 1 + count)", labelpad=12)
    ax.set_ylabel("Word-like tokens in the frozen article (log scale)", labelpad=12)
    ax.set_title("More incoming links, longer articles — with revealing exceptions", fontsize=17, pad=18)
    ax.legend(loc="lower right", frameon=False, fontsize=10)
    ax.text(.025, .965, f"303 pages · rank correlation = {result['correlations']['spearman_words']:.3f}",
            transform=ax.transAxes, va="top", fontsize=11)
    fig.tight_layout()
    fig.savefig(FIG / "links_and_words.png", dpi=180)
    plt.close(fig)


def main():
    OUT.mkdir(parents=True, exist_ok=True)
    FIG.mkdir(parents=True, exist_ok=True)
    rows, result, pages = analyse()
    assert result["isolates"] == 17
    (OUT / "results.json").write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    with (OUT / "page_lengths.csv").open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=list(rows[0]), lineterminator="\n")
        writer.writeheader()
        writer.writerows(rows)
    plot(rows, result)
    print(json.dumps({k: v for k, v in result.items() if k not in {"pages", "text_sha256", "source_sha256", "largest_positive_residuals", "largest_negative_residuals"}}, indent=2, ensure_ascii=False))
    return result


if __name__ == "__main__":
    main()