#!/usr/bin/env python3
"""
Ainglish panel harness — the runnable version of the panel protocol.

The vetoing metrics (comprehension_accuracy_delta, interpretation_entropy_delta) need a decorrelated
MODEL panel, and until now "panel" existed only as prose. This file makes it an executable protocol:
give it a manifest and model endpoints, and it produces a measurement ready to submit to
POST /api/v1/proposals/{slug}/measurements — with the methodology enforced by construction:

  COUNTERBALANCED ARMS   Each panelist answers every item exactly once — half in the standard-English
                         arm, half in the Ainglish arm, split deterministically by seed — so both arms
                         share readers without any reader seeing both forms of one item.
  MINIMAL PAIRS          The two arms of an item must differ only by the construct (the register's
                         minimal-pairs rule; the harness warns on big length divergence).
  CALIBRATION GATE       Planted-effect items (the correct answer is derivable in one arm and NOT in
                         the other) are the panel's positive control: a panel that cannot detect the
                         planted difference is not measuring, and the harness REFUSES to emit a
                         measurement — ctl() applied to the panel itself. Fails closed.
  DECORRELATION          The panel should span model families, and for disambiguation constructs
                         include a QUANTIZED member (a construct whose markers collapse at 4-bit earns
                         "helps, except under quantization", not a clean pass).
  HONEST INTERVALS       value_lo/value_hi come from bootstrap resampling over items; the register
                         only spends measurements whose whole interval clears neutral.

Adapters: any OpenAI-compatible chat endpoint (OpenAI, Anthropic gateways, ollama, vllm, llama.cpp)
via {"base_url", "api_key_env", "model"}. temperature=0. Pure stdlib.

Usage:
  python3 panel.py manifest.json            # run the panel, print the measurement JSON
  python3 panel.py --demo-manifest          # print a ready manifest skeleton for wit/pred
  python3 panel.py --selftest               # mock panelists prove the scoring + the calibration gate

A measurement produced here is still only EVIDENCE once a disjoint party reproduces the same
manifest within tolerance — this file replaces the excuse, not the replication.
"""
import hashlib
import json
import os
import random
import sys
import urllib.request

NEUTRAL_EPS = 1e-9


# ------------------------------------------------------------------ adapters
def chat(endpoint, prompt):
    """One deterministic completion from an OpenAI-compatible endpoint."""
    body = {
        "model": endpoint["model"],
        "temperature": 0,
        "messages": [{"role": "user", "content": prompt}],
    }
    headers = {"Content-Type": "application/json", "User-Agent": "ainglish-panel"}
    key = os.environ.get(endpoint.get("api_key_env", ""), "")
    if key:
        headers["Authorization"] = f"Bearer {key}"
    req = urllib.request.Request(endpoint["base_url"].rstrip("/") + "/chat/completions",
                                 json.dumps(body).encode(), headers)
    with urllib.request.urlopen(req, timeout=120) as r:
        data = json.loads(r.read())
    return data["choices"][0]["message"]["content"]


def ask(endpoint, text, question, options):
    """Present one item arm and force a choice from the fixed options."""
    prompt = (f"Read this message written by one agent to another:\n\n---\n{text}\n---\n\n"
              f"Question: {question}\nAnswer with EXACTLY one of these options and nothing else: "
              + " | ".join(options))
    out = chat(endpoint, prompt).strip().lower()
    for o in options:
        if o.lower() in out:
            return o
    return out[:40]  # off-option answer counts as wrong and inflates entropy — as it should


# ------------------------------------------------------------------ assignment & scoring
def arm_for(seed, panelist, item_id):
    """Deterministic counterbalancing: which arm this panelist reads for this item."""
    h = hashlib.sha256(f"{seed}|{panelist}|{item_id}".encode()).digest()
    return "ainglish" if h[0] % 2 else "english"


def entropy(counts):
    import math
    total = sum(counts.values())
    if total == 0:
        return 0.0
    return -sum((c / total) * math.log2(c / total) for c in counts.values() if c)


def score(rows, items):
    """rows: (item_id, arm, panelist, answer). Returns per-arm accuracy and mean answer-entropy."""
    key = {i["id"]: i for i in items}
    acc, ent = {}, {}
    for arm in ("english", "ainglish"):
        arm_rows = [r for r in rows if r[1] == arm]
        graded = [r for r in arm_rows if key[r[0]].get("answer") is not None]
        acc[arm] = (sum(1 for r in graded if str(r[3]).lower() == str(key[r[0]]["answer"]).lower()) / len(graded)) if graded else None
        by_item = {}
        for r in arm_rows:
            by_item.setdefault(r[0], {}).setdefault(str(r[3]).lower(), 0)
            by_item[r[0]][str(r[3]).lower()] += 1
        ent[arm] = (sum(entropy(c) for c in by_item.values()) / len(by_item)) if by_item else None
    return acc, ent


def bootstrap_delta(rows, items, metric, n=2000, seed=0):
    """Resample ITEMS with replacement; recompute the arm delta each time. Percentile 2.5/97.5."""
    rng = random.Random(seed)
    ids = sorted({i["id"] for i in items})
    deltas = []
    for _ in range(n):
        sample_ids = [rng.choice(ids) for _ in ids]
        # rebuild a resampled row/item set (items may repeat; suffix keeps ids distinct)
        r2, i2 = [], []
        for k, sid in enumerate(sample_ids):
            i2.append({**next(i for i in items if i["id"] == sid), "id": f"{sid}#{k}"})
            r2.extend((f"{sid}#{k}", arm, p, a) for (iid, arm, p, a) in rows if iid == sid)
        acc, ent = score(r2, i2)
        if metric == "comprehension_accuracy_delta" and acc["ainglish"] is not None and acc["english"] is not None:
            deltas.append(100 * (acc["ainglish"] - acc["english"]))
        elif metric == "interpretation_entropy_delta" and ent["ainglish"] is not None and ent["english"] is not None:
            deltas.append(ent["ainglish"] - ent["english"])
    if not deltas:
        return None, None
    deltas.sort()
    return deltas[int(0.025 * len(deltas))], deltas[int(0.975 * len(deltas))]


# ------------------------------------------------------------------ the run
def run_panel(manifest, ask_fn=ask):
    items = manifest["items"]
    calib = [i for i in items if i.get("calibration")]
    real = [i for i in items if not i.get("calibration")]
    panel = manifest["panel"]
    seed = manifest.get("seed", 0)
    if not calib:
        print("REFUSING to run: no calibration items. A panel that was never shown a detectable "
              "difference proves nothing when it detects none (ctl(none) is not evidence).")
        return None

    rows = []
    for item in items:
        for ep in panel:
            arm = arm_for(seed, ep["name"], item["id"])
            answer = ask_fn(ep, item[arm], item["question"], item["options"])
            rows.append((item["id"], arm, ep["name"], answer))

    # --- the calibration gate: the planted effect must be detected, or nothing is emitted ---
    calib_rows = [r for r in rows if r[0] in {c["id"] for c in calib}]
    cacc, _ = score(calib_rows, calib)
    detectable, undetectable = cacc.get(manifest.get("planted_arm", "ainglish")), cacc.get("english")
    if detectable is None or undetectable is None or (detectable - undetectable) < manifest.get("calibration_min_gap", 0.5):
        print(f"CALIBRATION FAILED: planted-effect gap {detectable} vs {undetectable} — this panel "
              "cannot detect a known difference, so its null on the real items is vacuous. "
              "No measurement emitted. (The panel failed its positive control, not the construct.)")
        return None
    print(f"calibration: planted arm {detectable:.2f} vs other {undetectable:.2f} — panel can detect. ctl(planted-items) passes.")

    real_rows = [r for r in rows if r[0] in {i["id"] for i in real}]
    acc, ent = score(real_rows, real)
    metric = manifest["metric"]
    if metric == "comprehension_accuracy_delta":
        value = round(100 * (acc["ainglish"] - acc["english"]), 2)
    elif metric == "interpretation_entropy_delta":
        value = round(ent["ainglish"] - ent["english"], 4)
    else:
        print(f"unsupported metric {metric}"); return None
    lo, hi = bootstrap_delta(real_rows, real, metric, seed=seed)

    spec = {k: manifest[k] for k in ("construct", "metric", "items", "seed") if k in manifest}
    spec["models"] = [p["name"] for p in panel]
    spec["protocol"] = "panel.py counterbalanced-arms + planted-effect calibration gate"
    measurement = {
        "metric": metric, "value": value,
        "value_lo": round(lo, 4) if lo is not None else None,
        "value_hi": round(hi, 4) if hi is not None else None,
        "panel_models": [p["name"] for p in panel], "panel_neff": len(panel),
        "is_adversarial": bool(manifest.get("is_adversarial")),
        "manifest": spec,
    }
    print(json.dumps(measurement, indent=1))
    print(f"\nSubmit: POST /api/v1/proposals/{manifest.get('slug','<slug>')}/measurements with a "
          "Colony Bearer (see /developers). Evidence once a DISJOINT party reproduces this manifest.")
    return measurement


# ------------------------------------------------------------------ selftest (mock panelists)
def selftest():
    """A perfect reader and a coin-flipper prove the scoring and the gate, no models needed."""
    items = [
        # calibration: answer derivable ONLY in the ainglish arm (planted effect)
        {"id": f"c{k}", "calibration": True,
         "english": "The check passed.", "ainglish": "The check passed wit(counterparty-settled).",
         "question": "Did a counterparty settle this?", "options": ["yes", "cannot tell"], "answer": "yes"}
        for k in range(4)
    ] + [
        {"id": f"r{k}",
         "english": f"Suite {k} passed, and the evidence generator is of class process-ran.",
         "ainglish": f"Suite {k} passed wit(process-ran).",
         "question": "What class is the evidence generator?", "options": ["process-ran", "visible", "cannot tell"],
         "answer": "process-ran"}
        for k in range(8)
    ]

    def tag_reliant(ep, text, q, options):
        # Simulates what the metric measures: recovery RELIABILITY. Reads the compact tag perfectly;
        # extracts from prose only ~half the time (deterministic on item text) — the minimal pair
        # holds the same information in both arms, so any delta is about recovery, not content.
        if "wit(counterparty-settled)" in text: return "yes"
        if "counterparty" in q: return "cannot tell"
        if "wit(" in text: return "process-ran"
        return "process-ran" if hashlib.sha256((text + q + ep["name"]).encode()).digest()[0] % 2 else "cannot tell"

    def coinflip(ep, text, q, options):
        return random.Random(hash(text + q) & 0xFFFF).choice(options)

    good = {"construct": "wit-demo", "slug": "demo", "metric": "comprehension_accuracy_delta",
            "seed": 7, "items": items, "panel": [{"name": "reader-a"}, {"name": "reader-b"}]}
    m = run_panel(good, ask_fn=tag_reliant)
    assert m is not None and m["value"] > 0, "calibrated tag-reliant panel must find the recovery effect"

    bad = dict(good, panel=[{"name": "flip-a"}, {"name": "flip-b"}])
    assert run_panel(bad, ask_fn=coinflip) is None, "a coin-flipping panel must FAIL the calibration gate"
    print("\nselftest OK: real effect measured by a calibrated panel; uncalibrated panel refused.")


DEMO_NOTE = """{
  "construct": "wit-class-and-pred-class-witness-and-settle-axes",
  "slug": "wit-class-and-pred-class-witness-and-settle-axes",
  "metric": "comprehension_accuracy_delta",
  "seed": 7,
  "planted_arm": "ainglish",
  "panel": [
    {"name": "gpt-4o", "base_url": "https://api.openai.com/v1", "api_key_env": "OPENAI_API_KEY", "model": "gpt-4o"},
    {"name": "local-q4", "base_url": "http://localhost:11434/v1", "api_key_env": "", "model": "llama3:8b-instruct-q4_K_M"}
  ],
  "items": [
    {"id": "c1", "calibration": true,
     "english": "The check passed.",
     "ainglish": "The check passed wit(counterparty-settled).",
     "question": "Did a counterparty settle this?", "options": ["yes", "cannot tell"], "answer": "yes"},
    {"id": "r1",
     "english": "The digest matched, and the evidence generator is of class public-path.",
     "ainglish": "The digest matched wit(public-path).",
     "question": "Could a stranger have observed this evidence?", "options": ["yes", "no", "cannot tell"], "answer": "yes"}
  ]
}"""


def main(argv):
    if "--selftest" in argv:
        selftest(); return 0
    if "--demo-manifest" in argv:
        print(DEMO_NOTE); return 0
    if len(argv) < 2:
        print(__doc__.strip().split("\n\n")[0]); print("\nusage: panel.py manifest.json | --demo-manifest | --selftest"); return 0
    manifest = json.loads(sys.stdin.read() if argv[1] == "-" else open(argv[1]).read())
    return 0 if run_panel(manifest) else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv))