Methodology — the work, shown

"""Derive the site's entire API content from the Postman collection.

This command is the provenance boundary: everything under /endpoints/ and
/schema/ comes out of this parse, nothing is hand-entered. It also re-runs the
audit checks (api/audit.py) so every analytical claim is re-verified against
the current file, and records the file's SHA-256 for reader verification.
"""
import hashlib
import json
import re

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

from api import audit, seeds
from api.models import (
    AuditCheck,
    Endpoint,
    Parameter,
    RequestBody,
    ResponseExample,
    SchemaField,
    Source,
)

FAMILY_ORDER = {"profile": 0, "project": 1, "credits": 2, "score-request": 3}


class Command(BaseCommand):
    help = "Parse the Postman collection into the database (idempotent)."

    def handle(self, *args, **opts):
        path = settings.SOURCE_COLLECTION_PATH
        if not path.exists():
            raise CommandError(f"Source collection not found: {path}")
        raw = path.read_bytes()
        sha = hashlib.sha256(raw).hexdigest()
        col = json.loads(raw)

        with transaction.atomic():
            self._wipe()
            seeds.seed_sources(sha)
            seeds.seed_reference_keys()
            endpoints = self._ingest_endpoints(col)
            self._ingest_schema(endpoints)
            self._run_audits(col)
            seeds.seed_findings()
            seeds.seed_lessons()

        self.stdout.write(self.style.SUCCESS(
            f"Ingested {Endpoint.objects.count()} endpoints, "
            f"{ResponseExample.objects.count()} response examples, "
            f"{SchemaField.objects.count()} schema fields, "
            f"{AuditCheck.objects.count()} audit checks. "
            f"Source sha256={sha[:16]}…"
        ))

    # ------------------------------------------------------------------
    def _wipe(self):
        for model in (Endpoint, AuditCheck, SchemaField):
            model.objects.all().delete()

    def _ingest_endpoints(self, col):
        created = []
        order = [0]

        def walk(items, folders):
            for it in items:
                if "item" in it:
                    walk(it["item"], folders + [it["name"]])
                    continue
                req = it.get("request", {})
                url = req.get("url", {})
                raw_url = url.get("raw") if isinstance(url, dict) else url
                rel = (raw_url or "").replace("{{baseUrl}}", "")
                desc = req.get("description") or ""
                name = it["name"]

                # Detected, never assumed:
                deprecated = "Deprecated" in name or "DEPRECATED" in desc
                m_step = re.match(r"\*\*Step (\d+)", desc)
                m_use = re.search(r"Please use the `([^`]+)` endpoint instead", desc)
                path_only = rel.split("?")[0]
                seg = path_only.split("/")[3] if len(path_only.split("/")) > 3 else ""

                ep = Endpoint.objects.create(
                    slug=re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-"),
                    name=name,
                    method=req.get("method", ""),
                    path=rel,
                    folder_path=" / ".join(folders),
                    description=desc,
                    family=seg,
                    is_deprecated=deprecated,
                    use_instead=m_use.group(1) if m_use else "",
                    lifecycle_step=int(m_step.group(1)) if m_step else None,
                    order=order[0],
                )
                order[0] += 1

                if isinstance(url, dict):
                    for v in url.get("variable") or []:
                        Parameter.objects.create(
                            endpoint=ep, kind="path", key=v.get("key", ""),
                            example=str(v.get("value") or ""),
                            description=v.get("description") or "",
                        )
                    for q in url.get("query") or []:
                        Parameter.objects.create(
                            endpoint=ep, kind="query", key=q.get("key", ""),
                            example=str(q.get("value") or ""),
                            description=q.get("description") or "",
                        )
                for h in req.get("header") or []:
                    Parameter.objects.create(
                        endpoint=ep, kind="header", key=h.get("key", ""),
                        example=str(h.get("value") or ""),
                    )
                body = req.get("body") or {}
                if body.get("raw"):
                    RequestBody.objects.create(
                        endpoint=ep, raw=body["raw"],
                        content_type="application/json",
                    )
                for i, resp in enumerate(it.get("response") or []):
                    ctype = ""
                    for h in resp.get("header") or []:
                        if h.get("key") == "Content-Type":
                            ctype = h.get("value") or ""
                    ResponseExample.objects.create(
                        endpoint=ep, name=resp.get("name", ""),
                        status_code=resp.get("code"),
                        status_text=resp.get("status", ""),
                        body=resp.get("body") or "", content_type=ctype, order=i,
                    )
                created.append(ep)

        walk(col.get("item", []), [])

        # Stable ordering: family, then lifecycle step, then original order.
        for ep in created:
            ep.order = (
                FAMILY_ORDER.get(ep.family, 9) * 100
                + (ep.lifecycle_step or 50)
                + (0 if not ep.is_deprecated else 30)
            )
            ep.save(update_fields=["order"])
        return created

    def _ingest_schema(self, endpoints):
        """Flatten each endpoint's first 2xx JSON example into SchemaFields."""
        for ep in endpoints:
            ex = ep.examples.filter(status_code__lt=300).first()
            if not ex or not ex.body.strip().startswith(("{", "[")):
                continue
            try:
                data = json.loads(ex.body)
            except json.JSONDecodeError:
                continue
            if isinstance(data, list):
                data = data[0] if data else {}
            for path, leaf, depth in _flatten(data):
                field, _ = SchemaField.objects.get_or_create(
                    path=path, defaults={"leaf_type": leaf, "depth": depth}
                )
                field.endpoints.add(ep)

    def _run_audits(self, col):
        for result in audit.run_all(col):
            AuditCheck.objects.create(**result)


def _flatten(obj, prefix="", depth=0):
    out = []
    if isinstance(obj, dict):
        for k, v in obj.items():
            p = f"{prefix}.{k}" if prefix else k
            if isinstance(v, dict):
                out.append((p, "", depth))
                out += _flatten(v, p, depth + 1)
            elif isinstance(v, list):
                out.append((p, "array", depth))
                if v:
                    out += _flatten(v[0], p + "[]", depth + 1)
            else:
                out.append((p, type(v).__name__, depth))
    return out

"""Ingest-time verification of every factual claim the site makes.

Each function inspects the raw collection dict and returns a result the site
displays on /methodology/. Nothing here is hardcoded as "true": if Kindsight
ships a fixed collection, these checks flip and the site's claims flip with
them. This is the mechanism behind the site's "show your work" promise.
"""
import re


def check_auth_gap(col):
    """G1: the collection description promises auth machinery the file lacks."""
    desc = col.get("info", {}).get("description") or ""
    folder_names = _all_folder_names(col.get("item", []))
    variables = [v.get("key") for v in col.get("variable", [])]
    details = {
        "description_mentions_auth_folder": "Authentication folder" in desc,
        "description_mentions_collection_auth": "Collection-level authorization" in desc,
        "has_collection_auth_block": "auth" in col,
        "has_event_scripts": "event" in col,
        "has_authentication_folder": any("auth" in f.lower() for f in folder_names),
        "defined_variables": variables,
        "has_client_id_variable": "clientId" in variables,
        "has_client_secret_variable": "clientSecret" in variables,
        "top_level_keys": list(col.keys()),
    }
    gap = (
        details["description_mentions_auth_folder"]
        and not details["has_authentication_folder"]
        and not details["has_collection_auth_block"]
    )
    return {
        "code": "G1",
        "title": "Auth machinery promised by the description vs present in the file",
        "passed": not gap,  # passed=False means the gap exists
        "summary": (
            "The collection description instructs readers to use an "
            "'Authentication folder' and collection-level authorization. "
            "This check verifies whether either exists in the file."
        ),
        "details": details,
    }


def check_capacity_ranges(col):
    """G2a: capacityRangeLow must not exceed capacityRangeHigh."""
    pairs, inverted = [], 0
    for name, body in _iter_example_bodies(col):
        for lo, hi in re.findall(
            r'"capacityRangeLow":\s*(\d+),\s*"capacityRangeHigh":\s*(\d+)', body
        ):
            bad = int(lo) > int(hi)
            inverted += bad
            pairs.append({"endpoint": name, "low": int(lo), "high": int(hi), "inverted": bad})
    return {
        "code": "G2a",
        "title": "Capacity ranges: low ≤ high",
        "passed": inverted == 0,
        "summary": f"{inverted} of {len(pairs)} capacity range pairs are inverted "
        "(low > high) in the saved examples.",
        "details": {"pairs": pairs, "inverted_count": inverted, "total": len(pairs)},
    }


def check_ages(col):
    """G2b: ages in sample data must be humanly possible."""
    ages = []
    for name, body in _iter_example_bodies(col):
        for v in re.findall(r'"age":\s*(\d+)', body):
            ages.append({"endpoint": name, "age": int(v), "possible": int(v) <= 120})
    impossible = sum(1 for a in ages if not a["possible"])
    return {
        "code": "G2b",
        "title": "Ages within human range (≤120)",
        "passed": impossible == 0,
        "summary": f"{impossible} of {len(ages)} age values in the saved examples "
        "are humanly impossible.",
        "details": {"ages": ages, "impossible_count": impossible},
    }


def check_row_counts(col):
    """G2c: validRows must not exceed totalRows."""
    rows = []
    for name, body in _iter_example_bodies(col):
        for t, v in re.findall(r'"totalRows":\s*(\d+),\s*"validRows":\s*(\d+)', body):
            rows.append(
                {"endpoint": name, "totalRows": int(t), "validRows": int(v),
                 "coherent": int(v) <= int(t)}
            )
    bad = sum(1 for r in rows if not r["coherent"])
    return {
        "code": "G2c",
        "title": "Row counts: validRows ≤ totalRows",
        "passed": bad == 0,
        "summary": f"{bad} of {len(rows)} totalRows/validRows pairs are incoherent.",
        "details": {"rows": rows, "incoherent_count": bad},
    }


def check_score_bounds(col):
    """Inference check: every *Score field observed falls in 0–4.

    NOTE: this is the one INFERENCE the site makes (docs/ANALYSIS.md §G2). It
    is not stated by any source; it is an observed bound across all examples.
    """
    fields = (
        "capacityScore|affinityScore|propensityScore|rfmScore|recencyScore|"
        "frequencyScore|monetaryScore|primaryAffinityScore|secondaryAffinityScore"
    )
    values = []
    for _, body in _iter_example_bodies(col):
        values += [int(v) for v in re.findall(rf'"(?:{fields})":\s*(-?\d+)', body)]
    in_band = all(0 <= v <= 4 for v in values) if values else None
    return {
        "code": "INF1",
        "title": "Inference: score fields observed within 0–4",
        "passed": in_band,
        "summary": f"All {len(values)} observed score values fall in "
        f"[{min(values)}, {max(values)}]. This is an inference from sample "
        "data, not a documented bound — treat accordingly.",
        "details": {"count": len(values), "min": min(values), "max": max(values)},
    }


def check_statuses(col):
    """Which project status values are evidenced, and how."""
    in_bodies = set()
    for _, body in _iter_example_bodies(col):
        in_bodies.update(re.findall(r'"status":\s*"([a-z-]+)"', body))
    prose = set()
    for item in _iter_requests(col):
        desc = (item.get("request") or {}).get("description") or ""
        if "`processed`" in desc:
            prose.add("processed")
    return {
        "code": "OBS1",
        "title": "Project status values observed (not an exhaustive enumeration)",
        "passed": None,  # informational
        "summary": "Status values seen in example bodies vs mentioned only in "
        "prose. The collection never enumerates the complete set.",
        "details": {
            "in_example_bodies": sorted(in_bodies),
            "in_prose_only": sorted(prose - in_bodies),
        },
    }


def check_pagination(col):
    """G5: which array-returning GETs carry pagination parameters."""
    rows = []
    for item in _iter_requests(col):
        req = item.get("request") or {}
        if req.get("method") != "GET":
            continue
        first = (item.get("response") or [{}])[0].get("body") or ""
        if not first.strip().startswith("["):
            continue
        url = req.get("url") or {}
        q = [x.get("key") for x in (url.get("query") or [])] if isinstance(url, dict) else []
        rows.append({"endpoint": item["name"], "query_params": q, "paginated": bool(q)})
    unpag = sum(1 for r in rows if not r["paginated"])
    return {
        "code": "G5",
        "title": "Pagination on array-returning endpoints",
        "passed": unpag == 0,
        "summary": f"{unpag} of {len(rows)} array-returning GET endpoints have no "
        "pagination parameters.",
        "details": {"endpoints": rows},
    }


def check_error_envelopes(col):
    """G6: how many distinct error body shapes appear in 4xx examples."""
    shapes = {}
    for item in _iter_requests(col):
        for resp in item.get("response") or []:
            code = resp.get("code") or 0
            if 400 <= code < 500 and (resp.get("body") or "").strip().startswith("{"):
                keys = tuple(sorted(re.findall(r'"(\w+)":', resp["body"])))
                shapes.setdefault(keys, []).append(f"{item['name']} ({code})")
    return {
        "code": "G6",
        "title": "Error envelope consistency",
        "passed": len(shapes) <= 1,
        "summary": f"{len(shapes)} distinct error body shapes appear across 4xx "
        "examples. A client needs to parse defensively.",
        "details": {"+".join(k): v for k, v in shapes.items()},
    }


def check_csv_schema(col):
    """G3: text/csv is documented but no CSV schema or example exists."""
    raw = str(col)
    mentions = raw.count("text/csv")
    has_csv_header = False
    for item in _iter_requests(col):
        for h in (item.get("request") or {}).get("header") or []:
            if "csv" in str(h.get("value", "")):
                has_csv_header = True
    return {
        "code": "G3",
        "title": "CSV upload: documented vs specified",
        "passed": has_csv_header,
        "summary": f"'text/csv' is mentioned {mentions} times in prose, but no "
        "request declares a CSV content type and no CSV example or column "
        "schema exists in the file.",
        "details": {"prose_mentions": mentions, "csv_request_present": has_csv_header},
    }


ALL_CHECKS = [
    check_auth_gap,
    check_capacity_ranges,
    check_ages,
    check_row_counts,
    check_score_bounds,
    check_statuses,
    check_pagination,
    check_error_envelopes,
    check_csv_schema,
]


def run_all(col):
    return [fn(col) for fn in ALL_CHECKS]


# -- helpers ----------------------------------------------------------------

def _iter_requests(col):
    def walk(items):
        for it in items:
            if "item" in it:
                yield from walk(it["item"])
            else:
                yield it

    yield from walk(col.get("item", []))


def _iter_example_bodies(col):
    for item in _iter_requests(col):
        for resp in item.get("response") or []:
            yield item["name"], resp.get("body") or ""


def _all_folder_names(items):
    names = []
    for it in items:
        if "item" in it:
            names.append(it["name"])
            names += _all_folder_names(it["item"])
    return names