#!/usr/bin/env python3
"""Aggregate the FHIR write-integrity probe runs into the site's data file and the public CSV.

    python3 -I aggregate.py results/*.json --json OUT.json --csv OUT.csv

Every number on /research/fhir-write-integrity-2026 is read from OUT.json, never typed.
"""
import csv, json, re, sys, argparse
from collections import Counter, OrderedDict

ap = argparse.ArgumentParser()
ap.add_argument('runs', nargs='+')
ap.add_argument('--json', required=True)
ap.add_argument('--csv', required=True)
a = ap.parse_args()

SUBJECTS = OrderedDict([
    ('hapi-8.12.0-local-default', dict(name='HAPI FHIR JPA Server', short='HAPI 8.12.0 (defaults)', where='local', config='hapiproject/hapi:v8.12.0-2 image contents run with Java 21, in-memory H2, starter defaults (referential integrity on write, lenient parser, request validation off)')),
    ('hapi-8.12.0-local-validation', dict(name='HAPI FHIR JPA Server', short='HAPI 8.12.0 (validation on)', where='local', config='as the defaults run, plus hapi.fhir.validation.requests_enabled=true (base FHIR R4 profiles only; no Indian IG package loaded)')),
    ('hapi-public-r4', dict(name='HAPI FHIR Server', short='HAPI public test server', where='public', config='hapi.fhir.org/baseR4, the public test server hosted by University Health Network; configuration as the operator set it')),
    ('firely-public-r4', dict(name='Firely Server', short='Firely public sandbox', where='public', config='server.fire.ly/R4, the public sandbox run by Firely; configuration as the operator set it')),
])
# The probe's own cleanup left 7 resources on hapi.fhir.org (deletes refused while other run resources still
# referenced them); they were deleted by hand right after the run (all answered 200), and searches for the study's
# identifier systems on both public servers returned 0.
CLEANUP_NOTES = {'hapi-public-r4': {'deleted': 427, 'left': 0, 'note': 'the probe deleted 420; 7 were deleted by hand right after the run; identifier searches returned 0'}}
runs = {}
for f in a.runs:
    r = json.load(open(f))
    if r['label'] in SUBJECTS:
        runs[r['label']] = r
order = [k for k in SUBJECTS if k in runs]

CAT = OrderedDict([
    ('field-drops', 'Field fidelity'), ('extensions', 'Extensions'), ('references', 'References'),
    ('conditional', 'Conditional create and update'), ('transactions', 'Transactions and batches'),
    ('validation', 'Validation strictness'), ('india', 'ABDM and NHCX examples'),
])

def ok(s):
    return isinstance(s, int) and 200 <= s < 300

def differs(row):
    """A 2xx answer after which the read-back is not what was sent."""
    if not ok(row.get('status')):
        return False
    o = row['outcome']
    if o in ('stored-with-changes', '2xx-nothing-new', 'partial'):
        return True
    if o == 'accepted-invalid' and (row.get('dropped') or row.get('changed')):
        return True
    return False

servers, rows_csv = [], []
probes = OrderedDict()
def renormalize(r):
    # Runs made before the probe learned that a server may return a relative reference as an absolute URL on its own
    # base (same identity): treat that as a normalization, exactly as the current probe does.
    base = r['base'].rstrip('/')
    for x in r['results']:
        ch = x.get('changed')
        if isinstance(ch, list) and ch:
            keep = [c for c in ch if not (str(c['path']).endswith('reference') and str(c.get('stored')) == f"{base}/{c.get('sent')}")]
            moved = len(ch) - len(keep)
            if moved:
                x['changed'] = keep
                x.setdefault('normalized', []).extend([{'path': c['path'], 'kind': 'reference-made-absolute'} for c in ch if c not in keep])
                if not keep and not x.get('dropped') and x['outcome'] == 'stored-with-changes':
                    x['outcome'] = 'stored-as-sent'
                    x['detail'] = 'normalized: reference made absolute'
                elif x['outcome'] == 'accepted-invalid' and not keep and not x.get('dropped'):
                    x['detail'] = 'stored as sent (reference made absolute)'
        # Probe artifact: the per-bundle run marker was added as an extension to every entry, including Binary, which
        # in R4 is not a DomainResource and has no extension element. A server dropping it is right; exclude it.
        if x.get('category') == 'india' and x['outcome'] == 'stored-with-changes':
            if x.get('mode') == 'store':
                drops = x.get('dropped') or []
                if drops and all(re.search(r'\.resource\.extension\[\d+\]\.(url|valueString)$', d) for d in drops) and not x.get('changed_count'):
                    bad = {int(re.search(r'entry\[(\d+)\]', d).group(1)) for d in drops}
                    x['outcome'] = 'stored-as-sent'; x['artifact_excluded'] = f'run marker on {len(bad)} Binary entr(y/ies) excluded'
            elif x.get('mode') == 'transaction':
                paths = x.get('paths') or []
                if paths and all(p_.startswith('Binary.extension') for p_ in paths) and not x.get('entries_missing'):
                    x['entries_identical'] = x.get('entries_identical', 0) + x.get('entries_with_drops', 0)
                    x['entries_with_drops'] = 0; x['outcome'] = 'stored-as-sent'; x['artifact_excluded'] = 'run marker on Binary excluded'
        if x['id'] == 'E4-tx-urn-uuid-resolution' and x['outcome'] == 'stored-with-changes' and f"stored subject={base}/Patient/" in (x.get('detail') or ''):
            x['outcome'] = 'stored-as-sent'

for k in order:
    r = runs[k]
    renormalize(r)
    res = r['results']
    core = [x for x in res if x['category'] != 'india']
    india = [x for x in res if x['category'] == 'india']
    invalid = [x for x in core if x.get('valid') is False and not x.get('control')]
    acc_invalid = [x for x in invalid if ok(x.get('status'))]
    acc_invalid_dropped = [x for x in acc_invalid if x.get('dropped') or x.get('changed')]
    valid = [x for x in core if x.get('valid') is not False]
    valid_2xx = [x for x in valid if ok(x.get('status'))]
    valid_diff = [x for x in valid_2xx if differs(x)]
    core_2xx = [x for x in core if ok(x.get('status'))]
    core_diff = [x for x in core_2xx if differs(x)]
    store = [x for x in india if x.get('mode') == 'store']
    tx = [x for x in india if x.get('mode') == 'transaction']
    s = dict(key=k, **SUBJECTS[k],
             software=(r.get('software') or {}).get('name'), version=(r.get('software') or {}).get('version'),
             fhirVersion=r.get('fhirVersion'), base=r['base'], started=r['started'], finished=r['finished'], requests=r['requests'],
             cleanup=CLEANUP_NOTES.get(k, r.get('cleanup')),
             core=dict(probes=len(core), answered_2xx=len(core_2xx), differs_after_2xx=len(core_diff),
                       valid=len(valid), valid_2xx=len(valid_2xx), valid_differs=len(valid_diff),
                       invalid=len(invalid), invalid_accepted=len(acc_invalid), invalid_accepted_changed=len(acc_invalid_dropped),
                       refused=len([x for x in core if not ok(x.get('status'))])),
             india=dict(bundles=len(store),
                        store_2xx=len([x for x in store if ok(x.get('status'))]),
                        store_identical=len([x for x in store if x['outcome'] == 'stored-as-sent']),
                        store_normalized=len([x for x in store if x['outcome'] == 'stored-as-sent' and x.get('normalized_count')]),
                        tx_2xx=len([x for x in tx if ok(x.get('status'))]),
                        tx_identical=len([x for x in tx if x['outcome'] == 'stored-as-sent']),
                        entries=sum(x.get('entries', 0) for x in tx),
                        entries_identical=sum(x.get('entries_identical', 0) for x in tx),
                        refusal_reasons=Counter()),
             by_category=OrderedDict())
    s['india']['refused_profile_unknown'] = len([x for x in india if not ok(x.get('status')) and 'Profile reference' in (x.get('detail') or '')])
    s['india']['refused_duplicate'] = len([x for x in india if not ok(x.get('status')) and 'HAPI-2840' in (x.get('detail') or '')])
    del s['india']['refusal_reasons']
    for c in CAT:
        xs = [x for x in res if x['category'] == c]
        s['by_category'][c] = dict(probes=len(xs), answered_2xx=len([x for x in xs if ok(x.get('status'))]), differs_after_2xx=len([x for x in xs if differs(x)]),
                                   refused=len([x for x in xs if not ok(x.get('status'))]), outcomes=dict(Counter(x['outcome'] for x in xs)))
    servers.append(s)
    for x in res:
        pid = x['id']
        p = probes.setdefault(pid, dict(id=pid, category=x['category'], title=x.get('title', ''), valid=x.get('valid'), control=bool(x.get('control')), expected_by_spec=bool(x.get('expected_by_spec')), results=OrderedDict()))
        p['results'][k] = dict(status=x.get('status'), outcome=x['outcome'], differs=differs(x),
                               dropped=(x.get('dropped') or [])[:8], changed=[c['path'] for c in (x.get('changed') or [])][:8],
                               detail=(x.get('detail') or '')[:240])
        rows_csv.append(dict(server=SUBJECTS[k]['short'], software=s['software'], version=s['version'], where=SUBJECTS[k]['where'], category=CAT[x['category']], probe=pid,
                             title=x.get('title', ''), input_valid_per_r4={True: 'yes', False: 'no', None: ''}[x.get('valid')], http_status=x.get('status'), outcome=x['outcome'],
                             readback_differs_after_2xx='yes' if differs(x) else 'no',
                             dropped_paths=';'.join((x.get('dropped') or [])[:20]), changed_paths=';'.join(c['path'] for c in (x.get('changed') or [])[:20]),
                             detail=(x.get('detail') or '')[:300]))

out = dict(study='fhir-write-integrity-2026', generated_by='ops/healthcare/fhir-study/aggregate.py',
           run_date=min(s['started'] for s in servers)[:10], servers=servers, categories=CAT, probes=list(probes.values()))
json.dump(out, open(a.json, 'w'), indent=1, ensure_ascii=False)
with open(a.csv, 'w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=list(rows_csv[0].keys()))
    w.writeheader(); w.writerows(rows_csv)
for s in servers:
    print(s['short'], s['version'], json.dumps(s['core']), json.dumps(s['india']))