# -*- coding: utf-8 -*-
"""
THE ALIGNMENT CONTROL — the discovery adjustment for quantity (b).
Protocol frozen 7 September 2026, before any run, after discovery_control.py had
been run and read. Nothing below was altered after the numbers were seen.

QUESTION
  The chapter's residual claim is not that a search finds many holds but that the
  holds fall on one side of a line the text draws: openings the Qurʾān names as
  signs do not hold; the holds come from the rest. Does a matched search on a text
  with no stake in nineteen produce holds that avoid the sign side just as cleanly?

THE PARTITION IS THE TEXT'S, NOT OURS
  Every candidate case is labelled before counting, from the opening it is built on:
    letter-group rule: sign if the opening's letter-set is one the Qurʾān names as
      signs anywhere (A-L-M, A-L-M-Ṣ, A-L-R, A-L-M-R, Ṭ-S-M, Ṭ-S);
    formula-only rule: sign if the opening itself writes tilka āyāt (sūrahs 10, 12,
      13, 15, 26, 27, 28, 31).
  A cross-opening letter cluster (rung R2 and above) is a sign case if ANY of the
  openings it pools is a sign opening — the rule the chapter applies to the Ṣād
  cluster, which pools Sūrah 7. In Family A (shuffled targets) the label travels
  with the opening, so a permuted opening keeps its letter-set and its formula flag.

UNITS, TARGETS, FAMILIES, RUNGS, RUNS
  Identical to discovery_control.py (imported from it): the same five corpora, the
  same cuts, the same 29 openings, the same ladder R0–R5, 10,000 runs, seed 19.

THE STATISTIC
  At each rung, the 2 × 2 table (sign / non-sign) × (holds / does not) over the
  candidate cases, and Fisher's exact test, one-tailed in the direction the chapter
  predicts (fewer sign holds than chance). Where the investigator has a choice —
  R4's convention, R5's two ±1 wildcards — it is exercised in favour of alignment:
  the convention with the smallest p is taken, and the wildcards convert up to two
  non-sign near-misses (remainder 1 or 18) into holds. This is the most generous
  reading of the historical search; it errs against the Qurʾān's advantage.
  Reported, per corpus, per rung, per rule: the Qurʾān's table and p; the control
  distribution of p; and P(control p <= Qurʾān p) with the (k+1)/(N+1) correction.

ADDED AFTER THE FIRST RUN (Family A), AND SAID SO
  The frozen statistic above mixes two things: a control run with one hold cannot
  reach a small Fisher p however its hold falls, so P(control p <= Qurʾān p) is partly
  the rarity of holds over again. To separate alignment from rarity, a second figure
  is reported beside the first, conditional on the control search finding at least as
  many holds as the Qurʾān's at that rung: among those runs, the fraction whose p is
  at most the Qurʾān's. This was added after Family A's frozen result was seen and is
  labelled as a secondary analysis wherever it appears. The frozen figure is still
  reported first.
"""
import sys, random, statistics, json, time
from math import comb
sys.argv = ['alignment_control.py'] + sys.argv[1:]
NRUNS = int(sys.argv[1]) if len(sys.argv) > 1 else 10000
ONLY = sys.argv[2] if len(sys.argv) > 2 else None   # run one corpus per process; results merged afterwards
import importlib.util
spec = importlib.util.spec_from_file_location('dc', '/tmp/read/discovery_control.py')
dc = importlib.util.module_from_spec(spec); spec.loader.exec_module(dc)
random.seed(19)

SIGN_GROUPS = {('alif','l','m'), ('alif','l','m','sad'), ('alif','l','r'), ('alif','l','m','r'),
               ('ta','sin','m'), ('ta','sin')}
FORMULA = {10, 12, 13, 15, 26, 27, 28, 31}

# opening -> (letters, formula flag); the flag travels with the opening when shuffled
REAL = {s: [(dc.OPENINGS[s], s in FORMULA)] + ([(dc.SECOND[s], False)] if s in dc.SECOND else []) for s in dc.OPENINGS}

def shuffled():
    slots = random.sample(dc.SURAS, len(dc.OPENINGS))
    return {slot: REAL[s] for slot, s in zip(slots, dc.OPENINGS)}

def fisher_less(a, b, c, d):
    """P(sign holds <= a) — table [[a, b], [c, d]] = [[sign hold, sign fail], [non hold, non fail]]"""
    N = a + b + c + d; K = a + b; n = a + c
    if K == 0 or n == 0: return 1.0
    return sum(comb(K, k) * comb(N - K, n - k) for k in range(0, a + 1)) / comb(N, n)

def labelled_cases(assign):
    """cases at each rung with both labels. assign: {unit: [(letters, formula_flag), ...]}"""
    plain = {u: [ls for ls, f in v] for u, v in assign.items()}
    unit_sign_g = {u: any(ls in SIGN_GROUPS for ls, f in v) for u, v in assign.items()}
    unit_sign_f = {u: any(f for ls, f in v) for u, v in assign.items()}
    out = {}
    for r in (0, 1, 2):
        cases = dc.enumerate_cases(plain, r)
        lab = {}
        for case in cases:
            us, ls = case
            # a case built on one opening takes that opening's label; a pooled case is sign if any unit is
            g = any(unit_sign_g[u] for u in us); f = any(unit_sign_f[u] for u in us)
            # single-opening cases in a unit carrying two openings (42): label by the letter-set used
            if len(us) == 1:
                u = next(iter(us)); opts = [(l, fl) for l, fl in assign[u] if set(ls) <= set(l)]
                if opts:
                    g = any(l in SIGN_GROUPS for l, fl in opts); f = any(fl for l, fl in opts)
            lab[case] = (g, f)
        out[r] = lab
    return out, plain

def align_all(units_ct, assign):
    labs, plain = labelled_cases(assign)
    rem = {}
    for case in labs[2]:
        rem[case] = {(c, b): dc.count_case(units_ct, case, c, b) % 19 for c in dc.CONVS for b in (True, False)}
    def table(cases, conv, basms, wild, rule):
        a = b = c = d = 0; near_non = 0
        for case, (g, f) in cases.items():
            s = g if rule == 'g' else f
            rs = [rem[case][(conv, bb)] for bb in basms]
            h = 0 in rs
            if not h and any(x in (1, 18) for x in rs) and not s: near_non += 1
            if s: a += h; b += (not h)
            else: c += h; d += (not h)
        if wild:
            k = min(2, near_non); c += k; d -= k
        return (a, b, c, d), fisher_less(a, b, c, d)
    res = {}
    for rule in ('g', 'f'):
        spec_ = {0: (labs[0], ['T8'], [True], False), 1: (labs[1], ['T8'], [True], False),
                 2: (labs[2], ['T8'], [True], False), 3: (labs[2], ['T8'], [True, False], False),
                 4: (labs[2], dc.CONVS, [True, False], False), 5: (labs[2], dc.CONVS, [True, False], True)}
        for r, (cases, convs, basms, wild) in spec_.items():
            best = None
            for conv in convs:
                t, p = table(cases, conv, basms, wild, rule)
                if best is None or p < best[1]: best = (t, p)
            res[(rule, r)] = best
    return res

if __name__ == '__main__':
    t0 = time.time(); RUNGS = range(6)
    obs = align_all(dc.QURAN_UNITS, REAL)
    print('OBSERVED (the Qurʾān, real openings) — table = (sign hold, sign fail, non-sign hold, non-sign fail)')
    for rule, name in (('g', 'letter-group rule'), ('f', 'formula-only rule')):
        print(f'  {name}:')
        for r in RUNGS:
            t, p = obs[(rule, r)]; print(f'    R{r}: {t}  Fisher one-tailed p = {p:.4g}')
    results = {'observed': {f'{rule}{r}': {'table': obs[(rule, r)][0], 'p': obs[(rule, r)][1]} for rule in 'gf' for r in RUNGS}, 'controls': {}}

    def report(name, dist):
        results['controls'][name] = {}
        print(f'\n{name}  ({NRUNS} runs)')
        for rule, rn in (('g', 'letter-group'), ('f', 'formula-only')):
            print(f"  {rn:<13}{'rung':<5}{'Qurʾān p':>10}{'ctrl median p':>15}{'k (ctrl<=obs)':>15}{'(k+1)/(N+1)':>13}")
            for r in RUNGS:
                d = [x[0] for x in dist[(rule, r)]]; hs = [x[1] for x in dist[(rule, r)]]
                o = obs[(rule, r)][1]; oh = obs[(rule, r)][0][0] + obs[(rule, r)][0][2]
                k = sum(1 for x in d if x <= o); pc = (k + 1) / (len(d) + 1)
                cond = [p for p, h in zip(d, hs) if h >= oh]; kc = sum(1 for p in cond if p <= o)
                print(f'  {"":<13}R{r:<4}{o:>10.4g}{statistics.median(d):>15.4g}{k:>15}{pc:>13.5f}   conditional on holds>={oh}: {kc} of {len(cond)}')
                results['controls'][name][f'{rule}{r}'] = dict(obs_p=o, obs_holds=oh, median=statistics.median(d), k=k, N=len(d), p_corr=pc, cond_k=kc, cond_n=len(cond))

    if ONLY in (None, 'A'):
        dist = {(rule, r): [] for rule in 'gf' for r in RUNGS}
        for i in range(NRUNS):
            res = align_all(dc.QURAN_UNITS, shuffled())
            for key in dist: dist[key].append((res[key][1], res[key][0][0] + res[key][0][2]))
        report('Family A — the Qurʾān, openings shuffled', dist)

    for name, path, kind in [('Ṣaḥīḥ al-Bukhārī', '/tmp/ctrl/bukhari.txt', 'openiti'),
                             ('Ṣaḥīḥ Muslim', '/tmp/ctrl/muslim.txt', 'openiti'),
                             ('Sīra of Ibn Hishām', '/tmp/ctrl/sira.txt', 'openiti'),
                             ('Jāhilī poetry', '/tmp/ctrl/jahili_raw.txt', 'plain')]:
        if ONLY is not None and (ONLY == 'A' or not name.startswith(ONLY)): continue
        toks = dc.stream(dc.load_control(path, kind)); pre = dc.Prefix(toks); dc.selfcheck(toks, pre)
        span = len(toks) - sum(dc.SURA_LEN.values())
        dist = {(rule, r): [] for rule in 'gf' for r in RUNGS}
        for i in range(NRUNS):
            res = align_all(dc.cut_units(pre, random.randrange(span + 1)), REAL)
            for key in dist: dist[key].append((res[key][1], res[key][0][0] + res[key][0][2]))
        report(f'Family B — {name}', dist)
    json.dump(results, open(f'/tmp/read/alignment_control_results{"-" + ONLY if ONLY else ""}.json', 'w'), ensure_ascii=False, indent=1)
    print(f'\nelapsed {time.time() - t0:.0f}s')