# -*- coding: utf-8 -*- """ THE PATTERN-DISCOVERY CONTROL — quantity (c), discovery-adjusted surprise. Protocol frozen 7 September 2026, before any run. Nothing below was altered after the numbers were seen. (Companion to test29.py; same discipline.) QUESTION Given the freedoms the historical Code 19 investigators had, how many "holds" (counts that are multiples of 19) does a motivated search find on a text with no stake in the number nineteen — and how does that compare with the same search run on the Qurʾān? WHAT IS COMPARED The SAME enumeration is run on the Qurʾān with its real openings and on each control. The observed statistic is the number of holds the enumeration finds on the real text. The control statistic is the number it finds on the control. The reported figure is p = P(control holds >= observed holds), per corpus, per rung. No figure here is multiplied with any figure in Appendix D Section V. UNITS The Qurʾān: 114 sūrahs, each with the basmala prepended, exactly as count19.py. A control text: 114 consecutive units cut from the text so that unit i has the same number of letters as sūrah i (letters = [ء-يٱ]); the basmala is prepended to every unit so that the basmala freedom exists on both sides. The cut starts at a random offset (uniform over the text) drawn fresh per run. TARGETS The 29 disconnected-letter openings, as letter-sets, with their multiplicities and positions (sūrah 42 carries two: Ḥ-M as a member of the Ḥ-M cluster, and ʿ-S-Q). Family A (same text, shuffled targets): the 29 openings are reassigned to a random permutation of the 114 sūrahs. This isolates the discovery process from the text. Family B (other texts, same targets): unit i receives sūrah i's opening. Controls: Ṣaḥīḥ al-Bukhārī; Ṣaḥīḥ Muslim; Ibn Hishām's Sīra (OpenITI, JK/Shamela editions); pre-Islamic (Jāhilī) poetry, 2,350 poems (tahaio/arabicpoetry, era = العصر الجاهلي). OpenITI markup, Latin, digits and punctuation are stripped; Arabic letters and vowel marks are kept. THE INVESTIGATOR'S FREEDOMS — a ladder of rungs, each adding one freedom R0 The frozen 29-opening protocol (test29.py): every opening is a case; identical openings are ONE case and are summed; the whole letter-set is counted; basmala in; primary convention T8 with Rule 2. (Real text: 14 cases, 5 hold.) R1 + case shaping: any opening unit singly OR the full cluster of units sharing its opening; and for each, EITHER the whole letter-set OR any single letter of it. (The Ḥā-Mīm cluster move; the Qāf-42 move.) R2 + cross-opening letter clusters: any single letter summed over every unit whose opening contains it. (The Ṣād-cluster move: 7 + 19 + 38.) R3 + basmala in or out, chosen per case. R4 + counting convention: one of T1–T8 chosen for the whole run to maximise holds. R5 + variant-reading wildcard: ±1 on at most two cases. (The Rule-8 move.) A case is a distinct (unit-set, letter-set) pair; the same pair reached two ways is counted once. At each rung the statistic is the number of distinct cases that hold. RUNS 10,000 per corpus per family (Family A: 10,000 permutations; Family B: 10,000 random offsets for each of four texts). Seed 19. WHAT IS REPORTED For each rung: the observed hold count on the real text; the control distribution (median, 95th percentile, maximum); and p = P(control >= observed). Per corpus. Also, for the reader's orientation: how many candidate cases each rung enumerates. """ import sys, io, re, contextlib, importlib.util, random, statistics, json, time from collections import defaultdict random.seed(19) NRUNS = int(sys.argv[1]) if len(sys.argv) > 1 else 10000 buf = io.StringIO() with contextlib.redirect_stdout(buf): spec = importlib.util.spec_from_file_location('c19', '/tmp/read/count19.py') m = importlib.util.module_from_spec(spec); sys.argv = ['count19.py'] try: spec.loader.exec_module(m) except SystemExit: pass CONVS = ['T1', 'T2', 'T3', 'T4', 'T5', 'T6', 'T7', 'T8'] LETTERS = ['alif', 'l', 'm', 'r', 'sad', 'n', 'q', 'ayn', 'sin', 'h', 'k', 'ha', 'ta', 'ya'] LETTER_RE = re.compile('[ء-يٱ]') # --- the 29 openings, as (sūrah, letter-set). 42 carries two. OPENINGS = { 2: ('alif','l','m'), 3: ('alif','l','m'), 7: ('alif','l','m','sad'), 10: ('alif','l','r'), 11: ('alif','l','r'), 12: ('alif','l','r'), 13: ('alif','l','m','r'), 14: ('alif','l','r'), 15: ('alif','l','r'), 19: ('k','h','ya','ayn','sad'), 20: ('ta','h'), 26: ('ta','sin','m'), 27: ('ta','sin'), 28: ('ta','sin','m'), 29: ('alif','l','m'), 30: ('alif','l','m'), 31: ('alif','l','m'), 32: ('alif','l','m'), 36: ('ya','sin'), 38: ('sad',), 40: ('ha','m'), 41: ('ha','m'), 42: ('ha','m'), 43: ('ha','m'), 44: ('ha','m'), 45: ('ha','m'), 46: ('ha','m'), 50: ('q',), 68: ('n',), } SECOND = {42: ('ayn','sin','q')} SURAS = list(range(1, 115)) # ------------------------------------------------------------------ text prep def unit_counts(text): """letter counts of one unit, per convention: {conv: {letter: n}}""" return {c: {L: m.count(text, L, c) for L in LETTERS} for c in CONVS} def basmala_counts(): return unit_counts(m.BASMALA) BASM = basmala_counts() SURA_LEN = {s: len(LETTER_RE.findall(m.SURA[s])) for s in SURAS} QURAN_UNITS = {s: unit_counts(m.SURA[s]) for s in SURAS} # without basmala def load_control(path, kind): raw = open(path, encoding='utf8').read() if kind == 'openiti': raw = raw.split('#META#Header#End#')[-1] # '# ' opens a paragraph in OpenITI mARkdown: keep those lines; non-Arabic is stripped below raw = re.sub(r'[^ء-يٱً-ْٰ\s]', ' ', raw) # letters + harakat only return re.sub(r'\s+', ' ', raw) def stream(text): """the text as a list of tokens, each a letter with its trailing marks""" return re.findall(r'[ء-يٱ][ً-ْٰ]*', text) import numpy as np SH = m.SHADDA HITS = {L: set(m.SIMPLE[L]) for L in m.SIMPLE} HITS['alif'] = set(m.ALIF_BASE); HITS['ya'] = {'ي', m.MAQSURA}; HITS['h'] = {'ه', m.MARBUTA} class Prefix: """prefix sums of the per-token features count19.count() depends on, so that count(slice, letter, conv) is O(1). Validated against m.count in selfcheck().""" def __init__(self, toks): n = len(toks); feats = {} for L in LETTERS: feats[L] = np.zeros(n, np.int8); feats[L + '_sh'] = np.zeros(n, np.int8) for k in ('wasla', 'wasla_sh', 'hy', 'hy_sh', 'dagger'): feats[k] = np.zeros(n, np.int8) for i, t in enumerate(toks): ch = t[0]; sh = SH in t for L in LETTERS: if ch in HITS[L]: feats[L][i] = 1 if sh: feats[L + '_sh'][i] = 1 if ch == m.WASLA: feats['wasla'][i] = 1 if sh: feats['wasla_sh'][i] = 1 if ch == m.HAMZA_YA: feats['hy'][i] = 1 if sh: feats['hy_sh'][i] = 1 feats['dagger'][i] = t.count(m.DAGGER) self.P = {k: np.concatenate([[0], np.cumsum(v, dtype=np.int64)]) for k, v in feats.items()} self.n = n def seg(self, k, a, b): return int(self.P[k][b] - self.P[k][a]) def count(self, a, b, L, conv): wasla, dagger, shadda, hy = m.CONV[conv] n = self.seg(L, a, b) + (self.seg(L + '_sh', a, b) if shadda else 0) if L == 'alif': if wasla: n += self.seg('wasla', a, b) + (self.seg('wasla_sh', a, b) if shadda else 0) if dagger: n += self.seg('dagger', a, b) if L == 'ya' and hy: n += self.seg('hy', a, b) + (self.seg('hy_sh', a, b) if shadda else 0) return n def cut_units(pre, offset): """114 units of the sūrah letter-lengths, consecutive from offset""" out = {}; pos = offset for s in SURAS: n = SURA_LEN[s] out[s] = {c: {L: pre.count(pos, pos + n, L, c) for L in LETTERS} for c in CONVS}; pos += n return out def selfcheck(toks, pre, k=40): """prefix counts must equal count19.count on the same slice, every letter, every convention""" for _ in range(k): a = random.randrange(len(toks) - 5000); b = a + random.randrange(1, 5000) seg = ''.join(toks[a:b]) for c in CONVS: for L in LETTERS: assert pre.count(a, b, L, c) == m.count(seg, L, c), (a, b, L, c) return True # ------------------------------------------------------------------ the enumeration def enumerate_cases(assign, rung): """assign: {unit: [letter-set, ...]} (a unit may carry two openings). returns a set of frozen cases: (frozenset(units), frozenset(letters))""" groups = defaultdict(list) # letter-set -> units for u, sets in assign.items(): for ls in sets: groups[ls].append(u) cases = set() if rung == 0: for ls, units in groups.items(): cases.add((frozenset(units), frozenset(ls))) return cases # R1: singles or full cluster; whole set or any single letter for ls, units in groups.items(): shapes = [frozenset([u]) for u in units] + ([frozenset(units)] if len(units) > 1 else []) letter_choices = [frozenset(ls)] + ([frozenset([L]) for L in ls] if len(ls) > 1 else []) for sh in shapes: for lc in letter_choices: cases.add((sh, lc)) if rung >= 2: by_letter = defaultdict(set) for u, sets in assign.items(): for ls in sets: for L in ls: by_letter[L].add(u) for L, units in by_letter.items(): if len(units) > 1: cases.add((frozenset(units), frozenset([L]))) return cases def count_case(units_ct, case, conv, basmala): us, ls = case n = 0 for u in us: for L in ls: n += units_ct[u][conv][L] + (BASM[conv][L] if basmala else 0) return n def holds_all(units_ct, assign): """holds at every rung, one pass. returns {rung: holds}, {rung: n_cases}""" c0 = enumerate_cases(assign, 0); c1 = enumerate_cases(assign, 1); c2 = enumerate_cases(assign, 2) rem = {} # case -> {(conv, basm): remainder} for case in c2: rem[case] = {(c, b): count_case(units_ct, case, c, b) % 19 for c in CONVS for b in (True, False)} def score(cases, convs, basms, wildcard): best = 0 for c in convs: h = 0; near = 0 for case in cases: rs = [rem[case][(c, b)] for b in basms] if 0 in rs: h += 1 elif any(r in (1, 18) for r in rs): near += 1 if wildcard: h += min(2, near) best = max(best, h) return best H = {0: score(c0, ['T8'], [True], False), 1: score(c1, ['T8'], [True], False), 2: score(c2, ['T8'], [True], False), 3: score(c2, ['T8'], [True, False], False), 4: score(c2, CONVS, [True, False], False), 5: score(c2, CONVS, [True, False], True)} N = {0: len(c0), 1: len(c1), 2: len(c2), 3: len(c2), 4: len(c2), 5: len(c2)} return H, N REAL_ASSIGN = {s: [OPENINGS[s]] + ([SECOND[s]] if s in SECOND else []) for s in OPENINGS} def shuffled_assign(): slots = random.sample(SURAS, len(OPENINGS)) return {slot: REAL_ASSIGN[s] for slot, s in zip(slots, OPENINGS)} # ------------------------------------------------------------------ run if __name__ == '__main__': t0 = time.time() RUNGS = range(6) H, N = holds_all(QURAN_UNITS, REAL_ASSIGN) obs = {r: (H[r], N[r]) for r in RUNGS} print('OBSERVED (the Qurʾān, real openings):') for r in RUNGS: print(f' R{r}: {obs[r][0]} holds among {obs[r][1]} candidate cases') results = {'observed': {r: obs[r][0] for r in RUNGS}, 'cases': {r: obs[r][1] for r in RUNGS}, 'controls': {}} def report(name, dist): results['controls'][name] = {} print(f'\n{name} ({NRUNS} runs)') print(f" {'rung':<5}{'observed':>9}{'median':>8}{'p95':>6}{'max':>5}{'P(ctrl>=obs)':>14}") for r in RUNGS: d = dist[r]; o = obs[r][0] p = sum(1 for x in d if x >= o) / len(d) q95 = sorted(d)[int(0.95 * len(d)) - 1] print(f' R{r:<4}{o:>9}{statistics.median(d):>8.0f}{q95:>6}{max(d):>5}{p:>14.4g}') results['controls'][name][r] = dict(observed=o, median=statistics.median(d), p95=q95, max=max(d), p=p) # Family A dist = {r: [] for r in RUNGS} for i in range(NRUNS): H, _ = holds_all(QURAN_UNITS, shuffled_assign()) for r in RUNGS: dist[r].append(H[r]) report('Family A — the Qurʾān, openings shuffled', dist) # Family B for name, path, kind in [('Ṣaḥīḥ al-Bukhārī', '/tmp/ctrl/bukhari.txt', 'openiti'), ('Ṣaḥīḥ Muslim', '/tmp/ctrl/muslim.txt', 'openiti'), ('Sīra of Ibn Hishām', '/tmp/ctrl/sira.txt', 'openiti'), ('Jāhilī poetry', '/tmp/ctrl/jahili_raw.txt', 'plain')]: toks = stream(load_control(path, kind)); pre = Prefix(toks); selfcheck(toks, pre) need = sum(SURA_LEN.values()) span = len(toks) - need print(f'\n[{name}: {len(toks):,} letters; Qurʾān needs {need:,}; offset range {span:,}; prefix self-check passed]') dist = {r: [] for r in RUNGS} for i in range(NRUNS): H, _ = holds_all(cut_units(pre, random.randrange(span + 1)), REAL_ASSIGN) for r in RUNGS: dist[r].append(H[r]) report(f'Family B — {name}', dist) json.dump(results, open('/tmp/read/discovery_control_results.json', 'w'), ensure_ascii=False, indent=1) print(f'\nelapsed {time.time() - t0:.0f}s')