"""Frozen-edition structural analysis. Python 3 standard library; no network or raw exports.""" import argparse, csv, hashlib, json, re, statistics from pathlib import Path TOKEN = re.compile("[A-Za-z]+(?:['’][A-Za-z]+)*") SPECS = {1342: ('3f6bb9d6f78e0293b56acd4714dd68cb7d6d1d293402031ce9d5a216bcaf9d75', 'It is a truth universally', 'been the means of uniting them.'), 42324: ('a94b1be5cf19ad0b9965577b86f683f8ee898bb8b0effe964bbd7421cd80abb5', 'LETTER I.', 'lost in darkness and distance.')} HEAD = re.compile('(?im)^(?:(CHAPTER|LETTER) ([IVXLCDM]+)\\.?|(WALTON, _in continuation_\\.))[ \\t]*$') def mask(s, prefix): count = words = 0 out = list(s) i = 0 while (i := s.find(prefix, i)) != -1: depth = 0 j = i while j < len(s): if s[j] == '[': depth += 1 elif s[j] == ']': depth -= 1 if not depth: break j += 1 if depth: raise ValueError('Unbalanced block') count += 1 words += len(TOKEN.findall(s[i:j + 1])) out[i:j + 1] = ['\n' if c == '\n' else ' ' for c in s[i:j + 1]] i = j + 1 return (''.join(out), count, words) def stats(xs, total): q = statistics.quantiles(xs, n=4, method='inclusive') return dict(n=len(xs), tokens=sum(xs), coverage_pct=100 * sum(xs) / total, min=min(xs), q1=q[0], median=statistics.median(xs), q3=q[2], max=max(xs), mean=statistics.mean(xs), population_cv=statistics.pstdev(xs) / statistics.mean(xs), largest_unit_share_pct=100 * max(xs) / sum(xs)) def run(inputs, out): out.mkdir(parents=True, exist_ok=True) rows = [] summaries = [] audits = [] for id, (sha, start, end) in SPECS.items(): raw = (inputs / f'fiction-pg{id}.txt').read_bytes() if not hashlib.sha256(raw).hexdigest() == sha: raise ValueError('Frozen source or structural invariant failed') s = raw.decode('utf-8-sig').replace('\r\n', '\n').replace('\r', '\n') a = s.index(start) z = s.index(end, a) + len(end) body, illustrations, _ = mask(s[a:z], '[Illustration') body, footnotes, footwords = mask(body, '[Footnote') heads = list(HEAD.finditer(body)) units = [] if id == 1342: units.append(('CHAPTER', 'I', 0, 0)) for m in heads: units.append((m[1].upper() if m[1] else 'FRAME', m[2] or 'WALTON', m.start(), m.end())) if not len(units) == (61 if id == 1342 else 29): raise ValueError('Frozen source or structural invariant failed') def roman(n): result = '' for v, g in [(1000, 'M'), (900, 'CM'), (500, 'D'), (400, 'CD'), (100, 'C'), (90, 'XC'), (50, 'L'), (40, 'XL'), (10, 'X'), (9, 'IX'), (5, 'V'), (4, 'IV'), (1, 'I')]: while n >= v: result += g n -= v return result if not [u[1] for u in units if u[0] == 'CHAPTER'] == [roman(i) for i in range(1, 62 if id == 1342 else 25)]: raise ValueError('Frozen source or structural invariant failed') if id == 42324: if not [u[1] for u in units if u[0] == 'LETTER'] == ['I', 'II', 'III', 'IV']: raise ValueError('Frozen source or structural invariant failed') total = 0 values = [] chapters = [] for i, (kind, num, h, b) in enumerate(units): e = units[i + 1][2] if i + 1 < len(units) else len(body) text = body[b:e] n = len(TOKEN.findall(text)) if not n > 0: raise ValueError('Frozen source or structural invariant failed') total += n values.append(n) if kind == 'CHAPTER': chapters.append(n) rows.append(dict(pg_id=id, unit_index=i + 1, kind=kind, number=num, tokens=n, normalized_start_char=a + b, normalized_end_char=a + e)) audits.append(dict(pg_id=id, unit_index=i + 1, kind=kind, number=num, heading_start_char=a + h, body_start_char=a + b, body_end_char=a + e, heading_sha256=hashlib.sha256(s[a + h:a + b].encode()).hexdigest(), body_sha256=hashlib.sha256(text.encode()).hexdigest(), boundary_review='Adjacent prose inspected; named sequence and exclusion checks passed' if not (id == 1342 and i == 0) else 'Opening narrative anchor inspected; chapter I label embedded in preceding illustration')) stripped = HEAD.sub('', body) if not len(TOKEN.findall(stripped)) == total: raise ValueError('Frozen source or structural invariant failed') result = dict(pg_id=id, total_tokens=total, all_units=stats(values, total), chapters_separate_frame=stats(chapters, total), illustration_blocks_excluded=illustrations, footnote_blocks_excluded=footnotes, footnote_tokens_excluded=footwords, partition_token_check=True) if id == 42324: result['chapters_frame_attached'] = stats(chapters[:-1] + [chapters[-1] + values[-1]], total) result['opening_letters_tokens'] = sum(values[:4]) result['closing_frame_tokens'] = values[-1] result['initial_letter_metadata_tokens'] = len(TOKEN.findall(body[units[0][3]:body.index('You will rejoice to hear')])) summaries.append(result) with (out / 'units.csv').open('w', newline='') as f: w = csv.DictWriter(f, fieldnames=rows[0].keys(), lineterminator='\n') w.writeheader() w.writerows(rows) for name, data in [('results.json', summaries), ('boundary-diagnostics.json', audits)]: (out / name).write_text(json.dumps(data, indent=2) + '\n') return summaries if __name__ == '__main__': p = argparse.ArgumentParser() p.add_argument('--input-dir', required=True, type=Path) p.add_argument('--output-dir', required=True, type=Path) a = p.parse_args() print(json.dumps(run(a.input_dir, a.output_dir), indent=2))