"""Stratified appendix to the shared European reading study; frozen bytes only."""
from pathlib import Path
import sys
sys.dont_write_bytecode=True
import csv,hashlib,html,json,collections
P=Path(sys.argv[2]) if len(sys.argv)>2 else Path.cwd()/'reproduced-appendix'
P.mkdir(parents=True,exist_ok=True)
raw=Path(sys.argv[1]).read_bytes()
assert hashlib.sha256(raw).hexdigest()=='985d8d2c7d29c0ff83a7d5eb25a923db7290ca7e2a0d5f93d01a7952b8b5c1c4'
j=json.loads(raw);dim=j['dimension'];ids=j['id'];sizes=j['size']
def cell(geo,sex,age,edu,book):
    keys={'freq':'A','n_book':book,'isced11':edu,'sex':sex,'age':age,'unit':'PC','geo':geo,'time':'2022'}
    idx=0
    for key,size in zip(ids,sizes):idx=idx*size+dim[key]['category']['index'][keys[key]]
    return idx,j.get('value',{}).get(str(idx)),j.get('status',{}).get(str(idx),'')
cells=[];rows=[]
for geo in ['PT','ES','EU27_2020']:
 for age in dim['age']['category']['index']:
  for edu in dim['isced11']['category']['index']:
   r={'geo':geo,'age':age,'education':edu,'age_label':dim['age']['category']['label'][age],'education_label':dim['isced11']['category']['label'][edu]};usable=True;flags=[];reason=[]
   for sex in ['F','M']:
    x=[cell(geo,sex,age,edu,b) for b in ['0','LT5','5-9','GE10']]
    for book,(idx,value,flag) in zip(['0','LT5','5-9','GE10'],x):cells.append({'geo':geo,'sex':sex,'age':age,'education':edu,'books':book,'source_index':idx,'source_value_pct':value,'source_flag':flag})
    vals=[v for _,v,_ in x];fs=','.join(sorted(set(f for _,_,f in x if f)));flags.extend(f for _,_,f in x if f)
    r[sex+'_source_indices']='|'.join(str(i) for i,_,_ in x);r[sex+'_flags']=fs;r[sex+'_sum_pct']=sum(vals) if all(v is not None for v in vals) else None
    valid=all(v is not None and 0<=v<=100 for v in vals) and abs(sum(v or 0 for v in vals)-100)<=0.200000001
    reach=sum(vals[1:]) if valid else None
    valid=valid and reach>0
    r[sex+'_reach_pct']=reach if valid else None;r[sex+'_conditional10_pct_readers']=100*vals[3]/reach if valid else None
    if not valid:usable=False;reason.append(sex+': missing/invalid bins or nonpositive reader denominator')
   r['numerically_computable']=usable;r['primary_unflagged']=usable and not flags
   r['reach_gap_F_minus_M_pp']=r['F_reach_pct']-r['M_reach_pct'] if usable else None
   r['conditional10_gap_F_minus_M_pp']=r['F_conditional10_pct_readers']-r['M_conditional10_pct_readers'] if usable else None
   r['status']='primary_unflagged' if r['primary_unflagged'] else ('flagged_numeric_only' if usable else 'excluded_missing_or_invalid')
   r['reason']='; '.join(reason) or ('source flags retained; excluded from primary' if flags else '')
   rows.append(r)
def write(name,rows):
 with (P/name).open('w',encoding='utf-8-sig',newline='') as f:
  w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
write('stratified-appendix.csv',rows);write('source-cells.csv',cells)