"""Reproduce an explicit-denominator reading matrix from frozen Eurostat JSON-stat."""
from pathlib import Path
import sys
sys.dont_write_bytecode=True
from datetime import datetime,timezone
from decimal import Decimal
from itertools import product
import json,csv,hashlib,html,math
import xml.etree.ElementTree as ET

raw=Path(sys.argv[1])
root=Path(sys.argv[2]) if len(sys.argv)>2 else Path.cwd()/'reproduced-country'
root.mkdir(parents=True,exist_ok=True)
assert hashlib.sha256(raw.read_bytes()).hexdigest()=='985d8d2c7d29c0ff83a7d5eb25a923db7290ca7e2a0d5f93d01a7952b8b5c1c4'
d=json.loads(raw.read_text(encoding='utf-8'))
ids=d['id'];sizes=d['size'];dimension=d['dimension']
codes={k:sorted(dimension[k]['category']['index'],key=dimension[k]['category']['index'].get) for k in ids}
labels={k:dimension[k]['category']['label'] for k in ids}
strides={k:math.prod(sizes[i+1:]) for i,k in enumerate(ids)}
def observation(**kwargs):
 i=sum(dimension[k]['category']['index'][kwargs[k]]*strides[k] for k in ids)
 return {'value':d['value'].get(str(i)),'flag':d.get('status',{}).get(str(i),''),'source_cell_index':i}
def write_csv(name,rows,columns):
 with (root/name).open('w',encoding='utf-8',newline='') as f:
  w=csv.DictWriter(f,fieldnames=columns,lineterminator='\n');w.writeheader();w.writerows(rows)
base={'freq':'A','unit':'PC','time':'2022'}
assert ids==['freq','n_book','isced11','sex','age','unit','geo','time']
assert len(d['value'])==27524
assert all(Decimal(str(v))*10==(Decimal(str(v))*10).to_integral_value() for v in d['value'].values())
rows=[]
for geo in codes['geo']:
 cells={n:observation(**base,n_book=n,isced11='TOTAL',sex='T',age='Y_GE16',geo=geo) for n in codes['n_book']}
 flags=sorted({c['flag'] for c in cells.values() if c['flag']})
 vals={n:c['value'] for n,c in cells.items()}
 reasons=[]
 if any(v is None for v in vals.values()):reasons.append('missing_category')
 if any(c['flag'] in ['u','n'] for c in cells.values()):reasons.append('source_reliability_flag_u_or_n')
 if any(v is not None and not 0<=v<=100 for v in vals.values()):reasons.append('outside_percent_range')
 total=sum(Decimal(str(v)) for v in vals.values()) if not any(v is None for v in vals.values()) else None
 if total is not None and abs(total-100)>Decimal('.2'):reasons.append('category_sum_outside_rounding_tolerance')
 reach=sum(Decimal(str(vals[n])) for n in ['LT5','5-9','GE10']) if all(vals[n] is not None for n in ['LT5','5-9','GE10']) else None
 ratio=Decimal(100)*Decimal(str(vals['GE10']))/reach if reach else None
 row={'geo':geo,'country_or_aggregate':labels['geo'][geo],'aggregate':geo in ['EU27_2020','EA20'],
  'no_books_pc':vals['0'],'one_to_four_pc':vals['LT5'],'five_to_nine_pc':vals['5-9'],'ten_plus_pc':vals['GE10'],
  'sum_categories_pc':float(total) if total is not None else None,'any_book_pc':float(reach) if reach is not None else None,
  'ten_plus_among_readers_pc':float(ratio) if ratio is not None else None,'source_flags':'|'.join(flags),
  'primary_eligible':not reasons,'exclusion_reason':'|'.join(reasons),
  **{f'cell_{n}_index':c['source_cell_index'] for n,c in cells.items()},
  **{f'flag_{n}':c['flag'] for n,c in cells.items()}}
 rows.append(row)
write_csv('country-denominator-matrix.csv',rows,list(rows[0]))
eligible=[r for r in rows if not r['aggregate'] and r['primary_eligible']]
excluded=[r for r in rows if not r['aggregate'] and not r['primary_eligible']]