"""Re-extract public aggregate tables and derive descriptive contrasts.

Run from this directory or any cwd. Requires requests and PyMuPDF.
No microdata, identities, inferred confidence intervals or exact decomposition.
"""
from pathlib import Path
from datetime import datetime, timezone
import csv, hashlib, json, re
import fitz
import requests

ROOT = Path(__file__).resolve().parent
URL = 'https://www.statistik.at/fileadmin/user_upload/ZVE_2021-22_barrierefrei.pdf'
PDF = ROOT / 'austria-time-use-2021-22.pdf'
if not PDF.exists():
    response = requests.get(URL, timeout=90)
    response.raise_for_status()
    PDF.write_bytes(response.content)
doc = fitz.open(PDF)
activities = ['Zeitungen und Zeitschriften lesen', 'Bücher lesen', 'Anderes und nicht näher bestimmtes']
rows = []
for page_index, period in [(157, 'all_days'), (161, 'weekdays_excluding_holidays'), (165, 'weekends_including_holidays')]:
    lines = [line.strip() for line in doc[page_index].get_text().splitlines()]
    for label in activities:
        start = lines.index(label) + 1
        if label == activities[2]:
            assert lines[start] == 'Lesen'
            start += 1
        values = lines[start:start+9]
        assert len(values) == 9
        for group_index, sex in enumerate(['all', 'female', 'male']):
            a, p, d = values[group_index*3:group_index*3+3]
            assert re.fullmatch(r'\d{1,2}:\d{2}', a), values
            assert re.fullmatch(r'\d{1,2}:\d{2}', d), values
            minute = lambda text: sum(int(x)*factor for x, factor in zip(text.split(':'), [60, 1]))
            rows.append({'period': period, 'sex': sex,
                         'activity': label + (' Lesen' if label == activities[2] else ''),
                         'average_minutes_all_person_days': minute(a),
                         'published_participation_percent': float(p.replace(',', '.')),
                         'participation_reference': 'at_least_one_of_two_diary_days' if period == 'all_days' else 'specified_day_type',
                         'average_minutes_active_days': minute(d),
                         'source_pdf_page': page_index+1,
                         'source_printed_page': page_index-1})
assert len(rows) == 27
def cell(period, sex, activity='Bücher lesen'):
    return next(x for x in rows if x['period']==period and x['sex']==sex and x['activity']==activity)
contrasts = []
for period in ['all_days', 'weekdays_excluding_holidays', 'weekends_including_holidays']:
    female, male = cell(period,'female'), cell(period,'male')
    contrasts.append({'period':period,
                      'female_minus_male_participation_percentage_points':round(female['published_participation_percent']-male['published_participation_percent'],1),
                      'female_minus_male_all_person_minutes':female['average_minutes_all_person_days']-male['average_minutes_all_person_days'],
                      'female_minus_male_active_day_minutes':female['average_minutes_active_days']-male['average_minutes_active_days']})
# Lock manually visually checked source cells, not implementation-only expectations.
assert cell('all_days','all')['published_participation_percent']==14.4
assert cell('all_days','all')['average_minutes_all_person_days']==7
assert cell('all_days','all')['average_minutes_active_days']==75
assert cell('all_days','female')['published_participation_percent']==18.1
assert cell('all_days','male')['average_minutes_active_days']==79
with (ROOT/'source-table.csv').open('w',encoding='utf-8-sig',newline='') as f:
    writer=csv.DictWriter(f,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
result={'study_id':'AT-READING-DENOMINATORS-2021-22','observation_period':'October 2021–December 2022',
        'universe':'People aged 10+ with principal residence in Austria; published survey estimates',
        'source_rows':rows,'derived_descriptive_contrasts':contrasts,
        'no_exact_participation_duration_decomposition':True,
        'reason':'All-days participation is a two-diary-day union; duration is conditional on active days. Different denominators cannot be multiplied.',
        'uncertainty':'Published rounded aggregates only. No reading-cell standard errors in these tables; differences are descriptive, not claims of statistical significance.',
        'source_attribution':'STATISTIK AUSTRIA, Zeitverwendung 2021/22; adapted extraction and descriptive comparisons by Alisson Bento.'}
(ROOT/'results.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
provenance={'download_url':URL,'retrieved_at_utc':datetime.fromtimestamp(PDF.stat().st_mtime,timezone.utc).isoformat(),
            'sha256':hashlib.sha256(PDF.read_bytes()).hexdigest(),'publisher':'STATISTIK AUSTRIA',
            'publication':'Zeitverwendung 2021/22 – Ergebnisse der Zeitverwendungserhebung','publication_year':2023,
            'source_pages_pdf':[4,30,31,36,155,158,159,162,163,166],
            'reuse':'Imprint permits reproduction, dissemination, public access and adaptation with accurate reproduction, STATISTIK AUSTRIA attribution and adaptation notice.',
            'source_scope':'Published aggregate statistics only; original survey, not an Alisson Bento survey.',
            'destatis_fallback_reason':'Legacy REST URL returned JavaScript HTML, not the requested table. No German household results fabricated.'}
(ROOT/'provenance.json').write_text(json.dumps(provenance,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
print(json.dumps({'rows':len(rows),'contrasts':contrasts,'sha256':provenance['sha256']},ensure_ascii=False))