#!/usr/bin/env python3 """Retained EPUB byte accounting; no network or text outputs. Python 3.10+ stdlib.""" from pathlib import Path from fractions import Fraction import argparse,json,csv,hashlib,zipfile,posixpath,urllib.parse,xml.etree.ElementTree as ET,itertools,math IDS=[11,84,98,120,140,161,345,768,1260,1342,2701,42324] ROLES=['package_control','navigation','cover_image','other_image','provider_xhtml','spine_xhtml','other_xhtml','stylesheet','font','other_declared','unmanifested_other'] FONTS={'application/font-sfnt','application/font-woff','application/vnd.ms-opentype','application/x-font-ttf'} def local(base,href): u=urllib.parse.urlsplit(href);assert not u.scheme and not u.netloc return posixpath.normpath(posixpath.join(posixpath.dirname(base),urllib.parse.unquote(u.path))) def tag(el):return el.tag.rsplit('}',1)[-1] if isinstance(el.tag,str) else '' def bodytext(el): parts=[] def walk(x): if tag(x) in ['script','style']:return if x.text:parts.append(x.text) for child in x: walk(child) if child.tail:parts.append(child.tail) walk(el);return ' '.join(' '.join(parts).split()) def writecsv(path,rows): with path.open('w',encoding='utf-8',newline='') as f: w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows) def ranks(vals): order=sorted(range(len(vals)),key=lambda i:vals[i]);r=[None]*len(vals);j=0 while j=0 and C==sum(i.compress_size for i in infos) and U==sum(i.file_size for i in infos) imgC=sum(ledger[k]['compressed_member_bytes'] for k in ['cover_image','other_image']);imgU=sum(ledger[k]['uncompressed_member_bytes'] for k in ['cover_image','other_image']);assert U>0 for role,d in ledger.items():role_rows.append({'gutenberg_id':ident,'role':role,**d,'compressed_archive_share_pct':100*d['compressed_member_bytes']/A,'uncompressed_member_share_pct':100*d['uncompressed_member_bytes']/U,'uncompressed_over_compressed_ratio':d['uncompressed_member_bytes']/d['compressed_member_bytes'] if d['compressed_member_bytes'] else None}) books.append({'gutenberg_id':ident,'title':' ; '.join(x['title']),'creator':' ; '.join(x['creator']),'canonical_source_url':x['received_source_url'],'canonical_landing_url':x['canonical_landing_url'],'source_retrieved_utc':x['original_retrieved_utc'],'archive_bytes':A,'member_count':len(infos),'compressed_member_bytes':C,'zip_container_overhead_bytes':O,'uncompressed_member_bytes':U,'uncompressed_spine_xhtml_bytes':ledger['spine_xhtml']['uncompressed_member_bytes'],'normalized_literal_body_characters':literal_chars,'compressed_spine_xhtml_bytes':ledger['spine_xhtml']['compressed_member_bytes'],'compressed_image_bytes':imgC,'uncompressed_image_bytes':imgU,'compressed_image_archive_share_pct':100*imgC/A,'uncompressed_image_member_share_pct':100*imgU/U,'image_share_difference_pp':100*imgC/A-100*imgU/U,'image_majority_compressed_archive':int(2*imgC>A),'image_majority_uncompressed_members':int(2*imgU>U),'image_majority_flip':int((2*imgC>A)!=(2*imgU>U)),'archive_overhead_pct':100*O/A,'uncompressed_members_over_archive':U/A,'declared_cover_compressed_bytes':ledger['cover_image']['compressed_member_bytes'],'other_image_compressed_bytes':ledger['other_image']['compressed_member_bytes']}) allpairs=[];sums={} for subset,omit in [('main',set()),('without_higher_id_frankenstein',{42324}),('without_two_largest_archives',{120,1342})]: bs=[b for b in books if b['gutenberg_id'] not in omit];result={'files':len(bs),'omitted_ids':sorted(omit),'majority_flip_files':[b['gutenberg_id'] for b in bs if b['image_majority_flip']],'proxies':{}} for proxy in ['uncompressed_spine_xhtml_bytes','normalized_literal_body_characters']: prs=comparisons(bs,proxy);counts={status:sum(x['comparison']==status for x in prs) for status in ['concordant','discordant','tie_proxy','tie_archive','tie_both']};result['proxies'][proxy]={'pairs':len(prs),**counts,'spearman_rank_correlation':corr([b[proxy] for b in bs],[b['archive_bytes'] for b in bs])} allpairs +=[{'subset':subset,**row} for row in prs] sums[subset]=result summary={'files':12,'same_work_caution':'Two Frankenstein transcriptions; file cases are not independent works or a population sample','measurement_definition':'Declared member media roles and two non-rendered text proxies, twelve fixed source files','received_archive_total_bytes':sum(b['archive_bytes'] for b in books),'uncompressed_members_total_bytes':sum(b['uncompressed_member_bytes'] for b in books),'zip_container_overhead_total_bytes':sum(b['zip_container_overhead_bytes'] for b in books),'roles':ROLES,'pooled_roles':{r:{k:sum(v[k] for v in role_rows if v['role']==r) for k in ['member_count','compressed_member_bytes','uncompressed_member_bytes']} for r in ROLES},'subsets':sums,'interpretation_limits':['No rendered page/display length or validated literary word count','Literal-body proxy not layout visibility or content equivalence','Archive bytes not network wire-size, latency, RAM or device performance','Declared media role not image quality or a recommendation to delete assets','No inferential p-values, CIs, population predictions or quality ranking','Navigation-failure and accessibility claims are outside this analysis'],'public_safe_metrics_only':True} writecsv(out/'file-metrics.csv',books);writecsv(out/'role-metrics.csv',role_rows);writecsv(out/'finite-order-comparisons.csv',allpairs); (out/'results.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n') print(json.dumps({'summary':sums,'file_metrics':[{k:b[k] for k in ['gutenberg_id','archive_bytes','uncompressed_spine_xhtml_bytes','normalized_literal_body_characters','compressed_image_archive_share_pct','uncompressed_image_member_share_pct','image_majority_flip','declared_cover_compressed_bytes','other_image_compressed_bytes']} for b in books],'pooled_roles':summary['pooled_roles']},indent=2)) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('--input-dir',type=Path,required=True);p.add_argument('--source-frame',type=Path,required=True);p.add_argument('--output-dir',type=Path,required=True);a=p.parse_args();main(a.input_dir,a.source_frame,a.output_dir)