"""Refresh CSO snapshots and generate public data. Stdlib only; run from any directory."""
import argparse
import csv
import hashlib
import io
import json
import math
from html import escape
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import Request, urlopen

ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'everyday-movement'
TABLES = ('F7065', 'F7068', 'F7122')
API = 'https://ws.cso.ie/public/api.restful/PxStat.Data.Cube_API.ReadDataset/{}/JSON-stat/2.0/en'
GROUPS = {'work': ('F7065C05', '904', 'Travel to work · workers aged 15+'),
          'primary': ('F7065C02', '901', 'Travel to school · pupils aged 5–12'),
          'secondary': ('F7065C03', '902', 'Travel to school or college · students aged 13–18'),
          'college': ('F7065C04', '903', 'Travel to college · students aged 19+')}


def codes(dimension):
    index = dimension['category']['index']
    return index if isinstance(index, list) else sorted(index, key=index.get)


def cell(cube, **selection):
    offset = 0
    for dim, size in zip(cube['id'], cube['size']):
        key = selection[dim]
        offset = offset * size + codes(cube['dimension'][dim]).index(key)
    values = cube['value']
    value = values[offset] if isinstance(values, list) else values.get(str(offset))
    if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0):
        raise ValueError(f'Invalid source value: {value}')
    return value


def percentage(numerator, denominator):
    return None if numerator is None or not denominator else numerator / denominator * 100


def fmt(value):
    return 'Unavailable' if value is None else f'{value:,.0f}'


def pct(value):
    return 'Unavailable' if value is None else f'{value:.1f}%'


def panel(r):
    name = 'Ireland · State' if r['area'] == 'IE0' else r['name']
    walk, cycle = percentage(r['walking'], r['total']), percentage(r['cycling'], r['total'])
    return f'''<div class="compare-panel"><h3>{escape(name)}</h3><div class="share-number">{pct(r['share'])}<small>walk or cycle</small></div><div class="stack" aria-hidden="true"><span class="walk" style="width:{walk or 0}%"></span><span class="cycle" style="width:{cycle or 0}%"></span></div><div class="legend"><span><i class="walk"></i>Walk {pct(walk)}</span><span><i class="cycle"></i>Cycle {pct(cycle)}</span></div><p class="denominator">{fmt(r['active'])} of {fmt(r['total'])} people.<br>Not stated: {fmt(r['unknown'])} ({pct(percentage(r['unknown'], r['total']))}).</p></div>'''


def svg(content, width, height, title):
    return f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}" role="img"><title>{escape(title)}</title><rect width="{width}" height="{height}" fill="white"/><rect width="{width}" height="8" fill="#14b8a6"/><g font-family="Arial,sans-serif" fill="#111">{content}</g></svg>'


def render(result):
    records = result['records']
    local = sorted([r for r in records if r['group'] == 'work' and r['sex'] == '-'], key=lambda r: (r['area'] != 'IE0', r['name']))
    state = next(r for r in local if r['area'] == 'IE0')
    dublin = next(r for r in local if r['name'] == 'Dublin City')
    primary = next(r for r in records if r['area'] == 'IE0' and r['group'] == 'primary' and r['sex'] == '-')
    trend = [r for r in result['trends'] if r['group'] == 'work' and r['sex'] == '-']
    first, last = trend[0], trend[-1]
    findings = [f"{fmt(state['active'])} workers walked or cycled as their main travel mode in 2022: {pct(state['share'])} of {fmt(state['total'])} workers in the commuting population.",
                f"The number rose from {fmt(first['active'])} in 1986 to {fmt(last['active'])} in 2022, while the share fell from {pct(first['share'])} to {pct(last['share'])}.",
                f"Dublin City: {pct(dublin['share'])} walked or cycled to work ({fmt(dublin['active'])} of {fmt(dublin['total'])} people). This is Dublin City, not all Dublin.",
                f"Among schoolchildren aged 5–12, {pct(primary['share'])} walked or cycled ({fmt(primary['active'])} of {fmt(primary['total'])}). This is a different population from workers.",
                f"{fmt(state['unknown'])} workers had no stated travel mode ({pct(percentage(state['unknown'],state['total']))}). They remain in the denominator."]
    rows = ''.join(f'''<tr{' class="state"' if r['area']=='IE0' else ''}><th scope="row">{escape(r['name'])}</th><td class="bar-cell"><span class="bar-label">{pct(r['share'])}</span> <span class="bar-track" aria-hidden="true"><span style="width:{r['share'] or 0}%"></span></span></td><td>{fmt(r['walking'])}</td><td>{fmt(r['cycling'])}</td><td>{fmt(r['unknown'])}</td><td>{fmt(r['total'])}</td></tr>''' for r in local)
    options = lambda chosen: ''.join(f'<option value="{r["area"]}"{" selected" if r["area"] == chosen else ""}>{escape("Ireland · State" if r["area"] == "IE0" else r["name"])}</option>' for r in local)
    national = '<text x="60" y="60" font-size="20">NO NONSENSE FITNESS · EVERYDAY MOVEMENT IRELAND</text><text x="60" y="118" font-size="40" font-weight="700">More people. A smaller share.</text><text x="60" y="158" font-size="20">Walking + cycling to work · Republic of Ireland · Census 1986–2022</text>'
    for i, r in enumerate((first,last)):
        y = 230 + i*130
        national += f'<text x="60" y="{y}" font-size="28">{r["year"]}</text><rect x="170" y="{y-24}" width="650" height="30" fill="#e9eceb"/><rect x="170" y="{y-24}" width="{r["share"]*6.5}" height="30" fill="#14b8a6"/><text x="850" y="{y}" font-size="34" font-weight="700">{pct(r["share"])}</text><text x="170" y="{y+42}" font-size="20">{fmt(r["active"])} people / {fmt(r["total"])} total</text>'
    national += '<text x="60" y="488" font-size="18">Share fell as the commuting population grew. Bars use a 0–100% scale.</text><text x="60" y="522" font-size="17">Workers aged 15+, both sexes. Main travel mode; not stated included.</text><text x="60" y="556" font-size="17">Source: CSO F7122 · NNF calculations · CC BY 4.0 · Not a fitness measure.</text><text x="60" y="600" font-size="18">nononsensefitness.ie/everyday-movement/ · @nononsensefitness.ie</text>'
    national = svg(national,1200,630,'Walking and cycling to work: 1986 and 2022')
    (OUT / 'press-national.svg').write_text(national, encoding='utf-8')
    county_svg = '<text x="50" y="58" font-size="22">NO NONSENSE FITNESS · EVERYDAY MOVEMENT IRELAND</text><text x="50" y="112" font-size="35" font-weight="700">Walking + cycling to work, by area</text><text x="50" y="152" font-size="18">Census 2022 · Workers aged 15+ · Both sexes · Alphabetical order</text>'
    for i,r in enumerate(local):
        y=210+i*39
        county_svg += f'<text x="50" y="{y}" font-size="18">{escape(r["name"])}</text><rect x="420" y="{y-14}" width="400" height="16" fill="#e9eceb"/><rect x="420" y="{y-14}" width="{(r["share"] or 0)*4}" height="16" fill="#14b8a6"/><text x="900" y="{y}" font-size="19" text-anchor="end">{pct(r["share"])}</text><text x="1110" y="{y}" font-size="17" text-anchor="end">n = {fmt(r["total"])}</text>'
    county_svg += '<text x="50" y="1480" font-size="17">Source: CSO F7065 · NNF calculations · CC BY 4.0 · Bars 0–100%</text><text x="50" y="1514" font-size="17">Not stated included. Census areas differ from council boundaries. Not a fitness ranking.</text><text x="50" y="1555" font-size="18">nononsensefitness.ie/everyday-movement/ · @nononsensefitness.ie</text>'
    (OUT / 'press-counties.svg').write_text(svg(county_svg,1200,1600,'Census 2022 walking and cycling to work by area'),encoding='utf-8')
    context = ''
    for r in (dublin,state):
        times = [v for v in result['durations'] if v['area']==r['area'] and v['sex']=='-']
        now = times[-1]
        context += f'<div><h3>{escape(r["name"])} work journey</h3><p>Mean one-way work journey: <strong>{now["mean"]:.1f} minutes</strong> in 2022.</p><p>{" · ".join(v["year"]+": "+str(v["mean"])+" min" for v in times)}</p><p class="note">Workers only · CSO F7068 · {fmt(now["total"]-now["unknown"])} stated durations; {fmt(now["unknown"])} not stated.</p></div>'
    date = result['sources']['retrieved'][:10]
    values = {'DATE':date,'OPTIONS':options(dublin['area']),'COMPARE_OPTIONS':options('IE0'),
              'COMPARISON':panel(dublin)+panel(state),'CONTEXT':context,'ROWS':rows,
              'DIFFERENCE':f'Dublin City is {dublin["share"]-state["share"]:.1f} percentage points above the State for workers aged 15+.',
              'TREND':national,'TREND_ROWS':''.join(f'<tr><th scope="row">{r["year"]}</th><td>{pct(r["share"])}</td><td>{fmt(r["active"])}</td><td>{fmt(r["total"])}</td></tr>' for r in trend),
              'FINDINGS':'<ul>'+''.join('<li>'+escape(f)+'</li>' for f in findings)+'</ul>'}
    template = (ROOT/'qa'/'movement.template.html.in').read_text(encoding='utf-8')
    for key,value in values.items():
        template=template.replace('{{'+key+'}}',value)
    assert '{{' not in template, 'Unresolved template placeholder'
    (OUT/'index.html').write_text(template,encoding='utf-8')
    (OUT/'pipeline.py').write_text(Path(__file__).read_text(encoding='utf-8'),encoding='utf-8')
    print('Findings:')
    print('\n'.join(findings))


def derive(cubes):
    for cube in cubes.values():
        assert len(cube['id']) == len(cube['size']), 'Dimension/size mismatch'
        for dim, size in zip(cube['id'], cube['size']):
            keys = codes(cube['dimension'][dim])
            assert len(keys) == size == len(set(keys)), 'Invalid category index'
        if isinstance(cube['value'], list):
            assert len(cube['value']) == math.prod(cube['size']), 'Truncated source cube'
    local, times, history = (cubes[t] for t in TABLES)
    geo = 'C04104V04868'
    areas = local['dimension'][geo]['category']['label']
    assert len(areas) == 31 and 'IE0' in areas, 'Review changed geography'
    assert areas == times['dimension'][geo]['category']['label'], 'Geography mismatch'
    records, trends, durations = [], [], []
    modes = [x for x in codes(local['dimension']['C02734V03302']) if x != '-']
    for area_id, name in areas.items():
        for sex in ('-', '1', '2'):
            for group, (stat, hist_group, label) in GROUPS.items():
                base = {'STATISTIC': stat, 'TLIST(A1)': '2022', 'C02199V02655': sex, geo: area_id}
                counts = {m: cell(local, **base, C02734V03302=m) for m in [*modes, '-']}
                total = counts['-']
                if all(v is not None for v in counts.values()):
                    assert sum(counts[m] for m in modes) == total, (name, group, sex, 'mode total')
                active = None if counts['01'] is None or counts['02'] is None else counts['01'] + counts['02']
                for mode in ('01', '02'):
                    published = cell(local, **{**base, 'STATISTIC': stat[:5] + f'C{int(stat[-2:])+6:02d}'}, C02734V03302=mode)
                    calculated = percentage(counts[mode], total)
                    if published is not None and calculated is not None:
                        assert abs(published - calculated) <= .051, (name, group, 'published percentage')
                records.append(dict(area=area_id, name=name, group=group, sex=sex, total=total,
                                    walking=counts['01'], cycling=counts['02'], active=active,
                                    unknown=counts['98'], share=percentage(active, total)))
            for year in ('2011', '2016', '2022'):
                base = {'TLIST(A1)': year, 'C02199V02655': sex, geo: area_id}
                values = {str(i): cell(times, **base, STATISTIC=f'F7068C{i:02d}') for i in range(1, 10)}
                if all(values[str(i)] is not None for i in range(1, 9)):
                    assert sum(values[str(i)] for i in range(2, 9)) == values['1']
                durations.append(dict(area=area_id, sex=sex, year=year, total=values['1'],
                                      unknown=values['8'], mean=values['9']))
    for group, (stat, hist_group, label) in GROUPS.items():
        for sex in ('-', '1', '2'):
            for year in codes(history['dimension']['TLIST(A1)']):
                base = {'STATISTIC': 'F7122C01', 'TLIST(A1)': year, 'C02199V02655': sex,
                        'C02704V03272': hist_group}
                vals = {m: cell(history, **base, C02734V03302=m) for m in ('01', '02', '-')}
                active = None if vals['01'] is None or vals['02'] is None else vals['01'] + vals['02']
                trends.append(dict(group=group, sex=sex, year=year, total=vals['-'], active=active,
                                   share=percentage(active, vals['-'])))
            subset = [r for r in records if r['group'] == group and r['sex'] == sex]
            state = next(r for r in subset if r['area'] == 'IE0')
            for field in ('total', 'walking', 'cycling', 'unknown'):
                assert sum(r[field] for r in subset if r['area'] != 'IE0') == state[field], (group, field)
            trend = next(r for r in trends if r['group'] == group and r['sex'] == sex and r['year'] == '2022')
            assert (trend['total'], trend['active']) == (state['total'], state['active']), 'Cross-table mismatch'
    return dict(areas=areas, groups={k: v[2] for k, v in GROUPS.items()}, records=records,
                trends=trends, durations=durations)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--refresh', action='store_true', help='Download fresh source snapshots')
    args = parser.parse_args()
    raw, cubes = {}, {}
    for table in TABLES:
        path = OUT / 'sources' / f'{table}.json'
        if args.refresh:
            with urlopen(Request(API.format(table), headers={'User-Agent': 'NNF-PublicData/1.0'}), timeout=60) as response:
                raw[table] = response.read()
        else:
            raw[table] = path.read_bytes()
        cubes[table] = json.loads(raw[table])
    result = derive(cubes)  # Validate entire candidate before writing anything.
    stamp = datetime.now(timezone.utc).isoformat(timespec='seconds')
    metadata_path = OUT / 'sources' / 'manifest.json'
    if not args.refresh and metadata_path.exists():
        manifest = json.loads(metadata_path.read_text(encoding='utf-8'))
    else:
        manifest = {'retrieved': stamp, 'owner': 'Central Statistics Office, Ireland',
                    'licence': 'CC BY 4.0', 'licence_url': 'https://www.cso.ie/en/aboutus/whoweare/copyrightpolicy/',
                    'reuse': 'Public and commercial reuse permitted with attribution; NNF calculations are adaptations. No CSO endorsement.',
                    'frequency': 'Census release cycle; refresh after CSO revisions or a comparable census release.',
                    'datasets': [{'table': t, 'url': API.format(t), 'page': f'https://data.cso.ie/table/{t}',
                                  'published': cubes[t]['updated'], 'sha256': hashlib.sha256(raw[t]).hexdigest(),
                                  'geography': 'State' if t == 'F7122' else 'State and 30 county/city areas',
                                  'missing_cells': sum(v is None for v in cubes[t]['value']) if isinstance(cubes[t]['value'], list) else None}
                                 for t in TABLES]}
    result['sources'] = manifest
    (OUT / 'sources').mkdir(parents=True, exist_ok=True)
    for t in TABLES:
        (OUT / 'sources' / f'{t}.json').write_bytes(raw[t])
    metadata_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
    (OUT / 'data.json').write_text(json.dumps(result, ensure_ascii=False, separators=(',', ':')) + '\n', encoding='utf-8')
    stream = io.StringIO(newline='')
    writer = csv.DictWriter(stream, fieldnames=list(result['records'][0]), lineterminator='\n')
    writer.writeheader()
    writer.writerows(result['records'])
    (OUT / 'county-data.csv').write_text(stream.getvalue(), encoding='utf-8-sig', newline='\n')
    render(result)
    print(f'PASS: {len(result["records"])} local records; {len(result["trends"])} trend records; {len(result["durations"])} journey-time records. Totals and published percentages reconciled.')
    for r in result['records']:
        if r['area'] == 'IE0' and r['sex'] == '-':
            print(r)


if __name__ == '__main__':
    main()