"""Fetch all ComCat events with an 'oaf' product, every version (includesuperseded), and each version's forecast.json.
Writes raw/eventlist.json, raw/events/<id>.json, raw/fc/<id>_<updateTime>.json."""
import json, os, time, urllib.request
from concurrent.futures import ThreadPoolExecutor
B = 'https://earthquake.usgs.gov/fdsnws/event/1/query'
os.chdir('/work/aftershock-forecasts')
def get(url, tries=4):
    for i in range(tries):
        try:
            with urllib.request.urlopen(url, timeout=60) as r: return r.read()
        except Exception as e:
            time.sleep(2 * (i + 1)); err = e
    raise err
if not os.path.exists('raw/eventlist.json'):
    open('raw/eventlist.json', 'wb').write(get(B + '?producttype=oaf&starttime=2010-01-01&format=geojson&orderby=time-asc'))
ev = json.load(open('raw/eventlist.json'))['features']
print(len(ev), 'events')
def fetch_event(f):
    p = f'raw/events/{f["id"]}.json'
    if not os.path.exists(p):
        open(p, 'wb').write(get(B + f'?eventid={f["id"]}&format=geojson&includesuperseded=true'))
    d = json.load(open(p))
    out = []
    for pr in d['properties']['products'].get('oaf', []):
        if not isinstance(pr['contents'], dict): continue
        c = pr['contents'].get('forecast.json')
        if c: out.append((f'raw/fc/{f["id"]}_{pr["updateTime"]}_{pr["source"]}.json', c['url']))
    return out
with ThreadPoolExecutor(4) as ex: jobs = [j for r in ex.map(fetch_event, ev) for j in r]
print(len(jobs), 'forecast versions')
def fetch_fc(j):
    p, u = j
    if not os.path.exists(p): open(p, 'wb').write(get(u))
with ThreadPoolExecutor(4) as ex: list(ex.map(fetch_fc, jobs))
print('done')