Complete worked example
python
from hashlib import sha256
import json
import os
from pathlib import Path
def inspect_example(rows, fail):
root = Path('lake')
(root / 'old').mkdir(parents=True)
(root / 'old' / 'part.json').write_text(json.dumps([['IL','previous']]))
pointer = root / 'manifest.json'
pointer.write_text(json.dumps({'generation':'old','parts':['part.json']}))
groups = {}
for country, identity in rows:
if country not in ['IL','UA','US']:
raise ValueError('unsupported partition')
groups.setdefault(country, []).append([country,identity])
generation = 'g-' + sha256(json.dumps(sorted(rows)).encode()).hexdigest()[:12]
folder = root / generation
folder.mkdir()
manifest = {'generation':generation,'parts':[c+'.json' for c in sorted(groups)]}
temporary = root / 'manifest.tmp'
try:
for country in sorted(groups):
(folder / (country+'.json')).write_text(json.dumps(groups[country]))
if fail:
raise RuntimeError('controlled interruption during staging')
temporary.write_text(json.dumps(manifest))
os.replace(temporary, pointer)
except RuntimeError:
pass
active = json.loads(pointer.read_text())
visible = [row for part in active['parts'] for row in json.loads((root/active['generation']/part).read_text())]
return {'published': active['generation'] == generation, 'ids': sorted(r[1] for r in visible)}- Reasoning
- The program starts with an old published generation, derives a deterministic generation name from candidate content and writes country-partitioned JSON files beneath it. A controlled exception can interrupt the first partition. The active manifest is replaced only after the loop finishes successfully and a complete candidate manifest has been written.