1"""Download + preprocess the NYC TLC data and build the generated instance set.2 3 python make_data.py # uses cached raw files if present4 python make_data.py --force # re-download5 6Outputs: data/processed/* (gitignored), tests/fixtures/* (small, committed),7instances/*.json (generated orders, committed).8"""9import argparse10import json11import shutil12 13from delivery import config as C14from delivery import data as D15from delivery.instances import generate_instance16 17 18def disk_guard(min_free_mb=150):19 free = shutil.disk_usage(C.ROOT).free / 1e620 print(f" free disk: {free:,.0f} MB")21 if free < min_free_mb:22 raise SystemExit(f"stopping: less than {min_free_mb} MB free on the project drive")23 24 25def main():26 ap = argparse.ArgumentParser()27 ap.add_argument("--force", action="store_true")28 args = ap.parse_args()29 30 print("[1/5] download"); disk_guard()31 paths = D.download(force=args.force); disk_guard()32 33 print("[2/5] read + check schema")34 raw = D.load_raw_trips(paths["trips"])35 import pandas as pd36 lookup = pd.read_csv(paths["lookup"])37 cents = D.zone_centroids(paths["shapes"])38 print(f" {len(raw):,} raw rows, columns OK: {D.NEEDED_COLS}")39 40 print("[3/5] filter, top zones, sample, split by day")41 trips, zones, stats = D.preprocess(raw, lookup, cents)42 for k, v in stats.items():43 if not isinstance(v, list):44 print(f" {k:>22}: {v:,}")45 print(" zones:", ", ".join(zones.zone))46 47 print("[4/5] zone-pair mean distance (train only)")48 dist, cnt, fit = D.pair_distance_matrix(trips, zones)49 stats["pair_distance_fallback"] = fit50 tr = trips[trips.split == "train"]51 g = tr.groupby(["pu", "do", "hour"]).size()52 stats["rare_cells_lt5_train"] = int((g < 5).sum())53 stats["cells_possible"] = len(zones) ** 2 * 2454 stats["cells_seen_train"] = int(len(g))55 night = tr[tr.hour < 6].groupby(["pu", "do"]).size()56 stats["night_pairs_seen_train"] = int(len(night))57 D.save_processed(trips, zones, dist, cnt, stats)58 D.write_fixture(trips, zones, dist)59 print(f" unseen pairs filled by fit: {fit['unseen_pairs']}; "60 f"(pu,do,hour) cells seen {stats['cells_seen_train']}/{stats['cells_possible']}, "61 f"rare (<5 trips) {stats['rare_cells_lt5_train']}")62 63 print("[5/5] generated instance set")64 C.INSTANCES.mkdir(exist_ok=True)65 weights = zones.sort_values("idx").n_dropoffs.values66 manifest = []67 for n in (15, 20, 30):68 for seed in range(10):69 inst = generate_instance(n, seed, n_zones=len(zones), zone_weights=weights)70 p = C.INSTANCES / f"{inst.name}.json"71 inst.save(p); manifest.append(p.name)72 (C.INSTANCES / "manifest.json").write_text(json.dumps(manifest, indent=1))73 print(f" wrote {len(manifest)} instances to instances/")74 disk_guard()75 76 77if __name__ == "__main__":78 main()