rickub
anthodev/lapin-futepublic Fork
⑂ acf5bba
Branches
31/fix/traffic/accept-256-application-periods develop
Tags
v1.0.0 v1.0.1 v1.0.2 v1.0.3 v1.0.4 v1.1.0 v1.2.0 v1.2.1 v1.2.2 v1.2.3 v1.2.4 View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/anthodev/lapin-fute.git
git clone ssh://git@rickub.com/anthodev/lapin-fute.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

Download ZIP

Resolve IDFM service catalog with bounded importer (#11)Unverified

Add frozen catalog contracts (routes, transport modes, validators), a
node:sqlite-backed CatalogManager with accent-tolerant FTS search and
opaque service resolution, and a streaming IDFM importer that gates
every service on an exact realtime perimeter pair. Operator commands
cover refresh (atomic candidate activation, TOCTOU snapshot check),
measurement (>=1000 warm searches, p95 <= 200 ms), and the operator-only
five-call PRIM probe. Fixture-derived offline tests prove five-mode
coverage, stable IDs across revisions, removal as INVALID_SERVICE, and
rollback on failed replacement.
Cédric Anthony committed 2026-08-31T19:57:36+02:00 Browse files
acf5bba parent: ffd859a
modified .gitignore +1 -0
@@ -1,4 +1,5 @@
11 node_modules/
2+var/
23 packages/watch/build/
34 packages/watch/src/pkjs/
45 docs/
@@ -1,4 +1,5 @@
1 node_modules/1 node_modules/
2+var/
2 packages/watch/build/3 packages/watch/build/
3 packages/watch/src/pkjs/4 packages/watch/src/pkjs/
4 docs/5 docs/
added fixtures/catalog/idfm-v1.json +167 -0
new file mode 100644
@@ -0,0 +1,167 @@
1+{
2+ "schemaVersion": 1,
3+ "fixtureVersion": "idfm-catalog-v1",
4+ "recordedAt": "2026-08-30T00:00:00.000Z",
5+ "dataClassification": "Schema-faithful deterministic reduction of official source layouts; fixture-only identifiers, no credential and no personal departure response.",
6+ "attribution": [
7+ { "dataset": "offre-horaires-tc-gtfs-idfm", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/offre-horaires-tc-gtfs-idfm", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Mobilité" },
8+ { "dataset": "perimetre-des-donnees-tr-disponibles-plateforme-idfm", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/perimetre-des-donnees-tr-disponibles-plateforme-idfm", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Mobilité" },
9+ { "dataset": "arrets", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
10+ { "dataset": "zones-d-arrets", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/zones-d-arrets", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
11+ { "dataset": "relations", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/relations", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
12+ { "dataset": "referentiel-des-lignes", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/referentiel-des-lignes", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "ODbL 1.0" },
13+ { "dataset": "arrets-lignes", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets-lignes", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "ODbL 1.0" }
14+ ],
15+ "revisions": [
16+ { "id": "fixture-2026-08-a", "preservedTripId": "trip-bus-preserved", "removedTripId": "trip-metro-west" },
17+ { "id": "fixture-2026-08-b", "preservedTripId": "trip-bus-preserved", "removedTripId": "trip-metro-west" },
18+ { "id": "fixture-invalid-no-transilien", "invalidReason": "No exact TRANSILIEN perimeter pair; five-mode validation rejects the candidate" }
19+ ],
20+ "sources": {
21+ "agency": [
22+ { "agency_name": "Fixture IDFM operator", "agency_url": "https://www.iledefrance-mobilites.fr/", "agency_timezone": "Europe/Paris" }
23+ ],
24+ "lines": [
25+ { "id_line": "C100", "name_line": "Bus 42", "shortname_line": "42", "transportmode": "bus", "transportsubmode": "", "status": "active" },
26+ { "id_line": "C200", "name_line": "Métro 4", "shortname_line": "4", "transportmode": "metro", "transportsubmode": "", "status": "active" },
27+ { "id_line": "C300", "name_line": "Tram T3a", "shortname_line": "T3a", "transportmode": "tram", "transportsubmode": "", "status": "active" },
28+ { "id_line": "C400", "name_line": "RER A", "shortname_line": "A", "transportmode": "rail", "transportsubmode": "local", "status": "active" },
29+ { "id_line": "C500", "name_line": "Transilien L", "shortname_line": "L", "transportmode": "rail", "transportsubmode": "suburbanRailway", "status": "active" },
30+ { "id_line": "C600", "name_line": "Funiculaire", "shortname_line": "F", "transportmode": "funicular", "transportsubmode": "", "status": "active" }
31+ ],
32+ "routes": [
33+ { "route_id": "IDFM:C100", "route_short_name": "42", "route_long_name": "Bus 42" },
34+ { "route_id": "IDFM:C200", "route_short_name": "4", "route_long_name": "Métro 4" },
35+ { "route_id": "IDFM:C300", "route_short_name": "T3a", "route_long_name": "Tram T3a" },
36+ { "route_id": "IDFM:C400", "route_short_name": "A", "route_long_name": "RER A" },
37+ { "route_id": "IDFM:C500", "route_short_name": "L", "route_long_name": "Transilien L" },
38+ { "route_id": "IDFM:C600", "route_short_name": "F", "route_long_name": "Funiculaire" }
39+ ],
40+ "zones": [
41+ { "zdaid": "ZDA-BUS-1", "zdaname": "Gare du Nord", "zdatown": "Paris" },
42+ { "zdaid": "ZDA-BUS-2", "zdaname": "Gare du Nord", "zdatown": "Saint-Denis" },
43+ { "zdaid": "ZDA-METRO", "zdaname": "Châtelet", "zdatown": "Paris" },
44+ { "zdaid": "ZDA-TRAM", "zdaname": "Porte de Versailles", "zdatown": "Paris" },
45+ { "zdaid": "ZDA-RER", "zdaname": "Châtelet - Les Halles", "zdatown": "Paris" },
46+ { "zdaid": "ZDA-TRANSILIEN", "zdaname": "Saint-Lazare", "zdatown": "Paris" },
47+ { "zdaid": "ZDA-FUNICULAR", "zdaname": "Montmartre", "zdatown": "Paris" }
48+ ],
49+ "arrets": [
50+ { "arrid": "ARR-BUS-1", "zdaid": "ZDA-BUS-1" },
51+ { "arrid": "ARR-BUS-2", "zdaid": "ZDA-BUS-2" },
52+ { "arrid": "ARR-METRO-EAST", "zdaid": "ZDA-METRO" },
53+ { "arrid": "ARR-METRO-WEST", "zdaid": "ZDA-METRO" },
54+ { "arrid": "ARR-TRAM", "zdaid": "ZDA-TRAM" },
55+ { "arrid": "ARR-FUNICULAR", "zdaid": "ZDA-FUNICULAR" }
56+ ],
57+ "relations": [
58+ { "zdaid": "ZDA-BUS-1", "arrid": "ARR-BUS-1", "artid": "ART-BUS-1" },
59+ { "zdaid": "ZDA-BUS-2", "arrid": "ARR-BUS-2", "artid": "ART-BUS-2" },
60+ { "zdaid": "ZDA-METRO", "arrid": "ARR-METRO-EAST", "artid": "ART-METRO-EAST" },
61+ { "zdaid": "ZDA-METRO", "arrid": "ARR-METRO-WEST", "artid": "ART-METRO-WEST" },
62+ { "zdaid": "ZDA-TRAM", "arrid": "ARR-TRAM", "artid": "ART-TRAM" },
63+ { "zdaid": "ZDA-FUNICULAR", "arrid": "ARR-FUNICULAR", "artid": "ART-FUNICULAR" }
64+ ],
65+ "stops": [
66+ { "stop_id": "IDFM:ART-BUS-1", "stop_name": "Gare du Nord quai bus", "parent_station": "IDFM:ARR-BUS-1" },
67+ { "stop_id": "IDFM:TERM-BUS-1", "stop_name": "Hôpital Européen", "parent_station": "" },
68+ { "stop_id": "IDFM:ART-BUS-2", "stop_name": "Gare du Nord annexe", "parent_station": "IDFM:ARR-BUS-2" },
69+ { "stop_id": "IDFM:TERM-BUS-2", "stop_name": "Mairie", "parent_station": "" },
70+ { "stop_id": "IDFM:ART-BUS-MISMATCH", "stop_name": "Mismatch", "parent_station": "IDFM:ARR-BUS-1" },
71+ { "stop_id": "IDFM:TERM-BUS-MISMATCH", "stop_name": "Wrong perimeter destination", "parent_station": "" },
72+ { "stop_id": "IDFM:ART-METRO-EAST", "stop_name": "Châtelet sud", "parent_station": "IDFM:ARR-METRO-EAST" },
73+ { "stop_id": "IDFM:TERM-METRO-SOUTH", "stop_name": "Bagneux - Lucie Aubrac", "parent_station": "" },
74+ { "stop_id": "IDFM:ART-METRO-WEST", "stop_name": "Châtelet nord", "parent_station": "IDFM:ARR-METRO-WEST" },
75+ { "stop_id": "IDFM:TERM-METRO-NORTH", "stop_name": "Porte de Clignancourt", "parent_station": "" },
76+ { "stop_id": "IDFM:ART-TRAM", "stop_name": "Porte de Versailles quai tram", "parent_station": "IDFM:ARR-TRAM" },
77+ { "stop_id": "IDFM:TERM-TRAM", "stop_name": "Pont du Garigliano", "parent_station": "" },
78+ { "stop_id": "IDFM:ART-RER-EAST", "stop_name": "Châtelet RER est", "parent_station": "IDFM:monomodalStopPlace:ZDA-RER" },
79+ { "stop_id": "IDFM:TERM-RER-EAST", "stop_name": "Marne-la-Vallée Chessy", "parent_station": "" },
80+ { "stop_id": "IDFM:ART-RER-WEST", "stop_name": "Châtelet RER ouest", "parent_station": "IDFM:monomodalStopPlace:ZDA-RER" },
81+ { "stop_id": "IDFM:TERM-RER-WEST", "stop_name": "Saint-Germain-en-Laye", "parent_station": "" },
82+ { "stop_id": "IDFM:ART-TRANSILIEN", "stop_name": "Saint-Lazare quai L", "parent_station": "IDFM:monomodalStopPlace:ZDA-TRANSILIEN" },
83+ { "stop_id": "IDFM:TERM-TRANSILIEN", "stop_name": "Versailles Rive Droite", "parent_station": "" },
84+ { "stop_id": "IDFM:ART-FUNICULAR", "stop_name": "Funiculaire bas", "parent_station": "IDFM:ARR-FUNICULAR" },
85+ { "stop_id": "IDFM:TERM-FUNICULAR", "stop_name": "Funiculaire haut", "parent_station": "" }
86+ ],
87+ "trips": [
88+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-preserved", "trip_headsign": "Hôpital Européen", "direction_id": "0" },
89+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-homonym", "trip_headsign": "Mairie", "direction_id": "1" },
90+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-no-exact-perimeter", "trip_headsign": "Wrong perimeter destination", "direction_id": "0" },
91+ { "route_id": "IDFM:C200", "trip_id": "trip-metro-east", "trip_headsign": "Bagneux - Lucie Aubrac", "direction_id": "0" },
92+ { "route_id": "IDFM:C200", "trip_id": "trip-metro-west", "trip_headsign": "Porte de Clignancourt", "direction_id": "1", "revisions": ["fixture-2026-08-a"] },
93+ { "route_id": "IDFM:C300", "trip_id": "trip-tram", "trip_headsign": "Pont du Garigliano", "direction_id": "0" },
94+ { "route_id": "IDFM:C400", "trip_id": "trip-rer-east", "trip_headsign": "NORA", "direction_id": "0" },
95+ { "route_id": "IDFM:C400", "trip_id": "trip-rer-west", "trip_headsign": "ZEBU", "direction_id": "1" },
96+ { "route_id": "IDFM:C500", "trip_id": "trip-transilien", "trip_headsign": "MILA", "direction_id": "0" },
97+ { "route_id": "IDFM:C600", "trip_id": "trip-funicular", "trip_headsign": "Sommet", "direction_id": "0" }
98+ ],
99+ "stopTimes": [
100+ { "trip_id": "trip-bus-preserved", "stop_id": "IDFM:ART-BUS-1", "stop_sequence": "1", "pickup_type": "0" },
101+ { "trip_id": "trip-bus-preserved", "stop_id": "IDFM:TERM-BUS-1", "stop_sequence": "2", "pickup_type": "1" },
102+ { "trip_id": "trip-bus-homonym", "stop_id": "IDFM:ART-BUS-2", "stop_sequence": "1", "pickup_type": "0" },
103+ { "trip_id": "trip-bus-homonym", "stop_id": "IDFM:TERM-BUS-2", "stop_sequence": "2", "pickup_type": "1" },
104+ { "trip_id": "trip-bus-no-exact-perimeter", "stop_id": "IDFM:ART-BUS-MISMATCH", "stop_sequence": "1", "pickup_type": "0" },
105+ { "trip_id": "trip-bus-no-exact-perimeter", "stop_id": "IDFM:TERM-BUS-MISMATCH", "stop_sequence": "2", "pickup_type": "1" },
106+ { "trip_id": "trip-metro-east", "stop_id": "IDFM:ART-METRO-EAST", "stop_sequence": "1", "pickup_type": "0" },
107+ { "trip_id": "trip-metro-east", "stop_id": "IDFM:TERM-METRO-SOUTH", "stop_sequence": "2", "pickup_type": "1" },
108+ { "trip_id": "trip-metro-west", "stop_id": "IDFM:ART-METRO-WEST", "stop_sequence": "1", "pickup_type": "0", "revisions": ["fixture-2026-08-a"] },
109+ { "trip_id": "trip-metro-west", "stop_id": "IDFM:TERM-METRO-NORTH", "stop_sequence": "2", "pickup_type": "1", "revisions": ["fixture-2026-08-a"] },
110+ { "trip_id": "trip-tram", "stop_id": "IDFM:ART-TRAM", "stop_sequence": "1", "pickup_type": "0" },
111+ { "trip_id": "trip-tram", "stop_id": "IDFM:TERM-TRAM", "stop_sequence": "2", "pickup_type": "1" },
112+ { "trip_id": "trip-rer-east", "stop_id": "IDFM:ART-RER-EAST", "stop_sequence": "1", "pickup_type": "0" },
113+ { "trip_id": "trip-rer-east", "stop_id": "IDFM:TERM-RER-EAST", "stop_sequence": "2", "pickup_type": "1" },
114+ { "trip_id": "trip-rer-west", "stop_id": "IDFM:ART-RER-WEST", "stop_sequence": "1", "pickup_type": "0" },
115+ { "trip_id": "trip-rer-west", "stop_id": "IDFM:TERM-RER-WEST", "stop_sequence": "2", "pickup_type": "1" },
116+ { "trip_id": "trip-transilien", "stop_id": "IDFM:ART-TRANSILIEN", "stop_sequence": "1", "pickup_type": "0" },
117+ { "trip_id": "trip-transilien", "stop_id": "IDFM:TERM-TRANSILIEN", "stop_sequence": "2", "pickup_type": "1" },
118+ { "trip_id": "trip-funicular", "stop_id": "IDFM:ART-FUNICULAR", "stop_sequence": "1", "pickup_type": "0" },
119+ { "trip_id": "trip-funicular", "stop_id": "IDFM:TERM-FUNICULAR", "stop_sequence": "2", "pickup_type": "1" }
120+ ],
121+ "objectCodes": [
122+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-1", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-1:" },
123+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-1", "object_system": "source", "object_code": "DEST:BUS-1" },
124+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-2", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-2:" },
125+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-2", "object_system": "source", "object_code": "DEST:BUS-2" },
126+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-MISMATCH", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-MISSING:" },
127+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-MISMATCH", "object_system": "source", "object_code": "DEST:BUS-MISMATCH" },
128+ { "object_type": "StopPoint", "object_id": "IDFM:ART-METRO-EAST", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:METRO-EAST:" },
129+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-METRO-SOUTH", "object_system": "source", "object_code": "DEST:METRO-SOUTH" },
130+ { "object_type": "StopPoint", "object_id": "IDFM:ART-METRO-WEST", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:METRO-WEST:", "revisions": ["fixture-2026-08-a"] },
131+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-METRO-NORTH", "object_system": "source", "object_code": "DEST:METRO-NORTH", "revisions": ["fixture-2026-08-a"] },
132+ { "object_type": "StopPoint", "object_id": "IDFM:ART-TRAM", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:TRAM:" },
133+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-TRAM", "object_system": "source", "object_code": "DEST:TRAM" },
134+ { "object_type": "StopPoint", "object_id": "IDFM:ART-RER-EAST", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:RER-CHAT:" },
135+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-RER-EAST", "object_system": "source", "object_code": "DEST:RER-EAST" },
136+ { "object_type": "StopPoint", "object_id": "IDFM:ART-RER-WEST", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:RER-CHAT:" },
137+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-RER-WEST", "object_system": "source", "object_code": "DEST:RER-WEST" },
138+ { "object_type": "StopPoint", "object_id": "IDFM:ART-TRANSILIEN", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:TRANSILIEN-STL:" },
139+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-TRANSILIEN", "object_system": "source", "object_code": "DEST:TRANSILIEN" },
140+ { "object_type": "StopPoint", "object_id": "IDFM:ART-FUNICULAR", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:FUNICULAR:" },
141+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-FUNICULAR", "object_system": "source", "object_code": "DEST:FUNICULAR" }
142+ ],
143+ "arretsLignes": [
144+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-1" },
145+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-2" },
146+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-MISMATCH" },
147+ { "id": "IDFM:C200", "stop_id": "IDFM:ART-METRO-EAST" },
148+ { "id": "IDFM:C200", "stop_id": "IDFM:ART-METRO-WEST" },
149+ { "id": "IDFM:C300", "stop_id": "IDFM:ART-TRAM" },
150+ { "id": "IDFM:C400", "stop_id": "IDFM:ART-RER-EAST" },
151+ { "id": "IDFM:C400", "stop_id": "IDFM:ART-RER-WEST" },
152+ { "id": "IDFM:C500", "stop_id": "IDFM:ART-TRANSILIEN" },
153+ { "id": "IDFM:C600", "stop_id": "IDFM:ART-FUNICULAR" }
154+ ],
155+ "perimeter": [
156+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-1:", "ns2_stopname": "Gare du Nord", "operatorname": "Fixture bus" },
157+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-2:", "ns2_stopname": "Gare du Nord", "operatorname": "Fixture bus" },
158+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-OTHER:", "ns2_stopname": "Mismatch", "operatorname": "Fixture bus" },
159+ { "line": "IDFM:C200", "name_line": "4", "ns2_stoppointref": "STIF:StopPoint:Q:METRO-EAST:", "ns2_stopname": "Châtelet", "operatorname": "Fixture metro" },
160+ { "line": "IDFM:C200", "name_line": "4", "ns2_stoppointref": "STIF:StopPoint:Q:METRO-WEST:", "ns2_stopname": "Châtelet", "operatorname": "Fixture metro", "revisions": ["fixture-2026-08-a"] },
161+ { "line": "IDFM:C300", "name_line": "T3a", "ns2_stoppointref": "STIF:StopPoint:Q:TRAM:", "ns2_stopname": "Porte de Versailles", "operatorname": "Fixture tram" },
162+ { "line": "IDFM:C400", "name_line": "A", "ns2_stoppointref": "STIF:StopArea:SP:RER-CHAT:", "ns2_stopname": "Châtelet - Les Halles", "operatorname": "Fixture rail" },
163+ { "line": "IDFM:C500", "name_line": "L", "ns2_stoppointref": "STIF:StopArea:SP:TRANSILIEN-STL:", "ns2_stopname": "Saint-Lazare", "operatorname": "Fixture rail", "revisions": ["fixture-2026-08-a", "fixture-2026-08-b"] },
164+ { "line": "IDFM:C600", "name_line": "F", "ns2_stoppointref": "STIF:StopPoint:Q:FUNICULAR:", "ns2_stopname": "Montmartre", "operatorname": "Fixture funicular" }
165+ ]
166+ }
167+}
new file mode 100644
@@ -0,0 +1,167 @@
1+{
2+ "schemaVersion": 1,
3+ "fixtureVersion": "idfm-catalog-v1",
4+ "recordedAt": "2026-08-30T00:00:00.000Z",
5+ "dataClassification": "Schema-faithful deterministic reduction of official source layouts; fixture-only identifiers, no credential and no personal departure response.",
6+ "attribution": [
7+ { "dataset": "offre-horaires-tc-gtfs-idfm", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/offre-horaires-tc-gtfs-idfm", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Mobilité" },
8+ { "dataset": "perimetre-des-donnees-tr-disponibles-plateforme-idfm", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/perimetre-des-donnees-tr-disponibles-plateforme-idfm", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Mobilité" },
9+ { "dataset": "arrets", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
10+ { "dataset": "zones-d-arrets", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/zones-d-arrets", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
11+ { "dataset": "relations", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/relations", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "Licence Ouverte 2.0" },
12+ { "dataset": "referentiel-des-lignes", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/referentiel-des-lignes", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "ODbL 1.0" },
13+ { "dataset": "arrets-lignes", "url": "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets-lignes", "retrievedAt": "2026-08-30T00:00:00.000Z", "license": "ODbL 1.0" }
14+ ],
15+ "revisions": [
16+ { "id": "fixture-2026-08-a", "preservedTripId": "trip-bus-preserved", "removedTripId": "trip-metro-west" },
17+ { "id": "fixture-2026-08-b", "preservedTripId": "trip-bus-preserved", "removedTripId": "trip-metro-west" },
18+ { "id": "fixture-invalid-no-transilien", "invalidReason": "No exact TRANSILIEN perimeter pair; five-mode validation rejects the candidate" }
19+ ],
20+ "sources": {
21+ "agency": [
22+ { "agency_name": "Fixture IDFM operator", "agency_url": "https://www.iledefrance-mobilites.fr/", "agency_timezone": "Europe/Paris" }
23+ ],
24+ "lines": [
25+ { "id_line": "C100", "name_line": "Bus 42", "shortname_line": "42", "transportmode": "bus", "transportsubmode": "", "status": "active" },
26+ { "id_line": "C200", "name_line": "Métro 4", "shortname_line": "4", "transportmode": "metro", "transportsubmode": "", "status": "active" },
27+ { "id_line": "C300", "name_line": "Tram T3a", "shortname_line": "T3a", "transportmode": "tram", "transportsubmode": "", "status": "active" },
28+ { "id_line": "C400", "name_line": "RER A", "shortname_line": "A", "transportmode": "rail", "transportsubmode": "local", "status": "active" },
29+ { "id_line": "C500", "name_line": "Transilien L", "shortname_line": "L", "transportmode": "rail", "transportsubmode": "suburbanRailway", "status": "active" },
30+ { "id_line": "C600", "name_line": "Funiculaire", "shortname_line": "F", "transportmode": "funicular", "transportsubmode": "", "status": "active" }
31+ ],
32+ "routes": [
33+ { "route_id": "IDFM:C100", "route_short_name": "42", "route_long_name": "Bus 42" },
34+ { "route_id": "IDFM:C200", "route_short_name": "4", "route_long_name": "Métro 4" },
35+ { "route_id": "IDFM:C300", "route_short_name": "T3a", "route_long_name": "Tram T3a" },
36+ { "route_id": "IDFM:C400", "route_short_name": "A", "route_long_name": "RER A" },
37+ { "route_id": "IDFM:C500", "route_short_name": "L", "route_long_name": "Transilien L" },
38+ { "route_id": "IDFM:C600", "route_short_name": "F", "route_long_name": "Funiculaire" }
39+ ],
40+ "zones": [
41+ { "zdaid": "ZDA-BUS-1", "zdaname": "Gare du Nord", "zdatown": "Paris" },
42+ { "zdaid": "ZDA-BUS-2", "zdaname": "Gare du Nord", "zdatown": "Saint-Denis" },
43+ { "zdaid": "ZDA-METRO", "zdaname": "Châtelet", "zdatown": "Paris" },
44+ { "zdaid": "ZDA-TRAM", "zdaname": "Porte de Versailles", "zdatown": "Paris" },
45+ { "zdaid": "ZDA-RER", "zdaname": "Châtelet - Les Halles", "zdatown": "Paris" },
46+ { "zdaid": "ZDA-TRANSILIEN", "zdaname": "Saint-Lazare", "zdatown": "Paris" },
47+ { "zdaid": "ZDA-FUNICULAR", "zdaname": "Montmartre", "zdatown": "Paris" }
48+ ],
49+ "arrets": [
50+ { "arrid": "ARR-BUS-1", "zdaid": "ZDA-BUS-1" },
51+ { "arrid": "ARR-BUS-2", "zdaid": "ZDA-BUS-2" },
52+ { "arrid": "ARR-METRO-EAST", "zdaid": "ZDA-METRO" },
53+ { "arrid": "ARR-METRO-WEST", "zdaid": "ZDA-METRO" },
54+ { "arrid": "ARR-TRAM", "zdaid": "ZDA-TRAM" },
55+ { "arrid": "ARR-FUNICULAR", "zdaid": "ZDA-FUNICULAR" }
56+ ],
57+ "relations": [
58+ { "zdaid": "ZDA-BUS-1", "arrid": "ARR-BUS-1", "artid": "ART-BUS-1" },
59+ { "zdaid": "ZDA-BUS-2", "arrid": "ARR-BUS-2", "artid": "ART-BUS-2" },
60+ { "zdaid": "ZDA-METRO", "arrid": "ARR-METRO-EAST", "artid": "ART-METRO-EAST" },
61+ { "zdaid": "ZDA-METRO", "arrid": "ARR-METRO-WEST", "artid": "ART-METRO-WEST" },
62+ { "zdaid": "ZDA-TRAM", "arrid": "ARR-TRAM", "artid": "ART-TRAM" },
63+ { "zdaid": "ZDA-FUNICULAR", "arrid": "ARR-FUNICULAR", "artid": "ART-FUNICULAR" }
64+ ],
65+ "stops": [
66+ { "stop_id": "IDFM:ART-BUS-1", "stop_name": "Gare du Nord quai bus", "parent_station": "IDFM:ARR-BUS-1" },
67+ { "stop_id": "IDFM:TERM-BUS-1", "stop_name": "Hôpital Européen", "parent_station": "" },
68+ { "stop_id": "IDFM:ART-BUS-2", "stop_name": "Gare du Nord annexe", "parent_station": "IDFM:ARR-BUS-2" },
69+ { "stop_id": "IDFM:TERM-BUS-2", "stop_name": "Mairie", "parent_station": "" },
70+ { "stop_id": "IDFM:ART-BUS-MISMATCH", "stop_name": "Mismatch", "parent_station": "IDFM:ARR-BUS-1" },
71+ { "stop_id": "IDFM:TERM-BUS-MISMATCH", "stop_name": "Wrong perimeter destination", "parent_station": "" },
72+ { "stop_id": "IDFM:ART-METRO-EAST", "stop_name": "Châtelet sud", "parent_station": "IDFM:ARR-METRO-EAST" },
73+ { "stop_id": "IDFM:TERM-METRO-SOUTH", "stop_name": "Bagneux - Lucie Aubrac", "parent_station": "" },
74+ { "stop_id": "IDFM:ART-METRO-WEST", "stop_name": "Châtelet nord", "parent_station": "IDFM:ARR-METRO-WEST" },
75+ { "stop_id": "IDFM:TERM-METRO-NORTH", "stop_name": "Porte de Clignancourt", "parent_station": "" },
76+ { "stop_id": "IDFM:ART-TRAM", "stop_name": "Porte de Versailles quai tram", "parent_station": "IDFM:ARR-TRAM" },
77+ { "stop_id": "IDFM:TERM-TRAM", "stop_name": "Pont du Garigliano", "parent_station": "" },
78+ { "stop_id": "IDFM:ART-RER-EAST", "stop_name": "Châtelet RER est", "parent_station": "IDFM:monomodalStopPlace:ZDA-RER" },
79+ { "stop_id": "IDFM:TERM-RER-EAST", "stop_name": "Marne-la-Vallée Chessy", "parent_station": "" },
80+ { "stop_id": "IDFM:ART-RER-WEST", "stop_name": "Châtelet RER ouest", "parent_station": "IDFM:monomodalStopPlace:ZDA-RER" },
81+ { "stop_id": "IDFM:TERM-RER-WEST", "stop_name": "Saint-Germain-en-Laye", "parent_station": "" },
82+ { "stop_id": "IDFM:ART-TRANSILIEN", "stop_name": "Saint-Lazare quai L", "parent_station": "IDFM:monomodalStopPlace:ZDA-TRANSILIEN" },
83+ { "stop_id": "IDFM:TERM-TRANSILIEN", "stop_name": "Versailles Rive Droite", "parent_station": "" },
84+ { "stop_id": "IDFM:ART-FUNICULAR", "stop_name": "Funiculaire bas", "parent_station": "IDFM:ARR-FUNICULAR" },
85+ { "stop_id": "IDFM:TERM-FUNICULAR", "stop_name": "Funiculaire haut", "parent_station": "" }
86+ ],
87+ "trips": [
88+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-preserved", "trip_headsign": "Hôpital Européen", "direction_id": "0" },
89+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-homonym", "trip_headsign": "Mairie", "direction_id": "1" },
90+ { "route_id": "IDFM:C100", "trip_id": "trip-bus-no-exact-perimeter", "trip_headsign": "Wrong perimeter destination", "direction_id": "0" },
91+ { "route_id": "IDFM:C200", "trip_id": "trip-metro-east", "trip_headsign": "Bagneux - Lucie Aubrac", "direction_id": "0" },
92+ { "route_id": "IDFM:C200", "trip_id": "trip-metro-west", "trip_headsign": "Porte de Clignancourt", "direction_id": "1", "revisions": ["fixture-2026-08-a"] },
93+ { "route_id": "IDFM:C300", "trip_id": "trip-tram", "trip_headsign": "Pont du Garigliano", "direction_id": "0" },
94+ { "route_id": "IDFM:C400", "trip_id": "trip-rer-east", "trip_headsign": "NORA", "direction_id": "0" },
95+ { "route_id": "IDFM:C400", "trip_id": "trip-rer-west", "trip_headsign": "ZEBU", "direction_id": "1" },
96+ { "route_id": "IDFM:C500", "trip_id": "trip-transilien", "trip_headsign": "MILA", "direction_id": "0" },
97+ { "route_id": "IDFM:C600", "trip_id": "trip-funicular", "trip_headsign": "Sommet", "direction_id": "0" }
98+ ],
99+ "stopTimes": [
100+ { "trip_id": "trip-bus-preserved", "stop_id": "IDFM:ART-BUS-1", "stop_sequence": "1", "pickup_type": "0" },
101+ { "trip_id": "trip-bus-preserved", "stop_id": "IDFM:TERM-BUS-1", "stop_sequence": "2", "pickup_type": "1" },
102+ { "trip_id": "trip-bus-homonym", "stop_id": "IDFM:ART-BUS-2", "stop_sequence": "1", "pickup_type": "0" },
103+ { "trip_id": "trip-bus-homonym", "stop_id": "IDFM:TERM-BUS-2", "stop_sequence": "2", "pickup_type": "1" },
104+ { "trip_id": "trip-bus-no-exact-perimeter", "stop_id": "IDFM:ART-BUS-MISMATCH", "stop_sequence": "1", "pickup_type": "0" },
105+ { "trip_id": "trip-bus-no-exact-perimeter", "stop_id": "IDFM:TERM-BUS-MISMATCH", "stop_sequence": "2", "pickup_type": "1" },
106+ { "trip_id": "trip-metro-east", "stop_id": "IDFM:ART-METRO-EAST", "stop_sequence": "1", "pickup_type": "0" },
107+ { "trip_id": "trip-metro-east", "stop_id": "IDFM:TERM-METRO-SOUTH", "stop_sequence": "2", "pickup_type": "1" },
108+ { "trip_id": "trip-metro-west", "stop_id": "IDFM:ART-METRO-WEST", "stop_sequence": "1", "pickup_type": "0", "revisions": ["fixture-2026-08-a"] },
109+ { "trip_id": "trip-metro-west", "stop_id": "IDFM:TERM-METRO-NORTH", "stop_sequence": "2", "pickup_type": "1", "revisions": ["fixture-2026-08-a"] },
110+ { "trip_id": "trip-tram", "stop_id": "IDFM:ART-TRAM", "stop_sequence": "1", "pickup_type": "0" },
111+ { "trip_id": "trip-tram", "stop_id": "IDFM:TERM-TRAM", "stop_sequence": "2", "pickup_type": "1" },
112+ { "trip_id": "trip-rer-east", "stop_id": "IDFM:ART-RER-EAST", "stop_sequence": "1", "pickup_type": "0" },
113+ { "trip_id": "trip-rer-east", "stop_id": "IDFM:TERM-RER-EAST", "stop_sequence": "2", "pickup_type": "1" },
114+ { "trip_id": "trip-rer-west", "stop_id": "IDFM:ART-RER-WEST", "stop_sequence": "1", "pickup_type": "0" },
115+ { "trip_id": "trip-rer-west", "stop_id": "IDFM:TERM-RER-WEST", "stop_sequence": "2", "pickup_type": "1" },
116+ { "trip_id": "trip-transilien", "stop_id": "IDFM:ART-TRANSILIEN", "stop_sequence": "1", "pickup_type": "0" },
117+ { "trip_id": "trip-transilien", "stop_id": "IDFM:TERM-TRANSILIEN", "stop_sequence": "2", "pickup_type": "1" },
118+ { "trip_id": "trip-funicular", "stop_id": "IDFM:ART-FUNICULAR", "stop_sequence": "1", "pickup_type": "0" },
119+ { "trip_id": "trip-funicular", "stop_id": "IDFM:TERM-FUNICULAR", "stop_sequence": "2", "pickup_type": "1" }
120+ ],
121+ "objectCodes": [
122+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-1", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-1:" },
123+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-1", "object_system": "source", "object_code": "DEST:BUS-1" },
124+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-2", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-2:" },
125+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-2", "object_system": "source", "object_code": "DEST:BUS-2" },
126+ { "object_type": "StopPoint", "object_id": "IDFM:ART-BUS-MISMATCH", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:BUS-MISSING:" },
127+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-BUS-MISMATCH", "object_system": "source", "object_code": "DEST:BUS-MISMATCH" },
128+ { "object_type": "StopPoint", "object_id": "IDFM:ART-METRO-EAST", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:METRO-EAST:" },
129+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-METRO-SOUTH", "object_system": "source", "object_code": "DEST:METRO-SOUTH" },
130+ { "object_type": "StopPoint", "object_id": "IDFM:ART-METRO-WEST", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:METRO-WEST:", "revisions": ["fixture-2026-08-a"] },
131+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-METRO-NORTH", "object_system": "source", "object_code": "DEST:METRO-NORTH", "revisions": ["fixture-2026-08-a"] },
132+ { "object_type": "StopPoint", "object_id": "IDFM:ART-TRAM", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:TRAM:" },
133+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-TRAM", "object_system": "source", "object_code": "DEST:TRAM" },
134+ { "object_type": "StopPoint", "object_id": "IDFM:ART-RER-EAST", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:RER-CHAT:" },
135+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-RER-EAST", "object_system": "source", "object_code": "DEST:RER-EAST" },
136+ { "object_type": "StopPoint", "object_id": "IDFM:ART-RER-WEST", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:RER-CHAT:" },
137+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-RER-WEST", "object_system": "source", "object_code": "DEST:RER-WEST" },
138+ { "object_type": "StopPoint", "object_id": "IDFM:ART-TRANSILIEN", "object_system": "SIRI", "object_code": "STIF:StopArea:SP:TRANSILIEN-STL:" },
139+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-TRANSILIEN", "object_system": "source", "object_code": "DEST:TRANSILIEN" },
140+ { "object_type": "StopPoint", "object_id": "IDFM:ART-FUNICULAR", "object_system": "SIRI", "object_code": "STIF:StopPoint:Q:FUNICULAR:" },
141+ { "object_type": "StopPoint", "object_id": "IDFM:TERM-FUNICULAR", "object_system": "source", "object_code": "DEST:FUNICULAR" }
142+ ],
143+ "arretsLignes": [
144+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-1" },
145+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-2" },
146+ { "id": "IDFM:C100", "stop_id": "IDFM:ART-BUS-MISMATCH" },
147+ { "id": "IDFM:C200", "stop_id": "IDFM:ART-METRO-EAST" },
148+ { "id": "IDFM:C200", "stop_id": "IDFM:ART-METRO-WEST" },
149+ { "id": "IDFM:C300", "stop_id": "IDFM:ART-TRAM" },
150+ { "id": "IDFM:C400", "stop_id": "IDFM:ART-RER-EAST" },
151+ { "id": "IDFM:C400", "stop_id": "IDFM:ART-RER-WEST" },
152+ { "id": "IDFM:C500", "stop_id": "IDFM:ART-TRANSILIEN" },
153+ { "id": "IDFM:C600", "stop_id": "IDFM:ART-FUNICULAR" }
154+ ],
155+ "perimeter": [
156+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-1:", "ns2_stopname": "Gare du Nord", "operatorname": "Fixture bus" },
157+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-2:", "ns2_stopname": "Gare du Nord", "operatorname": "Fixture bus" },
158+ { "line": "IDFM:C100", "name_line": "42", "ns2_stoppointref": "STIF:StopPoint:Q:BUS-OTHER:", "ns2_stopname": "Mismatch", "operatorname": "Fixture bus" },
159+ { "line": "IDFM:C200", "name_line": "4", "ns2_stoppointref": "STIF:StopPoint:Q:METRO-EAST:", "ns2_stopname": "Châtelet", "operatorname": "Fixture metro" },
160+ { "line": "IDFM:C200", "name_line": "4", "ns2_stoppointref": "STIF:StopPoint:Q:METRO-WEST:", "ns2_stopname": "Châtelet", "operatorname": "Fixture metro", "revisions": ["fixture-2026-08-a"] },
161+ { "line": "IDFM:C300", "name_line": "T3a", "ns2_stoppointref": "STIF:StopPoint:Q:TRAM:", "ns2_stopname": "Porte de Versailles", "operatorname": "Fixture tram" },
162+ { "line": "IDFM:C400", "name_line": "A", "ns2_stoppointref": "STIF:StopArea:SP:RER-CHAT:", "ns2_stopname": "Châtelet - Les Halles", "operatorname": "Fixture rail" },
163+ { "line": "IDFM:C500", "name_line": "L", "ns2_stoppointref": "STIF:StopArea:SP:TRANSILIEN-STL:", "ns2_stopname": "Saint-Lazare", "operatorname": "Fixture rail", "revisions": ["fixture-2026-08-a", "fixture-2026-08-b"] },
164+ { "line": "IDFM:C600", "name_line": "F", "ns2_stoppointref": "STIF:StopPoint:Q:FUNICULAR:", "ns2_stopname": "Montmartre", "operatorname": "Fixture funicular" }
165+ ]
166+ }
167+}
added package-lock.json +78 -0
new file mode 100644
@@ -0,0 +1,78 @@
1+{
2+ "name": "lapin-fute",
3+ "version": "0.1.0",
4+ "lockfileVersion": 3,
5+ "requires": true,
6+ "packages": {
7+ "": {
8+ "name": "lapin-fute",
9+ "version": "0.1.0",
10+ "workspaces": [
11+ "packages/*"
12+ ],
13+ "engines": {
14+ "node": "24.18.0"
15+ }
16+ },
17+ "node_modules/@lapin-fute/backend": {
18+ "resolved": "packages/backend",
19+ "link": true
20+ },
21+ "node_modules/@lapin-fute/companion": {
22+ "resolved": "packages/companion",
23+ "link": true
24+ },
25+ "node_modules/@lapin-fute/config-page": {
26+ "resolved": "packages/config-page",
27+ "link": true
28+ },
29+ "node_modules/csv-parse": {
30+ "version": "7.0.2",
31+ "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz",
32+ "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==",
33+ "license": "MIT"
34+ },
35+ "node_modules/lapin-fute-watch": {
36+ "resolved": "packages/watch",
37+ "link": true
38+ },
39+ "node_modules/pend": {
40+ "version": "1.2.0",
41+ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
42+ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
43+ "license": "MIT"
44+ },
45+ "node_modules/yauzl": {
46+ "version": "3.4.0",
47+ "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz",
48+ "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==",
49+ "license": "MIT",
50+ "dependencies": {
51+ "pend": "~1.2.0"
52+ },
53+ "engines": {
54+ "node": ">=12"
55+ }
56+ },
57+ "packages/backend": {
58+ "name": "@lapin-fute/backend",
59+ "version": "0.1.0",
60+ "dependencies": {
61+ "csv-parse": "7.0.2",
62+ "yauzl": "3.4.0"
63+ }
64+ },
65+ "packages/companion": {
66+ "name": "@lapin-fute/companion",
67+ "version": "0.1.0"
68+ },
69+ "packages/config-page": {
70+ "name": "@lapin-fute/config-page",
71+ "version": "0.1.0"
72+ },
73+ "packages/watch": {
74+ "name": "lapin-fute-watch",
75+ "version": "0.1.0"
76+ }
77+ }
78+}
new file mode 100644
@@ -0,0 +1,78 @@
1+{
2+ "name": "lapin-fute",
3+ "version": "0.1.0",
4+ "lockfileVersion": 3,
5+ "requires": true,
6+ "packages": {
7+ "": {
8+ "name": "lapin-fute",
9+ "version": "0.1.0",
10+ "workspaces": [
11+ "packages/*"
12+ ],
13+ "engines": {
14+ "node": "24.18.0"
15+ }
16+ },
17+ "node_modules/@lapin-fute/backend": {
18+ "resolved": "packages/backend",
19+ "link": true
20+ },
21+ "node_modules/@lapin-fute/companion": {
22+ "resolved": "packages/companion",
23+ "link": true
24+ },
25+ "node_modules/@lapin-fute/config-page": {
26+ "resolved": "packages/config-page",
27+ "link": true
28+ },
29+ "node_modules/csv-parse": {
30+ "version": "7.0.2",
31+ "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz",
32+ "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==",
33+ "license": "MIT"
34+ },
35+ "node_modules/lapin-fute-watch": {
36+ "resolved": "packages/watch",
37+ "link": true
38+ },
39+ "node_modules/pend": {
40+ "version": "1.2.0",
41+ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
42+ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
43+ "license": "MIT"
44+ },
45+ "node_modules/yauzl": {
46+ "version": "3.4.0",
47+ "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.4.0.tgz",
48+ "integrity": "sha512-jIH9yLR9wqr0wOS0TpBvo/g/2UgZH5qePVbjgRliiF0BYvOZyaBknKsF+x9Iht0O6sqgnB93rCICdOZFecJuDw==",
49+ "license": "MIT",
50+ "dependencies": {
51+ "pend": "~1.2.0"
52+ },
53+ "engines": {
54+ "node": ">=12"
55+ }
56+ },
57+ "packages/backend": {
58+ "name": "@lapin-fute/backend",
59+ "version": "0.1.0",
60+ "dependencies": {
61+ "csv-parse": "7.0.2",
62+ "yauzl": "3.4.0"
63+ }
64+ },
65+ "packages/companion": {
66+ "name": "@lapin-fute/companion",
67+ "version": "0.1.0"
68+ },
69+ "packages/config-page": {
70+ "name": "@lapin-fute/config-page",
71+ "version": "0.1.0"
72+ },
73+ "packages/watch": {
74+ "name": "lapin-fute-watch",
75+ "version": "0.1.0"
76+ }
77+ }
78+}
modified package.json +3 -0
@@ -11,7 +11,10 @@
1111 },
1212 "scripts": {
1313 "build": "node scripts/build.mjs",
14+ "catalog:probe": "node scripts/probe-catalog.mjs",
15+ "catalog:refresh": "node scripts/refresh-catalog.mjs",
1416 "measure:radio": "node scripts/measure-radio.mjs",
17+ "measure:catalog": "node scripts/measure-catalog.mjs",
1518 "test": "node scripts/test.mjs",
1619 "test:contracts": "node --test packages/contracts/test/contracts.test.ts",
1720 "test:watch": "node --test packages/watch/test/*.test.js",
@@ -11,7 +11,10 @@
11 },11 },
12 "scripts": {12 "scripts": {
13 "build": "node scripts/build.mjs",13 "build": "node scripts/build.mjs",
14+ "catalog:probe": "node scripts/probe-catalog.mjs",
15+ "catalog:refresh": "node scripts/refresh-catalog.mjs",
14 "measure:radio": "node scripts/measure-radio.mjs",16 "measure:radio": "node scripts/measure-radio.mjs",
17+ "measure:catalog": "node scripts/measure-catalog.mjs",
15 "test": "node scripts/test.mjs",18 "test": "node scripts/test.mjs",
16 "test:contracts": "node --test packages/contracts/test/contracts.test.ts",19 "test:contracts": "node --test packages/contracts/test/contracts.test.ts",
17 "test:watch": "node --test packages/watch/test/*.test.js",20 "test:watch": "node --test packages/watch/test/*.test.js",
modified packages/backend/package.json +5 -1
@@ -4,7 +4,11 @@
44 "private": true,
55 "type": "module",
66 "exports": "./src/index.ts",
7+ "dependencies": {
8+ "csv-parse": "7.0.2",
9+ "yauzl": "3.4.0"
10+ },
711 "scripts": {
8- "test": "node --test test/backend.test.ts"
12+ "test": "node --test test/*.test.ts"
913 }
1014 }
@@ -4,7 +4,11 @@
4 "private": true,4 "private": true,
5 "type": "module",5 "type": "module",
6 "exports": "./src/index.ts",6 "exports": "./src/index.ts",
7+ "dependencies": {
8+ "csv-parse": "7.0.2",
9+ "yauzl": "3.4.0"
10+ },
7 "scripts": {11 "scripts": {
8- "test": "node --test test/backend.test.ts"12+ "test": "node --test test/*.test.ts"
9 }13 }
10 }14 }
added packages/backend/src/catalog-import.ts +1462 -0
new file mode 100644
@@ -0,0 +1,1462 @@
1+import { createHash } from "node:crypto";
2+import {
3+ closeSync,
4+ createReadStream,
5+ existsSync,
6+ fsyncSync,
7+ mkdirSync,
8+ openSync,
9+ renameSync,
10+ statSync,
11+ unlinkSync,
12+} from "node:fs";
13+import { open as openFile } from "node:fs/promises";
14+import type { FileHandle } from "node:fs/promises";
15+import { basename, dirname, resolve } from "node:path";
16+import { Readable, Transform } from "node:stream";
17+import { DatabaseSync } from "node:sqlite";
18+import { parse } from "csv-parse";
19+import yauzl from "yauzl";
20+import {
21+ LIMITS,
22+ SCHEMA_VERSION,
23+ TRANSPORT_MODE,
24+} from "../../contracts/src/index.ts";
25+import type { TransportMode } from "../../contracts/src/index.ts";
26+
27+export const CATALOG_VERSION = 1 as const;
28+export const DOWNLOAD_TIMEOUT_MS = 15 * 60 * 1_000;
29+export const MAX_COMPRESSED_DOWNLOAD_BYTES = 2 * 1024 ** 3;
30+export const MAX_SELECTED_ENTRY_BYTES = 8 * 1024 ** 3;
31+export const MAX_SELECTED_ARCHIVE_BYTES = 16 * 1024 ** 3;
32+export const MAX_CSV_RECORD_BYTES = 1024 * 1024;
33+export const IMPORT_TRANSACTION_ROWS = 10_000;
34+export const IDFM_CATALOG_SOURCE_URL = {
35+ gtfsRecord: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/offre-horaires-tc-gtfs-idfm/records?limit=1",
36+ perimeter: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/perimetre-des-donnees-tr-disponibles-plateforme-idfm/exports/csv?limit=-1",
37+ arrets: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets/exports/csv?limit=-1",
38+ zones: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/zones-d-arrets/exports/csv?limit=-1",
39+ relations: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/relations/exports/csv?limit=-1",
40+ lines: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/referentiel-des-lignes/exports/csv?limit=-1",
41+ arretsLignes: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets-lignes/exports/csv?limit=-1",
42+} as const;
43+
44+const SUPPORTED_MODE: Readonly<Record<TransportMode, true>> = {
45+ BUS: true,
46+ METRO: true,
47+ TRAM: true,
48+ RER: true,
49+ TRANSILIEN: true,
50+};
51+const REQUIRED_GTFS_ENTRIES = [
52+ "routes.txt",
53+ "trips.txt",
54+ "stop_times.txt",
55+ "stops.txt",
56+ "agency.txt",
57+] as const;
58+const CODE_ENTRY_NAME: Readonly<Record<string, true>> = {
59+ "object_codes_extension.txt": true,
60+ "objects_codes_extension.txt": true,
61+};
62+
63+export class CatalogImportError extends Error {
64+ readonly code: string;
65+
66+ constructor(code: string, message: string, options?: ErrorOptions) {
67+ super(message, options);
68+ this.name = "CatalogImportError";
69+ this.code = code;
70+ }
71+}
72+
73+export type CsvValue = string | number | boolean | null | undefined;
74+export type CsvRecord = Readonly<Record<string, CsvValue>>;
75+
76+export interface CatalogRecordSourceSet {
77+ readonly routes: readonly CsvRecord[];
78+ readonly trips: readonly CsvRecord[];
79+ readonly stopTimes: readonly CsvRecord[];
80+ readonly stops: readonly CsvRecord[];
81+ readonly agency: readonly CsvRecord[];
82+ readonly objectCodes: readonly CsvRecord[];
83+ readonly perimeter: readonly CsvRecord[];
84+ readonly arrets: readonly CsvRecord[];
85+ readonly zones: readonly CsvRecord[];
86+ readonly relations: readonly CsvRecord[];
87+ readonly lines: readonly CsvRecord[];
88+ readonly arretsLignes: readonly CsvRecord[];
89+}
90+
91+export interface CsvFileSource {
92+ readonly path: string;
93+ readonly delimiter?: "," | ";";
94+ readonly expectedRows?: number;
95+}
96+
97+export interface CatalogFileSourceSet {
98+ readonly gtfsZipPath: string;
99+ readonly perimeter: CsvFileSource;
100+ readonly arrets: CsvFileSource;
101+ readonly zones: CsvFileSource;
102+ readonly relations: CsvFileSource;
103+ readonly lines: CsvFileSource;
104+ readonly arretsLignes: CsvFileSource;
105+}
106+
107+export interface CatalogSourceAttribution {
108+ readonly dataset: string;
109+ readonly url: string;
110+ readonly retrievedAt: string;
111+ readonly license: string;
112+ readonly restricted?: boolean;
113+}
114+
115+export interface DownloadCatalogSourceOptions {
116+ readonly source: CatalogSourceAttribution;
117+ readonly destinationPath: string;
118+ readonly expectedContentTypes: readonly string[];
119+ readonly datasetToken?: string;
120+ readonly fetch?: typeof globalThis.fetch;
121+}
122+
123+export interface DownloadedCatalogSource {
124+ readonly path: string;
125+ readonly bytes: number;
126+ readonly sha256: string;
127+ readonly source: CatalogSourceAttribution;
128+}
129+export interface ResolvedGtfsDownload {
130+ readonly filename: string;
131+ readonly url: string;
132+}
133+
134+export interface BuildCatalogOptions {
135+ readonly candidatePath: string;
136+ readonly sourceRevision: string;
137+ readonly createdAt?: string;
138+ readonly sources: CatalogFileSourceSet;
139+}
140+
141+export interface BuildCatalogFromRecordsOptions {
142+ readonly candidatePath: string;
143+ readonly sourceRevision: string;
144+ readonly createdAt?: string;
145+ readonly sources: CatalogRecordSourceSet;
146+}
147+
148+export interface PlaceResolutionCounts {
149+ readonly arrets: number;
150+ readonly relations: number;
151+ readonly direct: number;
152+}
153+
154+export interface CatalogBuildResult {
155+ readonly candidatePath: string;
156+ readonly sourceRevision: string;
157+ readonly createdAt: string;
158+ readonly placeCount: number;
159+ readonly serviceCount: number;
160+ readonly countsByMode: Readonly<Record<TransportMode, number>>;
161+ readonly placeResolution?: PlaceResolutionCounts;
162+ readonly revalidatedAfterRenameFailure?: true;
163+}
164+
165+interface CsvSpec {
166+ readonly required: readonly string[];
167+ readonly insert: (row: Readonly<Record<string, string>>) => void;
168+}
169+
170+interface ZipEntryLike {
171+ readonly fileName: string;
172+ readonly uncompressedSize: number;
173+ readonly generalPurposeBitFlag: number;
174+}
175+
176+interface ZipFileLike {
177+ readEntry(): void;
178+ close(): void;
179+ once(event: "entry", listener: (entry: ZipEntryLike) => void): this;
180+ once(event: "end" | "error", listener: (...args: unknown[]) => void): this;
181+ removeListener(event: string, listener: (...args: never[]) => void): this;
182+ openReadStream(
183+ entry: ZipEntryLike,
184+ callback: (error: Error | null, stream?: Readable) => void,
185+ ): void;
186+}
187+
188+interface QueryRow {
189+ readonly [key: string]: unknown;
190+}
191+
192+function fail(code: string, message: string, cause?: unknown): never {
193+ throw new CatalogImportError(code, message, cause === undefined ? undefined : { cause });
194+}
195+
196+function nonEmpty(value: unknown, field: string): string {
197+ const text = String(value ?? "").trim();
198+ if (text.length === 0 || text.includes("\0")) fail("INVALID_SOURCE", `${field} must be non-empty`);
199+ return text;
200+}
201+
202+function optionalText(value: unknown): string {
203+ const text = String(value ?? "").trim();
204+ if (text.includes("\0")) fail("INVALID_SOURCE", "source text contains NUL");
205+ return text;
206+}
207+
208+function integer(value: unknown, field: string): number {
209+ const text = nonEmpty(value, field);
210+ if (!/^-?\d+$/u.test(text)) fail("INVALID_SOURCE", `${field} must be an integer`);
211+ const parsed = Number(text);
212+ if (!Number.isSafeInteger(parsed)) fail("INVALID_SOURCE", `${field} exceeds safe integer range`);
213+ return parsed;
214+}
215+
216+function transportMode(value: unknown): TransportMode {
217+ const mode = nonEmpty(value, "mode");
218+ if (!Object.hasOwn(SUPPORTED_MODE, mode)) fail("INVALID_MODE", `unsupported transport mode ${mode}`);
219+ return mode as TransportMode;
220+}
221+
222+function canonicalLineId(value: unknown, field: string): string {
223+ const text = nonEmpty(value, field);
224+ return text.startsWith("IDFM:") ? text.slice("IDFM:".length) : text;
225+}
226+
227+function isoTimestamp(value: string, field: string): string {
228+ if (value.length === 0 || !Number.isFinite(Date.parse(value))) {
229+ fail("INVALID_METADATA", `${field} must be an ISO timestamp`);
230+ }
231+ return value;
232+}
233+
234+function assertSourceAttribution(source: CatalogSourceAttribution): void {
235+ nonEmpty(source.dataset, "source.dataset");
236+ nonEmpty(source.license, "source.license");
237+ isoTimestamp(source.retrievedAt, "source.retrievedAt");
238+ let url: URL;
239+ try {
240+ url = new URL(source.url);
241+ } catch (error) {
242+ fail("INVALID_SOURCE_URL", "source URL is invalid", error);
243+ }
244+ if (url.protocol !== "https:" || url.username !== "" || url.password !== "") {
245+ fail("INVALID_SOURCE_URL", "source URL must be credential-free HTTPS");
246+ }
247+}
248+
249+function safeUnlink(path: string): void {
250+ try {
251+ unlinkSync(path);
252+ } catch (error) {
253+ const code = (error as NodeJS.ErrnoException).code;
254+ if (code !== "ENOENT") throw error;
255+ }
256+}
257+
258+function removeCandidateFiles(path: string): void {
259+ safeUnlink(path);
260+ safeUnlink(`${path}-journal`);
261+ safeUnlink(`${path}-wal`);
262+ safeUnlink(`${path}-shm`);
263+}
264+
265+function fsyncFile(path: string): void {
266+ const descriptor = openSync(path, "r");
267+ try {
268+ fsyncSync(descriptor);
269+ } finally {
270+ closeSync(descriptor);
271+ }
272+}
273+
274+function fsyncDirectory(path: string): void {
275+ const descriptor = openSync(path, "r");
276+ try {
277+ fsyncSync(descriptor);
278+ } finally {
279+ closeSync(descriptor);
280+ }
281+}
282+
283+export function normalizeCatalogSearchText(value: string): string {
284+ return value
285+ .normalize("NFKD")
286+ .replace(/\p{M}+/gu, "")
287+ .toLocaleLowerCase("fr-FR")
288+ .replace(/[^\p{L}\p{N}]+/gu, " ")
289+ .trim()
290+ .replace(/\s+/gu, " ");
291+}
292+
293+function canonicalTuple(namespace: "place" | "service", fields: readonly string[]): string {
294+ let tuple = `lapin-fute:${namespace}:v1`;
295+ for (const field of fields) tuple += `|${Buffer.byteLength(field, "utf8")}:${field}`;
296+ return tuple;
297+}
298+
299+function opaqueId(prefix: "plc_" | "svc_", tuple: string): string {
300+ return prefix + createHash("sha256").update(tuple, "utf8").digest("base64url");
301+}
302+
303+export function createPlaceIdentity(mode: TransportMode, zdaId: string): {
304+ readonly placeId: string;
305+ readonly canonicalTuple: string;
306+} {
307+ const tuple = canonicalTuple("place", [transportMode(mode), nonEmpty(zdaId, "zdaId")]);
308+ return { placeId: opaqueId("plc_", tuple), canonicalTuple: tuple };
309+}
310+
311+export interface ServiceIdentityInput {
312+ readonly mode: TransportMode;
313+ readonly lineId: string;
314+ readonly monitoringRef: string;
315+ readonly directionId: string;
316+ readonly destinationRef: string;
317+}
318+
319+export function createServiceIdentity(input: ServiceIdentityInput): {
320+ readonly serviceId: string;
321+ readonly canonicalTuple: string;
322+} {
323+ const tuple = canonicalTuple("service", [
324+ transportMode(input.mode),
325+ nonEmpty(input.lineId, "lineId"),
326+ nonEmpty(input.monitoringRef, "monitoringRef"),
327+ nonEmpty(input.directionId, "directionId"),
328+ nonEmpty(input.destinationRef, "destinationRef"),
329+ ]);
330+ return { serviceId: opaqueId("svc_", tuple), canonicalTuple: tuple };
331+}
332+
333+export async function resolveIdfmGtfsDownload(
334+ datasetToken: string,
335+ fetchSource: typeof globalThis.fetch = globalThis.fetch,
336+): Promise<ResolvedGtfsDownload> {
337+ const token = nonEmpty(datasetToken, "datasetToken");
338+ const controller = new AbortController();
339+ const timeout = setTimeout(() => controller.abort(new Error("GTFS metadata request timed out")), DOWNLOAD_TIMEOUT_MS);
340+ try {
341+ const response = await fetchSource(IDFM_CATALOG_SOURCE_URL.gtfsRecord, {
342+ method: "GET",
343+ redirect: "manual",
344+ credentials: "omit",
345+ cache: "no-store",
346+ headers: { Authorization: `apikey ${token}` },
347+ signal: controller.signal,
348+ });
349+ if (response.status !== 200) fail("SOURCE_HTTP_STATUS", `GTFS metadata returned HTTP ${response.status}`);
350+ const mediaType = (response.headers.get("content-type") ?? "").split(";", 1)[0]!.trim().toLowerCase();
351+ if (mediaType !== "application/json") fail("SOURCE_CONTENT_TYPE", "GTFS metadata is not JSON");
352+ if (response.body === null) fail("SOURCE_EMPTY_BODY", "GTFS metadata returned no body");
353+
354+ const chunks: Uint8Array[] = [];
355+ let bytes = 0;
356+ for await (const chunk of response.body) {
357+ bytes += chunk.byteLength;
358+ if (bytes > 262_144) fail("SOURCE_METADATA_TOO_LARGE", "GTFS metadata exceeds 256 KiB");
359+ chunks.push(chunk);
360+ }
361+ let value: unknown;
362+ try {
363+ value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)));
364+ } catch (error) {
365+ fail("INVALID_SOURCE_METADATA", "GTFS metadata is not valid UTF-8 JSON", error);
366+ }
367+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
368+ fail("INVALID_SOURCE_METADATA", "GTFS metadata root is invalid");
369+ }
370+ const results = (value as Record<string, unknown>).results;
371+ if (!Array.isArray(results) || results.length !== 1 || typeof results[0] !== "object" || results[0] === null) {
372+ fail("INVALID_SOURCE_METADATA", "GTFS metadata must resolve exactly one file");
373+ }
374+ const record = results[0] as Record<string, unknown>;
375+ const filename = nonEmpty(record.filename, "GTFS metadata filename");
376+ const urlValue = record.url;
377+ const url = typeof urlValue === "string"
378+ ? urlValue
379+ : typeof urlValue === "object" && urlValue !== null
380+ ? String((urlValue as Record<string, unknown>).href ?? "")
381+ : "";
382+ let resolvedUrl: URL;
383+ try {
384+ resolvedUrl = new URL(nonEmpty(url, "GTFS metadata URL"));
385+ } catch (error) {
386+ fail("INVALID_SOURCE_URL", "GTFS metadata URL is invalid", error);
387+ }
388+ if (resolvedUrl.protocol !== "https:" || resolvedUrl.username !== "" || resolvedUrl.password !== ""
389+ || resolvedUrl.href.includes(token) || resolvedUrl.href.includes(encodeURIComponent(token))) {
390+ fail("INVALID_SOURCE_URL", "GTFS metadata URL must be credential-free HTTPS");
391+ }
392+ return { filename, url: resolvedUrl.href };
393+ } catch (error) {
394+ if (error instanceof CatalogImportError) throw error;
395+ fail("SOURCE_METADATA_FAILED", "GTFS metadata request failed", error);
396+ } finally {
397+ clearTimeout(timeout);
398+ }
399+}
400+
401+export async function downloadCatalogSource(
402+ options: DownloadCatalogSourceOptions,
403+): Promise<DownloadedCatalogSource> {
404+ assertSourceAttribution(options.source);
405+ if (existsSync(options.destinationPath)) fail("DESTINATION_EXISTS", "download destination already exists");
406+
407+ const token = options.datasetToken;
408+ if (options.source.restricted && !token) fail("DATASET_TOKEN_REQUIRED", "restricted source needs IDFM_DATASET_TOKEN");
409+ if (!options.source.restricted && token !== undefined) {
410+ fail("UNEXPECTED_DATASET_TOKEN", "credentials are forbidden for public catalog sources");
411+ }
412+ if (token !== undefined) {
413+ nonEmpty(token, "datasetToken");
414+ const encoded = encodeURIComponent(token);
415+ if (options.source.url.includes(token) || options.source.url.includes(encoded)) {
416+ fail("CREDENTIAL_IN_URL", "dataset token must not appear in a URL");
417+ }
418+ }
419+
420+ mkdirSync(dirname(options.destinationPath), { recursive: true });
421+ const controller = new AbortController();
422+ const timeout = setTimeout(() => controller.abort(new Error("catalog download timed out")), DOWNLOAD_TIMEOUT_MS);
423+ let handle: FileHandle | undefined;
424+ try {
425+ const fetchSource = options.fetch ?? globalThis.fetch;
426+ const response = await fetchSource(options.source.url, {
427+ method: "GET",
428+ redirect: "manual",
429+ credentials: "omit",
430+ cache: "no-store",
431+ headers: token === undefined ? {} : { Authorization: `apikey ${token}` },
432+ signal: controller.signal,
433+ });
434+ if (response.status !== 200) fail("SOURCE_HTTP_STATUS", `catalog source returned HTTP ${response.status}`);
435+ const mediaType = (response.headers.get("content-type") ?? "").split(";", 1)[0]!.trim().toLowerCase();
436+ const accepted = options.expectedContentTypes.map((value) => value.toLowerCase());
437+ if (!accepted.includes(mediaType)) fail("SOURCE_CONTENT_TYPE", `unexpected catalog content type ${mediaType || "<missing>"}`);
438+ if (response.body === null) fail("SOURCE_EMPTY_BODY", "catalog source returned no body");
439+
440+ handle = await openFile(options.destinationPath, "wx", 0o600);
441+ const hash = createHash("sha256");
442+ let bytes = 0;
443+ for await (const chunkValue of response.body) {
444+ const chunk = chunkValue instanceof Uint8Array ? chunkValue : new Uint8Array(chunkValue as ArrayBuffer);
445+ bytes += chunk.byteLength;
446+ if (bytes > MAX_COMPRESSED_DOWNLOAD_BYTES) {
447+ controller.abort(new Error("catalog download exceeded byte limit"));
448+ fail("SOURCE_TOO_LARGE", "catalog source exceeds the 2 GiB download limit");
449+ }
450+ hash.update(chunk);
451+ let offset = 0;
452+ while (offset < chunk.byteLength) {
453+ const result = await handle.write(chunk, offset, chunk.byteLength - offset);
454+ offset += result.bytesWritten;
455+ }
456+ }
457+ await handle.sync();
458+ await handle.close();
459+ handle = undefined;
460+ return {
461+ path: options.destinationPath,
462+ bytes,
463+ sha256: hash.digest("hex"),
464+ source: options.source,
465+ };
466+ } catch (error) {
467+ try {
468+ await handle?.close();
469+ } finally {
470+ safeUnlink(options.destinationPath);
471+ }
472+ if (error instanceof CatalogImportError) throw error;
473+ fail("SOURCE_DOWNLOAD_FAILED", "catalog source download failed", error);
474+ } finally {
475+ clearTimeout(timeout);
476+ }
477+}
478+
479+function createSchema(database: DatabaseSync): void {
480+ database.exec(`
481+ PRAGMA foreign_keys = ON;
482+ PRAGMA journal_mode = DELETE;
483+ PRAGMA synchronous = FULL;
484+ PRAGMA temp_store = FILE;
485+
486+ CREATE TABLE metadata (
487+ key TEXT PRIMARY KEY,
488+ value TEXT NOT NULL
489+ ) STRICT;
490+ CREATE TABLE places (
491+ place_id TEXT PRIMARY KEY,
492+ stop_label TEXT NOT NULL,
493+ locality_label TEXT,
494+ mode TEXT NOT NULL,
495+ canonical_tuple TEXT NOT NULL UNIQUE
496+ ) STRICT;
497+ CREATE TABLE services (
498+ service_id TEXT PRIMARY KEY,
499+ place_id TEXT NOT NULL REFERENCES places(place_id),
500+ stop_label TEXT NOT NULL,
501+ line_label TEXT NOT NULL,
502+ destination_label TEXT NOT NULL,
503+ monitoring_ref TEXT NOT NULL,
504+ line_ref TEXT NOT NULL,
505+ direction_id TEXT NOT NULL,
506+ destination_ref TEXT NOT NULL,
507+ canonical_tuple TEXT NOT NULL UNIQUE
508+ ) STRICT;
509+ CREATE VIRTUAL TABLE place_search USING fts5(
510+ search_text,
511+ place_id UNINDEXED,
512+ tokenize = 'unicode61 remove_diacritics 2'
513+ );
514+
515+ CREATE TABLE stage_routes (
516+ route_id TEXT PRIMARY KEY,
517+ route_short_name TEXT NOT NULL,
518+ route_long_name TEXT NOT NULL
519+ ) STRICT;
520+ CREATE TABLE stage_trips (
521+ trip_id TEXT PRIMARY KEY,
522+ route_id TEXT NOT NULL,
523+ trip_headsign TEXT NOT NULL,
524+ direction_id TEXT NOT NULL
525+ ) STRICT;
526+ CREATE TABLE stage_stop_times (
527+ trip_id TEXT NOT NULL,
528+ stop_id TEXT NOT NULL,
529+ stop_sequence INTEGER NOT NULL,
530+ pickup_type INTEGER NOT NULL,
531+ PRIMARY KEY (trip_id, stop_sequence)
532+ ) STRICT;
533+ CREATE TABLE stage_stops (
534+ stop_id TEXT PRIMARY KEY,
535+ stop_name TEXT NOT NULL,
536+ parent_station TEXT NOT NULL
537+ ) STRICT;
538+ CREATE TABLE stage_agency (
539+ agency_name TEXT PRIMARY KEY
540+ ) STRICT;
541+ CREATE TABLE stage_codes (
542+ object_type TEXT NOT NULL,
543+ object_id TEXT NOT NULL,
544+ object_system TEXT NOT NULL,
545+ object_code TEXT NOT NULL,
546+ PRIMARY KEY (object_type, object_id, object_system, object_code)
547+ ) STRICT;
548+ CREATE TABLE stage_perimeter (
549+ line TEXT NOT NULL,
550+ id_line TEXT NOT NULL,
551+ name_line TEXT NOT NULL,
552+ monitoring_ref TEXT NOT NULL,
553+ stop_name TEXT NOT NULL,
554+ operator_name TEXT NOT NULL,
555+ PRIMARY KEY (line, monitoring_ref)
556+ ) STRICT;
557+ CREATE TABLE stage_arrets (
558+ arrid TEXT PRIMARY KEY,
559+ zdaid TEXT NOT NULL
560+ ) STRICT;
561+ CREATE TABLE stage_zones (
562+ zdaid TEXT PRIMARY KEY,
563+ zdaname TEXT NOT NULL,
564+ zdatown TEXT NOT NULL
565+ ) STRICT;
566+ CREATE TABLE stage_relations (
567+ zdaid TEXT NOT NULL,
568+ arrid TEXT NOT NULL,
569+ artid TEXT NOT NULL,
570+ PRIMARY KEY (zdaid, arrid, artid)
571+ ) STRICT;
572+ CREATE TABLE stage_lines (
573+ id_line TEXT PRIMARY KEY,
574+ name_line TEXT NOT NULL,
575+ shortname_line TEXT NOT NULL,
576+ transport_mode TEXT NOT NULL,
577+ transport_submode TEXT NOT NULL,
578+ status TEXT NOT NULL
579+ ) STRICT;
580+ CREATE TABLE stage_arrets_lignes (
581+ route_id TEXT NOT NULL,
582+ stop_id TEXT NOT NULL,
583+ PRIMARY KEY (route_id, stop_id)
584+ ) STRICT;
585+ `);
586+}
587+
588+function prepareSpecs(database: DatabaseSync): Readonly<Record<keyof CatalogRecordSourceSet, CsvSpec>> {
589+ const route = database.prepare("INSERT INTO stage_routes VALUES (?, ?, ?)");
590+ const trip = database.prepare("INSERT INTO stage_trips VALUES (?, ?, ?, ?)");
591+ const stopTime = database.prepare("INSERT INTO stage_stop_times VALUES (?, ?, ?, ?)");
592+ const stop = database.prepare("INSERT INTO stage_stops VALUES (?, ?, ?)");
593+ const agency = database.prepare("INSERT OR IGNORE INTO stage_agency VALUES (?)");
594+ const code = database.prepare("INSERT OR IGNORE INTO stage_codes VALUES (?, ?, ?, ?)");
595+ const perimeter = database.prepare("INSERT OR IGNORE INTO stage_perimeter VALUES (?, ?, ?, ?, ?, ?)");
596+ const arret = database.prepare("INSERT INTO stage_arrets VALUES (?, ?)");
597+ const zone = database.prepare("INSERT INTO stage_zones VALUES (?, ?, ?)");
598+ const relation = database.prepare("INSERT OR IGNORE INTO stage_relations VALUES (?, ?, ?)");
599+ const line = database.prepare("INSERT INTO stage_lines VALUES (?, ?, ?, ?, ?, ?)");
600+ const arretLigne = database.prepare("INSERT OR IGNORE INTO stage_arrets_lignes VALUES (?, ?)");
601+
602+ return {
603+ routes: {
604+ required: ["route_id", "route_short_name", "route_long_name"],
605+ insert: (row) => route.run(canonicalLineId(row.route_id, "routes.route_id"), optionalText(row.route_short_name), optionalText(row.route_long_name)),
606+ },
607+ trips: {
608+ required: ["route_id", "trip_id", "trip_headsign", "direction_id"],
609+ insert: (row) => trip.run(nonEmpty(row.trip_id, "trips.trip_id"), nonEmpty(row.route_id, "trips.route_id"), optionalText(row.trip_headsign), nonEmpty(row.direction_id, "trips.direction_id")),
610+ },
611+ stopTimes: {
612+ required: ["trip_id", "stop_id", "stop_sequence", "pickup_type"],
613+ insert: (row) => {
614+ const pickupType = row.pickup_type === "" ? 0 : integer(row.pickup_type, "stop_times.pickup_type");
615+ if (pickupType < 0 || pickupType > 3) fail("INVALID_SOURCE", "stop_times.pickup_type is outside GTFS range");
616+ stopTime.run(nonEmpty(row.trip_id, "stop_times.trip_id"), nonEmpty(row.stop_id, "stop_times.stop_id"), integer(row.stop_sequence, "stop_times.stop_sequence"), pickupType);
617+ },
618+ },
619+ stops: {
620+ required: ["stop_id", "stop_name", "parent_station"],
621+ insert: (row) => stop.run(nonEmpty(row.stop_id, "stops.stop_id"), nonEmpty(row.stop_name, "stops.stop_name"), optionalText(row.parent_station)),
622+ },
623+ agency: {
624+ required: ["agency_name", "agency_url", "agency_timezone"],
625+ insert: (row) => {
626+ nonEmpty(row.agency_url, "agency.agency_url");
627+ nonEmpty(row.agency_timezone, "agency.agency_timezone");
628+ agency.run(nonEmpty(row.agency_name, "agency.agency_name"));
629+ },
630+ },
631+ objectCodes: {
632+ required: ["object_type", "object_id", "object_system", "object_code"],
633+ insert: (row) => code.run(nonEmpty(row.object_type, "codes.object_type"), nonEmpty(row.object_id, "codes.object_id"), nonEmpty(row.object_system, "codes.object_system"), nonEmpty(row.object_code, "codes.object_code")),
634+ },
635+ perimeter: {
636+ required: ["line", "name_line", "ns2_stoppointref", "ns2_stopname", "operatorname"],
637+ insert: (row) => perimeter.run(nonEmpty(row.line, "perimeter.line"), canonicalLineId(row.line, "perimeter.line"), nonEmpty(row.name_line, "perimeter.name_line"), nonEmpty(row.ns2_stoppointref, "perimeter.ns2_stoppointref"), nonEmpty(row.ns2_stopname, "perimeter.ns2_stopname"), nonEmpty(row.operatorname, "perimeter.operatorname")),
638+ },
639+ arrets: {
640+ required: ["arrid", "zdaid"],
641+ insert: (row) => arret.run(nonEmpty(row.arrid, "arrets.arrid"), nonEmpty(row.zdaid, "arrets.zdaid")),
642+ },
643+ zones: {
644+ required: ["zdaid", "zdaname", "zdatown"],
645+ insert: (row) => zone.run(nonEmpty(row.zdaid, "zones.zdaid"), nonEmpty(row.zdaname, "zones.zdaname"), optionalText(row.zdatown)),
646+ },
647+ relations: {
648+ required: ["zdaid", "arrid", "artid"],
649+ insert: (row) => relation.run(nonEmpty(row.zdaid, "relations.zdaid"), nonEmpty(row.arrid, "relations.arrid"), nonEmpty(row.artid, "relations.artid")),
650+ },
651+ lines: {
652+ required: ["id_line", "name_line", "shortname_line", "transportmode", "transportsubmode", "status"],
653+ insert: (row) => line.run(canonicalLineId(row.id_line, "lines.id_line"), nonEmpty(row.name_line, "lines.name_line"), optionalText(row.shortname_line), nonEmpty(row.transportmode, "lines.transportmode"), optionalText(row.transportsubmode), nonEmpty(row.status, "lines.status")),
654+ },
655+ arretsLignes: {
656+ required: ["id", "stop_id"],
657+ insert: (row) => arretLigne.run(canonicalLineId(row.id, "arrets_lignes.id"), nonEmpty(row.stop_id, "arrets_lignes.stop_id")),
658+ },
659+ };
660+}
661+
662+function runBatched(database: DatabaseSync, rows: Iterable<Readonly<Record<string, string>>>, spec: CsvSpec): void {
663+ let pending = 0;
664+ database.exec("BEGIN IMMEDIATE");
665+ try {
666+ for (const row of rows) {
667+ spec.insert(row);
668+ pending += 1;
669+ if (pending === IMPORT_TRANSACTION_ROWS) {
670+ database.exec("COMMIT; BEGIN IMMEDIATE");
671+ pending = 0;
672+ }
673+ }
674+ database.exec("COMMIT");
675+ } catch (error) {
676+ database.exec("ROLLBACK");
677+ throw error;
678+ }
679+}
680+
681+function stringRecord(record: CsvRecord, required: readonly string[]): Readonly<Record<string, string>> {
682+ const result: Record<string, string> = {};
683+ for (const key of required) {
684+ if (!Object.hasOwn(record, key)) fail("MISSING_COLUMN", `record is missing ${key}`);
685+ result[key] = String(record[key] ?? "");
686+ }
687+ return result;
688+}
689+
690+function loadRecordSources(database: DatabaseSync, sources: CatalogRecordSourceSet): void {
691+ const specs = prepareSpecs(database);
692+ for (const key of Object.keys(specs) as (keyof CatalogRecordSourceSet)[]) {
693+ const spec = specs[key];
694+ const rows = sources[key].map((row) => stringRecord(row, spec.required));
695+ runBatched(database, rows, spec);
696+ }
697+}
698+
699+class Utf8Decoder extends Transform {
700+ readonly #decoder = new TextDecoder("utf-8", { fatal: true });
701+
702+ override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
703+ try {
704+ this.push(this.#decoder.decode(chunk, { stream: true }));
705+ callback();
706+ } catch (error) {
707+ callback(error as Error);
708+ }
709+ }
710+
711+ override _flush(callback: (error?: Error | null) => void): void {
712+ try {
713+ this.push(this.#decoder.decode());
714+ callback();
715+ } catch (error) {
716+ callback(error as Error);
717+ }
718+ }
719+}
720+
721+class ByteLimit extends Transform {
722+ #bytes = 0;
723+ readonly #limit: number;
724+ readonly #onBytes: (bytes: number) => void;
725+
726+ constructor(limit: number, onBytes: (bytes: number) => void) {
727+ super();
728+ this.#limit = limit;
729+ this.#onBytes = onBytes;
730+ }
731+
732+ override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
733+ this.#bytes += chunk.byteLength;
734+ if (this.#bytes > this.#limit) {
735+ callback(new CatalogImportError("ZIP_ENTRY_TOO_LARGE", "selected GTFS entry exceeds the 8 GiB limit"));
736+ return;
737+ }
738+ try {
739+ this.#onBytes(chunk.byteLength);
740+ this.push(chunk);
741+ callback();
742+ } catch (error) {
743+ callback(error as Error);
744+ }
745+ }
746+}
747+
748+async function loadCsvStream(
749+ database: DatabaseSync,
750+ input: Readable,
751+ delimiter: "," | ";",
752+ spec: CsvSpec,
753+): Promise<number> {
754+ const decoder = new Utf8Decoder();
755+ const parser = parse({
756+ bom: true,
757+ columns: false,
758+ delimiter,
759+ max_record_size: MAX_CSV_RECORD_BYTES,
760+ relax_column_count: false,
761+ relax_quotes: false,
762+ skip_empty_lines: true,
763+ });
764+ const destroyParser = (error: Error): void => parser.destroy(error);
765+ input.once("error", destroyParser);
766+ decoder.once("error", destroyParser);
767+ input.pipe(decoder).pipe(parser);
768+
769+ let indexes: Readonly<Record<string, number>> | undefined;
770+ let pending = 0;
771+ let rows = 0;
772+ database.exec("BEGIN IMMEDIATE");
773+ try {
774+ for await (const value of parser) {
775+ if (!Array.isArray(value) || !value.every((field) => typeof field === "string")) {
776+ fail("INVALID_CSV", "CSV parser returned a non-string record");
777+ }
778+ const fields = value as string[];
779+ if (indexes === undefined) {
780+ const header = fields.map((field) => field.trim());
781+ if (header.length === 0 || header.some((field) => field.length === 0)) fail("INVALID_HEADER", "CSV header is empty");
782+ if (new Set(header).size !== header.length) fail("DUPLICATE_HEADER", "CSV contains duplicate columns");
783+ const mapped: Record<string, number> = {};
784+ for (const required of spec.required) {
785+ const index = header.indexOf(required);
786+ if (index === -1) fail("MISSING_COLUMN", `CSV is missing required column ${required}`);
787+ mapped[required] = index;
788+ }
789+ indexes = mapped;
790+ continue;
791+ }
792+ const row: Record<string, string> = {};
793+ for (const [key, index] of Object.entries(indexes)) row[key] = fields[index] ?? "";
794+ spec.insert(row);
795+ rows += 1;
796+ pending += 1;
797+ if (pending === IMPORT_TRANSACTION_ROWS) {
798+ database.exec("COMMIT; BEGIN IMMEDIATE");
799+ pending = 0;
800+ }
801+ }
802+ if (indexes === undefined) fail("INVALID_HEADER", "CSV source is empty");
803+ database.exec("COMMIT");
804+ return rows;
805+ } catch (error) {
806+ input.destroy();
807+ database.exec("ROLLBACK");
808+ if (error instanceof CatalogImportError) throw error;
809+ fail("CSV_PARSE_FAILED", "CSV source could not be parsed", error);
810+ }
811+}
812+
813+function assertSourceFile(path: string): void {
814+ const size = statSync(path).size;
815+ if (size > MAX_COMPRESSED_DOWNLOAD_BYTES) fail("SOURCE_TOO_LARGE", `${basename(path)} exceeds the 2 GiB source limit`);
816+}
817+
818+function openZip(path: string): Promise<ZipFileLike> {
819+ const { promise, resolve: resolveZip, reject } = Promise.withResolvers<ZipFileLike>();
820+ yauzl.open(path, {
821+ autoClose: false,
822+ decodeStrings: true,
823+ lazyEntries: true,
824+ strictFileNames: true,
825+ validateEntrySizes: true,
826+ }, (error: Error | null, zipFile?: ZipFileLike) => {
827+ if (error || zipFile === undefined) reject(error ?? new Error("ZIP did not open"));
828+ else resolveZip(zipFile);
829+ });
830+ return promise;
831+}
832+
833+function nextZipEntry(zip: ZipFileLike): Promise<ZipEntryLike | undefined> {
834+ const { promise, resolve: resolveEntry, reject } = Promise.withResolvers<ZipEntryLike | undefined>();
835+ const cleanup = (): void => {
836+ zip.removeListener("entry", onEntry as (...args: never[]) => void);
837+ zip.removeListener("end", onEnd as (...args: never[]) => void);
838+ zip.removeListener("error", onError as (...args: never[]) => void);
839+ };
840+ const onEntry = (entry: ZipEntryLike): void => {
841+ cleanup();
842+ resolveEntry(entry);
843+ };
844+ const onEnd = (): void => {
845+ cleanup();
846+ resolveEntry(undefined);
847+ };
848+ const onError = (error: unknown): void => {
849+ cleanup();
850+ reject(error);
851+ };
852+ zip.once("entry", onEntry);
853+ zip.once("end", onEnd);
854+ zip.once("error", onError);
855+ zip.readEntry();
856+ return promise;
857+}
858+
859+function openZipEntry(zip: ZipFileLike, entry: ZipEntryLike): Promise<Readable> {
860+ const { promise, resolve: resolveStream, reject } = Promise.withResolvers<Readable>();
861+ zip.openReadStream(entry, (error, stream) => {
862+ if (error || stream === undefined) reject(error ?? new Error("ZIP entry did not open"));
863+ else resolveStream(stream);
864+ });
865+ return promise;
866+}
867+
868+function gtfsKey(name: string): keyof CatalogRecordSourceSet | undefined {
869+ switch (name) {
870+ case "routes.txt": return "routes";
871+ case "trips.txt": return "trips";
872+ case "stop_times.txt": return "stopTimes";
873+ case "stops.txt": return "stops";
874+ case "agency.txt": return "agency";
875+ case "object_codes_extension.txt":
876+ case "objects_codes_extension.txt": return "objectCodes";
877+ default: return undefined;
878+ }
879+}
880+
881+async function loadGtfsZip(database: DatabaseSync, path: string, specs: Readonly<Record<keyof CatalogRecordSourceSet, CsvSpec>>): Promise<void> {
882+ assertSourceFile(path);
883+ let zip: ZipFileLike | undefined;
884+ const seen = new Set<string>();
885+ let selectedBytes = 0;
886+ try {
887+ zip = await openZip(path);
888+ while (true) {
889+ const entry = await nextZipEntry(zip);
890+ if (entry === undefined) break;
891+ const normalizedName = entry.fileName.replaceAll("\\", "/").split("/").at(-1)!.toLowerCase();
892+ const key = gtfsKey(normalizedName);
893+ if (key === undefined) continue;
894+ if ((entry.generalPurposeBitFlag & 1) !== 0) fail("ENCRYPTED_ZIP_ENTRY", `selected GTFS entry ${normalizedName} is encrypted`);
895+ const conceptualName = Object.hasOwn(CODE_ENTRY_NAME, normalizedName) ? "codes-extension" : normalizedName;
896+ if (seen.has(conceptualName)) fail("DUPLICATE_ZIP_ENTRY", `GTFS contains duplicate required entry ${normalizedName}`);
897+ seen.add(conceptualName);
898+ if (!Number.isSafeInteger(entry.uncompressedSize) || entry.uncompressedSize < 0 || entry.uncompressedSize > MAX_SELECTED_ENTRY_BYTES) {
899+ fail("ZIP_ENTRY_TOO_LARGE", `selected GTFS entry ${normalizedName} exceeds its byte limit`);
900+ }
901+ const stream = await openZipEntry(zip, entry);
902+ const limiter = new ByteLimit(MAX_SELECTED_ENTRY_BYTES, (bytes) => {
903+ selectedBytes += bytes;
904+ if (selectedBytes > MAX_SELECTED_ARCHIVE_BYTES) {
905+ fail("ZIP_AGGREGATE_TOO_LARGE", "selected GTFS entries exceed the 16 GiB aggregate limit");
906+ }
907+ });
908+ stream.once("error", (error) => limiter.destroy(error));
909+ await loadCsvStream(database, stream.pipe(limiter), ",", specs[key]);
910+ }
911+ for (const name of REQUIRED_GTFS_ENTRIES) {
912+ if (!seen.has(name)) fail("MISSING_ZIP_ENTRY", `GTFS is missing required entry ${name}`);
913+ }
914+ if (!seen.has("codes-extension")) fail("MISSING_ZIP_ENTRY", "GTFS is missing its codes-extension entry");
915+ } catch (error) {
916+ if (error instanceof CatalogImportError) throw error;
917+ fail("GTFS_ZIP_FAILED", "GTFS ZIP could not be imported", error);
918+ } finally {
919+ zip?.close();
920+ }
921+}
922+
923+async function loadFileSources(database: DatabaseSync, sources: CatalogFileSourceSet): Promise<void> {
924+ const specs = prepareSpecs(database);
925+ await loadGtfsZip(database, sources.gtfsZipPath, specs);
926+ const csvSources: readonly [keyof CatalogRecordSourceSet, CsvFileSource][] = [
927+ ["perimeter", sources.perimeter],
928+ ["arrets", sources.arrets],
929+ ["zones", sources.zones],
930+ ["relations", sources.relations],
931+ ["lines", sources.lines],
932+ ["arretsLignes", sources.arretsLignes],
933+ ];
934+ for (const [key, source] of csvSources) {
935+ assertSourceFile(source.path);
936+ const rows = await loadCsvStream(database, createReadStream(source.path), source.delimiter ?? ";", specs[key]);
937+ if (source.expectedRows !== undefined && rows !== source.expectedRows) {
938+ fail("SOURCE_ROW_COUNT", `${key} row count ${rows} does not match metadata count ${source.expectedRows}`);
939+ }
940+ }
941+}
942+
943+function createIndexes(database: DatabaseSync): void {
944+ database.exec(`
945+ CREATE INDEX stage_trips_route ON stage_trips(route_id);
946+ CREATE INDEX stage_stop_times_trip ON stage_stop_times(trip_id, stop_sequence);
947+ CREATE INDEX stage_stop_times_stop ON stage_stop_times(stop_id);
948+ CREATE INDEX stage_codes_object ON stage_codes(object_id, object_code);
949+ CREATE INDEX stage_codes_code ON stage_codes(object_code, object_id);
950+ CREATE INDEX stage_perimeter_line ON stage_perimeter(id_line, monitoring_ref);
951+ CREATE INDEX stage_arrets_zda ON stage_arrets(zdaid);
952+ CREATE INDEX stage_relations_art ON stage_relations(artid, zdaid);
953+ CREATE INDEX stage_arrets_lignes_stop ON stage_arrets_lignes(stop_id, route_id);
954+ `);
955+}
956+
957+function deriveCandidates(database: DatabaseSync): void {
958+ database.exec(`
959+ CREATE TABLE stage_line_modes (
960+ id_line TEXT PRIMARY KEY,
961+ mode TEXT NOT NULL,
962+ line_label TEXT NOT NULL
963+ ) STRICT;
964+
965+ INSERT INTO stage_line_modes
966+ SELECT id_line,
967+ CASE
968+ WHEN lower(trim(transport_mode)) = 'bus' THEN 'BUS'
969+ WHEN lower(trim(transport_mode)) = 'metro' THEN 'METRO'
970+ WHEN lower(trim(transport_mode)) = 'tram' THEN 'TRAM'
971+ WHEN lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) = 'local' THEN 'RER'
972+ WHEN lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) = 'suburbanrailway' THEN 'TRANSILIEN'
973+ END,
974+ coalesce(nullif(trim(shortname_line), ''), trim(name_line))
975+ FROM stage_lines
976+ WHERE lower(trim(status)) = 'active'
977+ AND (
978+ lower(trim(transport_mode)) IN ('bus', 'metro', 'tram')
979+ OR (lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) IN ('local', 'suburbanrailway'))
980+ );
981+
982+ CREATE TABLE stage_stop_zda (
983+ stop_id TEXT PRIMARY KEY,
984+ zdaid TEXT,
985+ rail_zdaid TEXT NOT NULL,
986+ resolution TEXT NOT NULL
987+ ) STRICT;
988+
989+ INSERT INTO stage_stop_zda (stop_id, zdaid, rail_zdaid, resolution)
990+ SELECT
991+ stop.stop_id,
992+ coalesce(area.zdaid, relation.zdaid),
993+ CASE
994+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:monomodalStopPlace:%'
995+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), length('IDFM:monomodalStopPlace:') + 1)
996+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:%'
997+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), 6)
998+ ELSE coalesce(nullif(stop.parent_station, ''), stop.stop_id)
999+ END,
1000+ CASE WHEN area.arrid IS NULL THEN 'relations' ELSE 'arrets' END
1001+ FROM stage_stops AS stop
1002+ LEFT JOIN stage_arrets AS area
1003+ ON area.arrid = CASE
1004+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:%'
1005+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), 6)
1006+ ELSE coalesce(nullif(stop.parent_station, ''), stop.stop_id)
1007+ END
1008+ LEFT JOIN (
1009+ SELECT artid, min(zdaid) AS zdaid
1010+ FROM stage_relations
1011+ GROUP BY artid
1012+ ) AS relation
1013+ ON relation.artid = CASE
1014+ WHEN stop.stop_id LIKE 'IDFM:%' THEN substr(stop.stop_id, 6)
1015+ ELSE stop.stop_id
1016+ END;
1017+
1018+ CREATE TABLE stage_terminals AS
1019+ SELECT stop_time.trip_id, stop_time.stop_id, stop.stop_name
1020+ FROM stage_stop_times AS stop_time
1021+ JOIN stage_stops AS stop ON stop.stop_id = stop_time.stop_id
1022+ JOIN (
1023+ SELECT trip_id, max(stop_sequence) AS stop_sequence
1024+ FROM stage_stop_times
1025+ GROUP BY trip_id
1026+ ) AS terminal
1027+ ON terminal.trip_id = stop_time.trip_id
1028+ AND terminal.stop_sequence = stop_time.stop_sequence;
1029+ CREATE UNIQUE INDEX stage_terminals_trip ON stage_terminals(trip_id);
1030+
1031+ CREATE TABLE stage_candidates AS
1032+ WITH joined AS (
1033+ SELECT DISTINCT
1034+ line_mode.mode AS mode,
1035+ CASE
1036+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN stop_zda.rail_zdaid
1037+ ELSE stop_zda.zdaid
1038+ END AS zdaid,
1039+ zone.zdaname AS stop_label,
1040+ zone.zdatown AS locality_label,
1041+ line_mode.id_line AS id_line,
1042+ line_mode.line_label AS line_label,
1043+ CASE
1044+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN terminal.stop_name
1045+ ELSE coalesce(nullif(trim(trip.trip_headsign), ''), terminal.stop_name)
1046+ END AS destination_label,
1047+ perimeter.monitoring_ref AS monitoring_ref,
1048+ perimeter.line AS line_ref,
1049+ trip.direction_id AS direction_id,
1050+ destination_code.object_code AS destination_ref,
1051+ stop_zda.resolution AS resolution
1052+ FROM stage_stop_times AS current_stop_time
1053+ JOIN stage_trips AS trip ON trip.trip_id = current_stop_time.trip_id
1054+ JOIN stage_routes AS route ON route.route_id = CASE WHEN trip.route_id LIKE 'IDFM:%' THEN substr(trip.route_id, 6) ELSE trip.route_id END
1055+ JOIN stage_line_modes AS line_mode ON line_mode.id_line = route.route_id
1056+ JOIN stage_stops AS current_stop ON current_stop.stop_id = current_stop_time.stop_id
1057+ JOIN stage_stop_zda AS stop_zda ON stop_zda.stop_id = current_stop.stop_id
1058+ JOIN stage_terminals AS terminal
1059+ ON terminal.trip_id = current_stop_time.trip_id
1060+ JOIN stage_codes AS destination_code
1061+ ON destination_code.object_id = terminal.stop_id
1062+ AND lower(trim(destination_code.object_system)) = 'source'
1063+ JOIN stage_arrets_lignes AS attached
1064+ ON attached.route_id = route.route_id
1065+ AND attached.stop_id IN (current_stop.stop_id, current_stop.parent_station)
1066+ JOIN stage_perimeter AS perimeter
1067+ ON perimeter.id_line = line_mode.id_line
1068+ AND (
1069+ EXISTS (
1070+ SELECT 1
1071+ FROM stage_codes AS monitoring_code
1072+ WHERE monitoring_code.object_id IN (current_stop.stop_id, current_stop.parent_station)
1073+ AND monitoring_code.object_code = perimeter.monitoring_ref
1074+ )
1075+ OR instr(perimeter.monitoring_ref, ':' || stop_zda.rail_zdaid || ':') > 0
1076+ )
1077+ JOIN stage_zones AS zone
1078+ ON zone.zdaid = CASE
1079+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN stop_zda.rail_zdaid
1080+ ELSE stop_zda.zdaid
1081+ END
1082+ WHERE current_stop_time.pickup_type <> 1
1083+ AND EXISTS (
1084+ SELECT 1 FROM stage_stop_times AS later
1085+ WHERE later.trip_id = current_stop_time.trip_id
1086+ AND later.stop_sequence > current_stop_time.stop_sequence
1087+ )
1088+ AND trim(trip.direction_id) <> ''
1089+ AND trim(destination_code.object_code) <> ''
1090+ )
1091+ SELECT mode, zdaid, min(stop_label) AS stop_label, min(locality_label) AS locality_label,
1092+ id_line, min(line_label) AS line_label, min(destination_label) AS destination_label,
1093+ monitoring_ref, line_ref, direction_id, destination_ref,
1094+ max(CASE WHEN resolution = 'arrets' THEN 1 ELSE 0 END) AS via_arrets
1095+ FROM joined
1096+ WHERE zdaid IS NOT NULL AND trim(zdaid) <> ''
1097+ GROUP BY mode, zdaid, monitoring_ref, id_line, line_ref, direction_id, destination_ref;
1098+ `);
1099+}
1100+
1101+function insertFinalCatalog(database: DatabaseSync, sourceRevision: string, createdAt: string): PlaceResolutionCounts {
1102+ const metadata = database.prepare("INSERT INTO metadata VALUES (?, ?)");
1103+ metadata.run("schema_version", String(SCHEMA_VERSION));
1104+ metadata.run("catalog_version", String(CATALOG_VERSION));
1105+ metadata.run("source_revision", sourceRevision);
1106+ metadata.run("created_at", createdAt);
1107+
1108+ const insertPlace = database.prepare("INSERT INTO places VALUES (?, ?, ?, ?, ?)");
1109+ const insertSearch = database.prepare("INSERT INTO place_search(search_text, place_id) VALUES (?, ?)");
1110+ const insertService = database.prepare("INSERT INTO services VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
1111+ const selectPlace = database.prepare("SELECT canonical_tuple FROM places WHERE place_id = ?");
1112+ const selectService = database.prepare("SELECT place_id, canonical_tuple FROM services WHERE service_id = ?");
1113+
1114+ const places = database.prepare(`
1115+ SELECT mode, zdaid, min(stop_label) AS stop_label, min(locality_label) AS locality_label,
1116+ max(via_arrets) AS via_arrets
1117+ FROM stage_candidates
1118+ GROUP BY mode, zdaid
1119+ ORDER BY mode, zdaid
1120+ `).all() as QueryRow[];
1121+
1122+ const placeResolution: PlaceResolutionCounts = { arrets: 0, relations: 0, direct: 0 };
1123+
1124+ database.exec("BEGIN IMMEDIATE");
1125+ try {
1126+ for (const row of places) {
1127+ const mode = transportMode(row.mode);
1128+ const zdaId = nonEmpty(row.zdaid, "candidate.zdaid");
1129+ const stopLabel = nonEmpty(row.stop_label, "candidate.stop_label");
1130+ const localityLabel = optionalText(row.locality_label);
1131+ const identity = createPlaceIdentity(mode, zdaId);
1132+ const collision = selectPlace.get(identity.placeId) as QueryRow | undefined;
1133+ if (collision !== undefined && collision.canonical_tuple !== identity.canonicalTuple) {
1134+ fail("HASH_COLLISION", "place ID collision detected");
1135+ }
1136+ if (collision === undefined) {
1137+ const searchText = normalizeCatalogSearchText(`${stopLabel} ${localityLabel}`);
1138+ if (searchText.length === 0) fail("CATALOG_SEARCH", "place label produces empty search text");
1139+ insertPlace.run(identity.placeId, stopLabel, localityLabel || null, mode, identity.canonicalTuple);
1140+ insertSearch.run(searchText, identity.placeId);
1141+ }
1142+ if (mode === "RER" || mode === "TRANSILIEN") placeResolution.direct += 1;
1143+ else if (Number(row.via_arrets) === 1) placeResolution.arrets += 1;
1144+ else placeResolution.relations += 1;
1145+ }
1146+
1147+ const candidates = database.prepare(`
1148+ SELECT mode, zdaid, stop_label, line_label, destination_label,
1149+ id_line, monitoring_ref, line_ref, direction_id, destination_ref
1150+ FROM stage_candidates
1151+ ORDER BY mode, zdaid, id_line, monitoring_ref, direction_id, destination_ref
1152+ `).iterate() as Iterable<QueryRow>;
1153+ for (const row of candidates) {
1154+ const mode = transportMode(row.mode);
1155+ const place = createPlaceIdentity(mode, nonEmpty(row.zdaid, "candidate.zdaid"));
1156+ const service = createServiceIdentity({
1157+ mode,
1158+ lineId: nonEmpty(row.id_line, "candidate.id_line"),
1159+ monitoringRef: nonEmpty(row.monitoring_ref, "candidate.monitoring_ref"),
1160+ directionId: nonEmpty(row.direction_id, "candidate.direction_id"),
1161+ destinationRef: nonEmpty(row.destination_ref, "candidate.destination_ref"),
1162+ });
1163+ const collision = selectService.get(service.serviceId) as QueryRow | undefined;
1164+ if (collision !== undefined && collision.canonical_tuple !== service.canonicalTuple) {
1165+ fail("HASH_COLLISION", "service ID collision detected");
1166+ }
1167+ if (collision !== undefined && collision.place_id !== place.placeId) {
1168+ fail("INVALID_SERVICE_PLACE", "one semantic service resolved to multiple places");
1169+ }
1170+ if (collision === undefined) {
1171+ insertService.run(
1172+ service.serviceId,
1173+ place.placeId,
1174+ nonEmpty(row.stop_label, "candidate.stop_label"),
1175+ nonEmpty(row.line_label, "candidate.line_label"),
1176+ nonEmpty(row.destination_label, "candidate.destination_label"),
1177+ nonEmpty(row.monitoring_ref, "candidate.monitoring_ref"),
1178+ nonEmpty(row.line_ref, "candidate.line_ref"),
1179+ nonEmpty(row.direction_id, "candidate.direction_id"),
1180+ nonEmpty(row.destination_ref, "candidate.destination_ref"),
1181+ service.canonicalTuple,
1182+ );
1183+ }
1184+ }
1185+ database.exec("COMMIT");
1186+ } catch (error) {
1187+ database.exec("ROLLBACK");
1188+ throw error;
1189+ }
1190+ return placeResolution;
1191+}
1192+
1193+function validateByteBounds(database: DatabaseSync): void {
1194+ const invalidPlace = database.prepare(`
1195+ SELECT place_id FROM places
1196+ WHERE length(CAST(place_id AS BLOB)) > ?
1197+ OR length(CAST(stop_label AS BLOB)) > ?
1198+ OR (locality_label IS NOT NULL AND length(CAST(locality_label AS BLOB)) > ?)
1199+ LIMIT 1
1200+ `).get(LIMITS.idUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes);
1201+ if (invalidPlace !== undefined) fail("CATALOG_LIMIT", "place exceeds a shared UTF-8 byte limit");
1202+
1203+ const invalidService = database.prepare(`
1204+ SELECT service_id FROM services
1205+ WHERE length(CAST(service_id AS BLOB)) > ?
1206+ OR length(CAST(stop_label AS BLOB)) > ?
1207+ OR length(CAST(line_label AS BLOB)) > ?
1208+ OR length(CAST(destination_label AS BLOB)) > ?
1209+ LIMIT 1
1210+ `).get(LIMITS.idUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes);
1211+ if (invalidService !== undefined) fail("CATALOG_LIMIT", "service exceeds a shared UTF-8 byte limit");
1212+}
1213+
1214+function exactColumns(database: DatabaseSync, table: string, expected: readonly string[]): void {
1215+ const rows = database.prepare(`PRAGMA table_info(${table})`).all() as QueryRow[];
1216+ const actual = rows.map((row) => String(row.name));
1217+ if (actual.length !== expected.length || actual.some((name, index) => name !== expected[index])) {
1218+ fail("CATALOG_SCHEMA", `${table} has an unexpected column contract`);
1219+ }
1220+}
1221+
1222+function metadataValue(database: DatabaseSync, key: string): string | undefined {
1223+ const row = database.prepare("SELECT value FROM metadata WHERE key = ?").get(key) as QueryRow | undefined;
1224+ return row === undefined ? undefined : String(row.value);
1225+}
1226+
1227+function validateOpenCatalog(database: DatabaseSync): CatalogBuildResult {
1228+ database.exec("PRAGMA foreign_keys = ON");
1229+ exactColumns(database, "metadata", ["key", "value"]);
1230+ exactColumns(database, "places", ["place_id", "stop_label", "locality_label", "mode", "canonical_tuple"]);
1231+ exactColumns(database, "services", ["service_id", "place_id", "stop_label", "line_label", "destination_label", "monitoring_ref", "line_ref", "direction_id", "destination_ref", "canonical_tuple"]);
1232+ exactColumns(database, "place_search", ["search_text", "place_id"]);
1233+
1234+ const sourceRevision = metadataValue(database, "source_revision");
1235+ const createdAt = metadataValue(database, "created_at");
1236+ if (metadataValue(database, "schema_version") !== String(SCHEMA_VERSION)
1237+ || metadataValue(database, "catalog_version") !== String(CATALOG_VERSION)
1238+ || sourceRevision === undefined || sourceRevision.trim() === ""
1239+ || createdAt === undefined) {
1240+ fail("CATALOG_METADATA", "catalog metadata is incomplete or incompatible");
1241+ }
1242+ isoTimestamp(createdAt, "metadata.created_at");
1243+
1244+ const integrity = database.prepare("PRAGMA integrity_check").all() as QueryRow[];
1245+ if (integrity.length !== 1 || String(integrity[0]?.integrity_check) !== "ok") fail("CATALOG_INTEGRITY", "SQLite integrity_check failed");
1246+ if ((database.prepare("PRAGMA foreign_key_check").all() as QueryRow[]).length !== 0) fail("CATALOG_FOREIGN_KEY", "SQLite foreign_key_check failed");
1247+ validateByteBounds(database);
1248+
1249+ const missingSearch = database.prepare(`
1250+ SELECT 1 FROM places AS place
1251+ LEFT JOIN place_search AS search ON search.place_id = place.place_id
1252+ WHERE search.place_id IS NULL
1253+ LIMIT 1
1254+ `).get();
1255+ const extraSearch = database.prepare(`
1256+ SELECT 1 FROM place_search AS search
1257+ LEFT JOIN places AS place ON place.place_id = search.place_id
1258+ WHERE place.place_id IS NULL
1259+ LIMIT 1
1260+ `).get();
1261+ if (missingSearch !== undefined || extraSearch !== undefined) fail("CATALOG_SEARCH", "FTS place index does not match places");
1262+ const invalidPlaceService = database.prepare(`
1263+ SELECT 1
1264+ FROM places AS place
1265+ LEFT JOIN services AS service ON service.place_id = place.place_id
1266+ WHERE service.service_id IS NULL OR service.stop_label <> place.stop_label
1267+ LIMIT 1
1268+ `).get();
1269+ if (invalidPlaceService !== undefined) {
1270+ fail("CATALOG_PLACE_SERVICE", "catalog place is empty or has inconsistent service labels");
1271+ }
1272+ if (database.prepare("SELECT 1 FROM place_search WHERE trim(search_text) = '' LIMIT 1").get() !== undefined) {
1273+ fail("CATALOG_SEARCH", "FTS place index contains empty search text");
1274+ }
1275+
1276+ const unresolved = database.prepare(`
1277+ SELECT 1 FROM services
1278+ WHERE trim(monitoring_ref) = '' OR trim(line_ref) = '' OR trim(direction_id) = '' OR trim(destination_ref) = ''
1279+ LIMIT 1
1280+ `).get();
1281+ if (unresolved !== undefined) fail("CATALOG_UNRESOLVED", "catalog contains an unresolvable offered service");
1282+
1283+ for (const row of database.prepare("SELECT place_id, canonical_tuple FROM places").iterate() as Iterable<QueryRow>) {
1284+ if (opaqueId("plc_", String(row.canonical_tuple)) !== row.place_id) fail("HASH_COLLISION", "place ID does not match its canonical tuple");
1285+ }
1286+ for (const row of database.prepare("SELECT service_id, canonical_tuple FROM services").iterate() as Iterable<QueryRow>) {
1287+ if (opaqueId("svc_", String(row.canonical_tuple)) !== row.service_id) fail("HASH_COLLISION", "service ID does not match its canonical tuple");
1288+ }
1289+
1290+ const counts = Object.fromEntries(TRANSPORT_MODE.map((mode) => [mode, 0])) as Record<TransportMode, number>;
1291+ for (const row of database.prepare(`
1292+ SELECT place.mode AS mode, count(service.service_id) AS service_count
1293+ FROM places AS place
1294+ JOIN services AS service ON service.place_id = place.place_id
1295+ GROUP BY place.mode
1296+ `).all() as QueryRow[]) {
1297+ const mode = transportMode(row.mode);
1298+ counts[mode] = Number(row.service_count);
1299+ }
1300+ for (const mode of TRANSPORT_MODE) {
1301+ if (counts[mode] < 1) fail("MODE_COVERAGE", `catalog has no resolvable ${mode} service`);
1302+ }
1303+
1304+ const placeCount = Number((database.prepare("SELECT count(*) AS count FROM places").get() as QueryRow).count);
1305+ const serviceCount = Number((database.prepare("SELECT count(*) AS count FROM services").get() as QueryRow).count);
1306+ return { candidatePath: "", sourceRevision, createdAt, placeCount, serviceCount, countsByMode: counts };
1307+}
1308+
1309+function validateStagedPerimeter(database: DatabaseSync): void {
1310+ const missing = database.prepare(`
1311+ SELECT 1 FROM stage_candidates AS candidate
1312+ WHERE NOT EXISTS (
1313+ SELECT 1 FROM stage_perimeter AS perimeter
1314+ WHERE perimeter.line = candidate.line_ref
1315+ AND perimeter.monitoring_ref = candidate.monitoring_ref
1316+ )
1317+ LIMIT 1
1318+ `).get();
1319+ if (missing !== undefined) fail("PERIMETER_MISMATCH", "offered service lacks its exact perimeter pair");
1320+}
1321+
1322+function validateCandidateSemanticTuples(database: DatabaseSync): void {
1323+ const ambiguous = database.prepare(`
1324+ SELECT 1
1325+ FROM stage_candidates
1326+ GROUP BY mode, id_line, monitoring_ref, direction_id, destination_ref
1327+ HAVING count(*) <> 1 OR count(DISTINCT zdaid) <> 1
1328+ LIMIT 1
1329+ `).get();
1330+ if (ambiguous !== undefined) {
1331+ fail("AMBIGUOUS_SERVICE", "a semantic service tuple resolves to multiple candidate rows or places");
1332+ }
1333+}
1334+
1335+function dropStaging(database: DatabaseSync): void {
1336+ database.exec(`
1337+ DROP TABLE stage_candidates;
1338+ DROP TABLE stage_terminals;
1339+ DROP TABLE stage_stop_zda;
1340+ DROP TABLE stage_line_modes;
1341+ DROP TABLE stage_arrets_lignes;
1342+ DROP TABLE stage_lines;
1343+ DROP TABLE stage_relations;
1344+ DROP TABLE stage_zones;
1345+ DROP TABLE stage_arrets;
1346+ DROP TABLE stage_perimeter;
1347+ DROP TABLE stage_codes;
1348+ DROP TABLE stage_agency;
1349+ DROP TABLE stage_stops;
1350+ DROP TABLE stage_stop_times;
1351+ DROP TABLE stage_trips;
1352+ DROP TABLE stage_routes;
1353+ PRAGMA optimize;
1354+ `);
1355+}
1356+
1357+async function build(
1358+ candidatePath: string,
1359+ sourceRevisionValue: string,
1360+ createdAtValue: string | undefined,
1361+ load: (database: DatabaseSync) => void | Promise<void>,
1362+): Promise<CatalogBuildResult> {
1363+ const candidate = resolve(candidatePath);
1364+ const sourceRevision = nonEmpty(sourceRevisionValue, "sourceRevision");
1365+ const createdAt = isoTimestamp(createdAtValue ?? new Date().toISOString(), "createdAt");
1366+ if (existsSync(candidate)) fail("CANDIDATE_EXISTS", "catalog candidate path already exists");
1367+ mkdirSync(dirname(candidate), { recursive: true });
1368+
1369+ let database: DatabaseSync | undefined;
1370+ try {
1371+ database = new DatabaseSync(candidate);
1372+ createSchema(database);
1373+ await load(database);
1374+ createIndexes(database);
1375+ deriveCandidates(database);
1376+ validateStagedPerimeter(database);
1377+ validateCandidateSemanticTuples(database);
1378+ const placeResolution = insertFinalCatalog(database, sourceRevision, createdAt);
1379+ dropStaging(database);
1380+ const result = validateOpenCatalog(database);
1381+ database.close();
1382+ database = undefined;
1383+ fsyncFile(candidate);
1384+ return { ...result, candidatePath: candidate, placeResolution };
1385+ } catch (error) {
1386+ try {
1387+ database?.close();
1388+ } finally {
1389+ removeCandidateFiles(candidate);
1390+ }
1391+ if (error instanceof CatalogImportError) throw error;
1392+ fail("CATALOG_BUILD_FAILED", "catalog candidate build failed", error);
1393+ }
1394+}
1395+
1396+export function buildCatalogCandidate(options: BuildCatalogOptions): Promise<CatalogBuildResult> {
1397+ return build(options.candidatePath, options.sourceRevision, options.createdAt, (database) => loadFileSources(database, options.sources));
1398+}
1399+
1400+export function buildCatalogCandidateFromRecords(options: BuildCatalogFromRecordsOptions): Promise<CatalogBuildResult> {
1401+ return build(options.candidatePath, options.sourceRevision, options.createdAt, (database) => loadRecordSources(database, options.sources));
1402+}
1403+
1404+export function validateCatalogCandidate(path: string): CatalogBuildResult {
1405+ const candidate = resolve(path);
1406+ let database: DatabaseSync | undefined;
1407+ try {
1408+ database = new DatabaseSync(candidate, { readOnly: true });
1409+ const result = validateOpenCatalog(database);
1410+ return { ...result, candidatePath: candidate };
1411+ } catch (error) {
1412+ if (error instanceof CatalogImportError) throw error;
1413+ fail("CATALOG_VALIDATION_FAILED", "catalog candidate validation failed", error);
1414+ } finally {
1415+ database?.close();
1416+ }
1417+}
1418+
1419+export function activateCatalogCandidate(candidatePath: string, activePath: string): CatalogBuildResult {
1420+ const candidate = resolve(candidatePath);
1421+ const active = resolve(activePath);
1422+ if (candidate === active) fail("INVALID_ACTIVATION", "candidate and active paths must differ");
1423+ mkdirSync(dirname(active), { recursive: true });
1424+ let result: CatalogBuildResult;
1425+ try {
1426+ result = validateCatalogCandidate(candidate);
1427+ if (statSync(candidate).dev !== statSync(dirname(active)).dev) {
1428+ fail("CROSS_DEVICE_ACTIVATION", "catalog candidate and active directory must share a filesystem");
1429+ }
1430+ fsyncFile(candidate);
1431+ } catch (error) {
1432+ removeCandidateFiles(candidate);
1433+ throw error;
1434+ }
1435+ try {
1436+ renameSync(candidate, active);
1437+ } catch (error) {
1438+ removeCandidateFiles(candidate);
1439+ throw error;
1440+ }
1441+ try {
1442+ fsyncDirectory(dirname(active));
1443+ } catch {
1444+ // The rename already replaced the active path; never unlink either file.
1445+ // Revalidate what is actually active and report the applied activation.
1446+ const reapplied = validateCatalogCandidate(active);
1447+ return { ...reapplied, candidatePath: active, revalidatedAfterRenameFailure: true };
1448+ }
1449+ return { ...result, candidatePath: active };
1450+}
1451+
1452+export async function refreshCatalog(options: BuildCatalogOptions & { readonly activePath: string }): Promise<CatalogBuildResult> {
1453+ await buildCatalogCandidate(options);
1454+ return activateCatalogCandidate(options.candidatePath, options.activePath);
1455+}
1456+
1457+export async function refreshCatalogFromRecords(
1458+ options: BuildCatalogFromRecordsOptions & { readonly activePath: string },
1459+): Promise<CatalogBuildResult> {
1460+ await buildCatalogCandidateFromRecords(options);
1461+ return activateCatalogCandidate(options.candidatePath, options.activePath);
1462+}
new file mode 100644
@@ -0,0 +1,1462 @@
1+import { createHash } from "node:crypto";
2+import {
3+ closeSync,
4+ createReadStream,
5+ existsSync,
6+ fsyncSync,
7+ mkdirSync,
8+ openSync,
9+ renameSync,
10+ statSync,
11+ unlinkSync,
12+} from "node:fs";
13+import { open as openFile } from "node:fs/promises";
14+import type { FileHandle } from "node:fs/promises";
15+import { basename, dirname, resolve } from "node:path";
16+import { Readable, Transform } from "node:stream";
17+import { DatabaseSync } from "node:sqlite";
18+import { parse } from "csv-parse";
19+import yauzl from "yauzl";
20+import {
21+ LIMITS,
22+ SCHEMA_VERSION,
23+ TRANSPORT_MODE,
24+} from "../../contracts/src/index.ts";
25+import type { TransportMode } from "../../contracts/src/index.ts";
26+
27+export const CATALOG_VERSION = 1 as const;
28+export const DOWNLOAD_TIMEOUT_MS = 15 * 60 * 1_000;
29+export const MAX_COMPRESSED_DOWNLOAD_BYTES = 2 * 1024 ** 3;
30+export const MAX_SELECTED_ENTRY_BYTES = 8 * 1024 ** 3;
31+export const MAX_SELECTED_ARCHIVE_BYTES = 16 * 1024 ** 3;
32+export const MAX_CSV_RECORD_BYTES = 1024 * 1024;
33+export const IMPORT_TRANSACTION_ROWS = 10_000;
34+export const IDFM_CATALOG_SOURCE_URL = {
35+ gtfsRecord: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/offre-horaires-tc-gtfs-idfm/records?limit=1",
36+ perimeter: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/perimetre-des-donnees-tr-disponibles-plateforme-idfm/exports/csv?limit=-1",
37+ arrets: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets/exports/csv?limit=-1",
38+ zones: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/zones-d-arrets/exports/csv?limit=-1",
39+ relations: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/relations/exports/csv?limit=-1",
40+ lines: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/referentiel-des-lignes/exports/csv?limit=-1",
41+ arretsLignes: "https://data.iledefrance-mobilites.fr/api/explore/v2.1/catalog/datasets/arrets-lignes/exports/csv?limit=-1",
42+} as const;
43+
44+const SUPPORTED_MODE: Readonly<Record<TransportMode, true>> = {
45+ BUS: true,
46+ METRO: true,
47+ TRAM: true,
48+ RER: true,
49+ TRANSILIEN: true,
50+};
51+const REQUIRED_GTFS_ENTRIES = [
52+ "routes.txt",
53+ "trips.txt",
54+ "stop_times.txt",
55+ "stops.txt",
56+ "agency.txt",
57+] as const;
58+const CODE_ENTRY_NAME: Readonly<Record<string, true>> = {
59+ "object_codes_extension.txt": true,
60+ "objects_codes_extension.txt": true,
61+};
62+
63+export class CatalogImportError extends Error {
64+ readonly code: string;
65+
66+ constructor(code: string, message: string, options?: ErrorOptions) {
67+ super(message, options);
68+ this.name = "CatalogImportError";
69+ this.code = code;
70+ }
71+}
72+
73+export type CsvValue = string | number | boolean | null | undefined;
74+export type CsvRecord = Readonly<Record<string, CsvValue>>;
75+
76+export interface CatalogRecordSourceSet {
77+ readonly routes: readonly CsvRecord[];
78+ readonly trips: readonly CsvRecord[];
79+ readonly stopTimes: readonly CsvRecord[];
80+ readonly stops: readonly CsvRecord[];
81+ readonly agency: readonly CsvRecord[];
82+ readonly objectCodes: readonly CsvRecord[];
83+ readonly perimeter: readonly CsvRecord[];
84+ readonly arrets: readonly CsvRecord[];
85+ readonly zones: readonly CsvRecord[];
86+ readonly relations: readonly CsvRecord[];
87+ readonly lines: readonly CsvRecord[];
88+ readonly arretsLignes: readonly CsvRecord[];
89+}
90+
91+export interface CsvFileSource {
92+ readonly path: string;
93+ readonly delimiter?: "," | ";";
94+ readonly expectedRows?: number;
95+}
96+
97+export interface CatalogFileSourceSet {
98+ readonly gtfsZipPath: string;
99+ readonly perimeter: CsvFileSource;
100+ readonly arrets: CsvFileSource;
101+ readonly zones: CsvFileSource;
102+ readonly relations: CsvFileSource;
103+ readonly lines: CsvFileSource;
104+ readonly arretsLignes: CsvFileSource;
105+}
106+
107+export interface CatalogSourceAttribution {
108+ readonly dataset: string;
109+ readonly url: string;
110+ readonly retrievedAt: string;
111+ readonly license: string;
112+ readonly restricted?: boolean;
113+}
114+
115+export interface DownloadCatalogSourceOptions {
116+ readonly source: CatalogSourceAttribution;
117+ readonly destinationPath: string;
118+ readonly expectedContentTypes: readonly string[];
119+ readonly datasetToken?: string;
120+ readonly fetch?: typeof globalThis.fetch;
121+}
122+
123+export interface DownloadedCatalogSource {
124+ readonly path: string;
125+ readonly bytes: number;
126+ readonly sha256: string;
127+ readonly source: CatalogSourceAttribution;
128+}
129+export interface ResolvedGtfsDownload {
130+ readonly filename: string;
131+ readonly url: string;
132+}
133+
134+export interface BuildCatalogOptions {
135+ readonly candidatePath: string;
136+ readonly sourceRevision: string;
137+ readonly createdAt?: string;
138+ readonly sources: CatalogFileSourceSet;
139+}
140+
141+export interface BuildCatalogFromRecordsOptions {
142+ readonly candidatePath: string;
143+ readonly sourceRevision: string;
144+ readonly createdAt?: string;
145+ readonly sources: CatalogRecordSourceSet;
146+}
147+
148+export interface PlaceResolutionCounts {
149+ readonly arrets: number;
150+ readonly relations: number;
151+ readonly direct: number;
152+}
153+
154+export interface CatalogBuildResult {
155+ readonly candidatePath: string;
156+ readonly sourceRevision: string;
157+ readonly createdAt: string;
158+ readonly placeCount: number;
159+ readonly serviceCount: number;
160+ readonly countsByMode: Readonly<Record<TransportMode, number>>;
161+ readonly placeResolution?: PlaceResolutionCounts;
162+ readonly revalidatedAfterRenameFailure?: true;
163+}
164+
165+interface CsvSpec {
166+ readonly required: readonly string[];
167+ readonly insert: (row: Readonly<Record<string, string>>) => void;
168+}
169+
170+interface ZipEntryLike {
171+ readonly fileName: string;
172+ readonly uncompressedSize: number;
173+ readonly generalPurposeBitFlag: number;
174+}
175+
176+interface ZipFileLike {
177+ readEntry(): void;
178+ close(): void;
179+ once(event: "entry", listener: (entry: ZipEntryLike) => void): this;
180+ once(event: "end" | "error", listener: (...args: unknown[]) => void): this;
181+ removeListener(event: string, listener: (...args: never[]) => void): this;
182+ openReadStream(
183+ entry: ZipEntryLike,
184+ callback: (error: Error | null, stream?: Readable) => void,
185+ ): void;
186+}
187+
188+interface QueryRow {
189+ readonly [key: string]: unknown;
190+}
191+
192+function fail(code: string, message: string, cause?: unknown): never {
193+ throw new CatalogImportError(code, message, cause === undefined ? undefined : { cause });
194+}
195+
196+function nonEmpty(value: unknown, field: string): string {
197+ const text = String(value ?? "").trim();
198+ if (text.length === 0 || text.includes("\0")) fail("INVALID_SOURCE", `${field} must be non-empty`);
199+ return text;
200+}
201+
202+function optionalText(value: unknown): string {
203+ const text = String(value ?? "").trim();
204+ if (text.includes("\0")) fail("INVALID_SOURCE", "source text contains NUL");
205+ return text;
206+}
207+
208+function integer(value: unknown, field: string): number {
209+ const text = nonEmpty(value, field);
210+ if (!/^-?\d+$/u.test(text)) fail("INVALID_SOURCE", `${field} must be an integer`);
211+ const parsed = Number(text);
212+ if (!Number.isSafeInteger(parsed)) fail("INVALID_SOURCE", `${field} exceeds safe integer range`);
213+ return parsed;
214+}
215+
216+function transportMode(value: unknown): TransportMode {
217+ const mode = nonEmpty(value, "mode");
218+ if (!Object.hasOwn(SUPPORTED_MODE, mode)) fail("INVALID_MODE", `unsupported transport mode ${mode}`);
219+ return mode as TransportMode;
220+}
221+
222+function canonicalLineId(value: unknown, field: string): string {
223+ const text = nonEmpty(value, field);
224+ return text.startsWith("IDFM:") ? text.slice("IDFM:".length) : text;
225+}
226+
227+function isoTimestamp(value: string, field: string): string {
228+ if (value.length === 0 || !Number.isFinite(Date.parse(value))) {
229+ fail("INVALID_METADATA", `${field} must be an ISO timestamp`);
230+ }
231+ return value;
232+}
233+
234+function assertSourceAttribution(source: CatalogSourceAttribution): void {
235+ nonEmpty(source.dataset, "source.dataset");
236+ nonEmpty(source.license, "source.license");
237+ isoTimestamp(source.retrievedAt, "source.retrievedAt");
238+ let url: URL;
239+ try {
240+ url = new URL(source.url);
241+ } catch (error) {
242+ fail("INVALID_SOURCE_URL", "source URL is invalid", error);
243+ }
244+ if (url.protocol !== "https:" || url.username !== "" || url.password !== "") {
245+ fail("INVALID_SOURCE_URL", "source URL must be credential-free HTTPS");
246+ }
247+}
248+
249+function safeUnlink(path: string): void {
250+ try {
251+ unlinkSync(path);
252+ } catch (error) {
253+ const code = (error as NodeJS.ErrnoException).code;
254+ if (code !== "ENOENT") throw error;
255+ }
256+}
257+
258+function removeCandidateFiles(path: string): void {
259+ safeUnlink(path);
260+ safeUnlink(`${path}-journal`);
261+ safeUnlink(`${path}-wal`);
262+ safeUnlink(`${path}-shm`);
263+}
264+
265+function fsyncFile(path: string): void {
266+ const descriptor = openSync(path, "r");
267+ try {
268+ fsyncSync(descriptor);
269+ } finally {
270+ closeSync(descriptor);
271+ }
272+}
273+
274+function fsyncDirectory(path: string): void {
275+ const descriptor = openSync(path, "r");
276+ try {
277+ fsyncSync(descriptor);
278+ } finally {
279+ closeSync(descriptor);
280+ }
281+}
282+
283+export function normalizeCatalogSearchText(value: string): string {
284+ return value
285+ .normalize("NFKD")
286+ .replace(/\p{M}+/gu, "")
287+ .toLocaleLowerCase("fr-FR")
288+ .replace(/[^\p{L}\p{N}]+/gu, " ")
289+ .trim()
290+ .replace(/\s+/gu, " ");
291+}
292+
293+function canonicalTuple(namespace: "place" | "service", fields: readonly string[]): string {
294+ let tuple = `lapin-fute:${namespace}:v1`;
295+ for (const field of fields) tuple += `|${Buffer.byteLength(field, "utf8")}:${field}`;
296+ return tuple;
297+}
298+
299+function opaqueId(prefix: "plc_" | "svc_", tuple: string): string {
300+ return prefix + createHash("sha256").update(tuple, "utf8").digest("base64url");
301+}
302+
303+export function createPlaceIdentity(mode: TransportMode, zdaId: string): {
304+ readonly placeId: string;
305+ readonly canonicalTuple: string;
306+} {
307+ const tuple = canonicalTuple("place", [transportMode(mode), nonEmpty(zdaId, "zdaId")]);
308+ return { placeId: opaqueId("plc_", tuple), canonicalTuple: tuple };
309+}
310+
311+export interface ServiceIdentityInput {
312+ readonly mode: TransportMode;
313+ readonly lineId: string;
314+ readonly monitoringRef: string;
315+ readonly directionId: string;
316+ readonly destinationRef: string;
317+}
318+
319+export function createServiceIdentity(input: ServiceIdentityInput): {
320+ readonly serviceId: string;
321+ readonly canonicalTuple: string;
322+} {
323+ const tuple = canonicalTuple("service", [
324+ transportMode(input.mode),
325+ nonEmpty(input.lineId, "lineId"),
326+ nonEmpty(input.monitoringRef, "monitoringRef"),
327+ nonEmpty(input.directionId, "directionId"),
328+ nonEmpty(input.destinationRef, "destinationRef"),
329+ ]);
330+ return { serviceId: opaqueId("svc_", tuple), canonicalTuple: tuple };
331+}
332+
333+export async function resolveIdfmGtfsDownload(
334+ datasetToken: string,
335+ fetchSource: typeof globalThis.fetch = globalThis.fetch,
336+): Promise<ResolvedGtfsDownload> {
337+ const token = nonEmpty(datasetToken, "datasetToken");
338+ const controller = new AbortController();
339+ const timeout = setTimeout(() => controller.abort(new Error("GTFS metadata request timed out")), DOWNLOAD_TIMEOUT_MS);
340+ try {
341+ const response = await fetchSource(IDFM_CATALOG_SOURCE_URL.gtfsRecord, {
342+ method: "GET",
343+ redirect: "manual",
344+ credentials: "omit",
345+ cache: "no-store",
346+ headers: { Authorization: `apikey ${token}` },
347+ signal: controller.signal,
348+ });
349+ if (response.status !== 200) fail("SOURCE_HTTP_STATUS", `GTFS metadata returned HTTP ${response.status}`);
350+ const mediaType = (response.headers.get("content-type") ?? "").split(";", 1)[0]!.trim().toLowerCase();
351+ if (mediaType !== "application/json") fail("SOURCE_CONTENT_TYPE", "GTFS metadata is not JSON");
352+ if (response.body === null) fail("SOURCE_EMPTY_BODY", "GTFS metadata returned no body");
353+
354+ const chunks: Uint8Array[] = [];
355+ let bytes = 0;
356+ for await (const chunk of response.body) {
357+ bytes += chunk.byteLength;
358+ if (bytes > 262_144) fail("SOURCE_METADATA_TOO_LARGE", "GTFS metadata exceeds 256 KiB");
359+ chunks.push(chunk);
360+ }
361+ let value: unknown;
362+ try {
363+ value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)));
364+ } catch (error) {
365+ fail("INVALID_SOURCE_METADATA", "GTFS metadata is not valid UTF-8 JSON", error);
366+ }
367+ if (typeof value !== "object" || value === null || Array.isArray(value)) {
368+ fail("INVALID_SOURCE_METADATA", "GTFS metadata root is invalid");
369+ }
370+ const results = (value as Record<string, unknown>).results;
371+ if (!Array.isArray(results) || results.length !== 1 || typeof results[0] !== "object" || results[0] === null) {
372+ fail("INVALID_SOURCE_METADATA", "GTFS metadata must resolve exactly one file");
373+ }
374+ const record = results[0] as Record<string, unknown>;
375+ const filename = nonEmpty(record.filename, "GTFS metadata filename");
376+ const urlValue = record.url;
377+ const url = typeof urlValue === "string"
378+ ? urlValue
379+ : typeof urlValue === "object" && urlValue !== null
380+ ? String((urlValue as Record<string, unknown>).href ?? "")
381+ : "";
382+ let resolvedUrl: URL;
383+ try {
384+ resolvedUrl = new URL(nonEmpty(url, "GTFS metadata URL"));
385+ } catch (error) {
386+ fail("INVALID_SOURCE_URL", "GTFS metadata URL is invalid", error);
387+ }
388+ if (resolvedUrl.protocol !== "https:" || resolvedUrl.username !== "" || resolvedUrl.password !== ""
389+ || resolvedUrl.href.includes(token) || resolvedUrl.href.includes(encodeURIComponent(token))) {
390+ fail("INVALID_SOURCE_URL", "GTFS metadata URL must be credential-free HTTPS");
391+ }
392+ return { filename, url: resolvedUrl.href };
393+ } catch (error) {
394+ if (error instanceof CatalogImportError) throw error;
395+ fail("SOURCE_METADATA_FAILED", "GTFS metadata request failed", error);
396+ } finally {
397+ clearTimeout(timeout);
398+ }
399+}
400+
401+export async function downloadCatalogSource(
402+ options: DownloadCatalogSourceOptions,
403+): Promise<DownloadedCatalogSource> {
404+ assertSourceAttribution(options.source);
405+ if (existsSync(options.destinationPath)) fail("DESTINATION_EXISTS", "download destination already exists");
406+
407+ const token = options.datasetToken;
408+ if (options.source.restricted && !token) fail("DATASET_TOKEN_REQUIRED", "restricted source needs IDFM_DATASET_TOKEN");
409+ if (!options.source.restricted && token !== undefined) {
410+ fail("UNEXPECTED_DATASET_TOKEN", "credentials are forbidden for public catalog sources");
411+ }
412+ if (token !== undefined) {
413+ nonEmpty(token, "datasetToken");
414+ const encoded = encodeURIComponent(token);
415+ if (options.source.url.includes(token) || options.source.url.includes(encoded)) {
416+ fail("CREDENTIAL_IN_URL", "dataset token must not appear in a URL");
417+ }
418+ }
419+
420+ mkdirSync(dirname(options.destinationPath), { recursive: true });
421+ const controller = new AbortController();
422+ const timeout = setTimeout(() => controller.abort(new Error("catalog download timed out")), DOWNLOAD_TIMEOUT_MS);
423+ let handle: FileHandle | undefined;
424+ try {
425+ const fetchSource = options.fetch ?? globalThis.fetch;
426+ const response = await fetchSource(options.source.url, {
427+ method: "GET",
428+ redirect: "manual",
429+ credentials: "omit",
430+ cache: "no-store",
431+ headers: token === undefined ? {} : { Authorization: `apikey ${token}` },
432+ signal: controller.signal,
433+ });
434+ if (response.status !== 200) fail("SOURCE_HTTP_STATUS", `catalog source returned HTTP ${response.status}`);
435+ const mediaType = (response.headers.get("content-type") ?? "").split(";", 1)[0]!.trim().toLowerCase();
436+ const accepted = options.expectedContentTypes.map((value) => value.toLowerCase());
437+ if (!accepted.includes(mediaType)) fail("SOURCE_CONTENT_TYPE", `unexpected catalog content type ${mediaType || "<missing>"}`);
438+ if (response.body === null) fail("SOURCE_EMPTY_BODY", "catalog source returned no body");
439+
440+ handle = await openFile(options.destinationPath, "wx", 0o600);
441+ const hash = createHash("sha256");
442+ let bytes = 0;
443+ for await (const chunkValue of response.body) {
444+ const chunk = chunkValue instanceof Uint8Array ? chunkValue : new Uint8Array(chunkValue as ArrayBuffer);
445+ bytes += chunk.byteLength;
446+ if (bytes > MAX_COMPRESSED_DOWNLOAD_BYTES) {
447+ controller.abort(new Error("catalog download exceeded byte limit"));
448+ fail("SOURCE_TOO_LARGE", "catalog source exceeds the 2 GiB download limit");
449+ }
450+ hash.update(chunk);
451+ let offset = 0;
452+ while (offset < chunk.byteLength) {
453+ const result = await handle.write(chunk, offset, chunk.byteLength - offset);
454+ offset += result.bytesWritten;
455+ }
456+ }
457+ await handle.sync();
458+ await handle.close();
459+ handle = undefined;
460+ return {
461+ path: options.destinationPath,
462+ bytes,
463+ sha256: hash.digest("hex"),
464+ source: options.source,
465+ };
466+ } catch (error) {
467+ try {
468+ await handle?.close();
469+ } finally {
470+ safeUnlink(options.destinationPath);
471+ }
472+ if (error instanceof CatalogImportError) throw error;
473+ fail("SOURCE_DOWNLOAD_FAILED", "catalog source download failed", error);
474+ } finally {
475+ clearTimeout(timeout);
476+ }
477+}
478+
479+function createSchema(database: DatabaseSync): void {
480+ database.exec(`
481+ PRAGMA foreign_keys = ON;
482+ PRAGMA journal_mode = DELETE;
483+ PRAGMA synchronous = FULL;
484+ PRAGMA temp_store = FILE;
485+
486+ CREATE TABLE metadata (
487+ key TEXT PRIMARY KEY,
488+ value TEXT NOT NULL
489+ ) STRICT;
490+ CREATE TABLE places (
491+ place_id TEXT PRIMARY KEY,
492+ stop_label TEXT NOT NULL,
493+ locality_label TEXT,
494+ mode TEXT NOT NULL,
495+ canonical_tuple TEXT NOT NULL UNIQUE
496+ ) STRICT;
497+ CREATE TABLE services (
498+ service_id TEXT PRIMARY KEY,
499+ place_id TEXT NOT NULL REFERENCES places(place_id),
500+ stop_label TEXT NOT NULL,
501+ line_label TEXT NOT NULL,
502+ destination_label TEXT NOT NULL,
503+ monitoring_ref TEXT NOT NULL,
504+ line_ref TEXT NOT NULL,
505+ direction_id TEXT NOT NULL,
506+ destination_ref TEXT NOT NULL,
507+ canonical_tuple TEXT NOT NULL UNIQUE
508+ ) STRICT;
509+ CREATE VIRTUAL TABLE place_search USING fts5(
510+ search_text,
511+ place_id UNINDEXED,
512+ tokenize = 'unicode61 remove_diacritics 2'
513+ );
514+
515+ CREATE TABLE stage_routes (
516+ route_id TEXT PRIMARY KEY,
517+ route_short_name TEXT NOT NULL,
518+ route_long_name TEXT NOT NULL
519+ ) STRICT;
520+ CREATE TABLE stage_trips (
521+ trip_id TEXT PRIMARY KEY,
522+ route_id TEXT NOT NULL,
523+ trip_headsign TEXT NOT NULL,
524+ direction_id TEXT NOT NULL
525+ ) STRICT;
526+ CREATE TABLE stage_stop_times (
527+ trip_id TEXT NOT NULL,
528+ stop_id TEXT NOT NULL,
529+ stop_sequence INTEGER NOT NULL,
530+ pickup_type INTEGER NOT NULL,
531+ PRIMARY KEY (trip_id, stop_sequence)
532+ ) STRICT;
533+ CREATE TABLE stage_stops (
534+ stop_id TEXT PRIMARY KEY,
535+ stop_name TEXT NOT NULL,
536+ parent_station TEXT NOT NULL
537+ ) STRICT;
538+ CREATE TABLE stage_agency (
539+ agency_name TEXT PRIMARY KEY
540+ ) STRICT;
541+ CREATE TABLE stage_codes (
542+ object_type TEXT NOT NULL,
543+ object_id TEXT NOT NULL,
544+ object_system TEXT NOT NULL,
545+ object_code TEXT NOT NULL,
546+ PRIMARY KEY (object_type, object_id, object_system, object_code)
547+ ) STRICT;
548+ CREATE TABLE stage_perimeter (
549+ line TEXT NOT NULL,
550+ id_line TEXT NOT NULL,
551+ name_line TEXT NOT NULL,
552+ monitoring_ref TEXT NOT NULL,
553+ stop_name TEXT NOT NULL,
554+ operator_name TEXT NOT NULL,
555+ PRIMARY KEY (line, monitoring_ref)
556+ ) STRICT;
557+ CREATE TABLE stage_arrets (
558+ arrid TEXT PRIMARY KEY,
559+ zdaid TEXT NOT NULL
560+ ) STRICT;
561+ CREATE TABLE stage_zones (
562+ zdaid TEXT PRIMARY KEY,
563+ zdaname TEXT NOT NULL,
564+ zdatown TEXT NOT NULL
565+ ) STRICT;
566+ CREATE TABLE stage_relations (
567+ zdaid TEXT NOT NULL,
568+ arrid TEXT NOT NULL,
569+ artid TEXT NOT NULL,
570+ PRIMARY KEY (zdaid, arrid, artid)
571+ ) STRICT;
572+ CREATE TABLE stage_lines (
573+ id_line TEXT PRIMARY KEY,
574+ name_line TEXT NOT NULL,
575+ shortname_line TEXT NOT NULL,
576+ transport_mode TEXT NOT NULL,
577+ transport_submode TEXT NOT NULL,
578+ status TEXT NOT NULL
579+ ) STRICT;
580+ CREATE TABLE stage_arrets_lignes (
581+ route_id TEXT NOT NULL,
582+ stop_id TEXT NOT NULL,
583+ PRIMARY KEY (route_id, stop_id)
584+ ) STRICT;
585+ `);
586+}
587+
588+function prepareSpecs(database: DatabaseSync): Readonly<Record<keyof CatalogRecordSourceSet, CsvSpec>> {
589+ const route = database.prepare("INSERT INTO stage_routes VALUES (?, ?, ?)");
590+ const trip = database.prepare("INSERT INTO stage_trips VALUES (?, ?, ?, ?)");
591+ const stopTime = database.prepare("INSERT INTO stage_stop_times VALUES (?, ?, ?, ?)");
592+ const stop = database.prepare("INSERT INTO stage_stops VALUES (?, ?, ?)");
593+ const agency = database.prepare("INSERT OR IGNORE INTO stage_agency VALUES (?)");
594+ const code = database.prepare("INSERT OR IGNORE INTO stage_codes VALUES (?, ?, ?, ?)");
595+ const perimeter = database.prepare("INSERT OR IGNORE INTO stage_perimeter VALUES (?, ?, ?, ?, ?, ?)");
596+ const arret = database.prepare("INSERT INTO stage_arrets VALUES (?, ?)");
597+ const zone = database.prepare("INSERT INTO stage_zones VALUES (?, ?, ?)");
598+ const relation = database.prepare("INSERT OR IGNORE INTO stage_relations VALUES (?, ?, ?)");
599+ const line = database.prepare("INSERT INTO stage_lines VALUES (?, ?, ?, ?, ?, ?)");
600+ const arretLigne = database.prepare("INSERT OR IGNORE INTO stage_arrets_lignes VALUES (?, ?)");
601+
602+ return {
603+ routes: {
604+ required: ["route_id", "route_short_name", "route_long_name"],
605+ insert: (row) => route.run(canonicalLineId(row.route_id, "routes.route_id"), optionalText(row.route_short_name), optionalText(row.route_long_name)),
606+ },
607+ trips: {
608+ required: ["route_id", "trip_id", "trip_headsign", "direction_id"],
609+ insert: (row) => trip.run(nonEmpty(row.trip_id, "trips.trip_id"), nonEmpty(row.route_id, "trips.route_id"), optionalText(row.trip_headsign), nonEmpty(row.direction_id, "trips.direction_id")),
610+ },
611+ stopTimes: {
612+ required: ["trip_id", "stop_id", "stop_sequence", "pickup_type"],
613+ insert: (row) => {
614+ const pickupType = row.pickup_type === "" ? 0 : integer(row.pickup_type, "stop_times.pickup_type");
615+ if (pickupType < 0 || pickupType > 3) fail("INVALID_SOURCE", "stop_times.pickup_type is outside GTFS range");
616+ stopTime.run(nonEmpty(row.trip_id, "stop_times.trip_id"), nonEmpty(row.stop_id, "stop_times.stop_id"), integer(row.stop_sequence, "stop_times.stop_sequence"), pickupType);
617+ },
618+ },
619+ stops: {
620+ required: ["stop_id", "stop_name", "parent_station"],
621+ insert: (row) => stop.run(nonEmpty(row.stop_id, "stops.stop_id"), nonEmpty(row.stop_name, "stops.stop_name"), optionalText(row.parent_station)),
622+ },
623+ agency: {
624+ required: ["agency_name", "agency_url", "agency_timezone"],
625+ insert: (row) => {
626+ nonEmpty(row.agency_url, "agency.agency_url");
627+ nonEmpty(row.agency_timezone, "agency.agency_timezone");
628+ agency.run(nonEmpty(row.agency_name, "agency.agency_name"));
629+ },
630+ },
631+ objectCodes: {
632+ required: ["object_type", "object_id", "object_system", "object_code"],
633+ insert: (row) => code.run(nonEmpty(row.object_type, "codes.object_type"), nonEmpty(row.object_id, "codes.object_id"), nonEmpty(row.object_system, "codes.object_system"), nonEmpty(row.object_code, "codes.object_code")),
634+ },
635+ perimeter: {
636+ required: ["line", "name_line", "ns2_stoppointref", "ns2_stopname", "operatorname"],
637+ insert: (row) => perimeter.run(nonEmpty(row.line, "perimeter.line"), canonicalLineId(row.line, "perimeter.line"), nonEmpty(row.name_line, "perimeter.name_line"), nonEmpty(row.ns2_stoppointref, "perimeter.ns2_stoppointref"), nonEmpty(row.ns2_stopname, "perimeter.ns2_stopname"), nonEmpty(row.operatorname, "perimeter.operatorname")),
638+ },
639+ arrets: {
640+ required: ["arrid", "zdaid"],
641+ insert: (row) => arret.run(nonEmpty(row.arrid, "arrets.arrid"), nonEmpty(row.zdaid, "arrets.zdaid")),
642+ },
643+ zones: {
644+ required: ["zdaid", "zdaname", "zdatown"],
645+ insert: (row) => zone.run(nonEmpty(row.zdaid, "zones.zdaid"), nonEmpty(row.zdaname, "zones.zdaname"), optionalText(row.zdatown)),
646+ },
647+ relations: {
648+ required: ["zdaid", "arrid", "artid"],
649+ insert: (row) => relation.run(nonEmpty(row.zdaid, "relations.zdaid"), nonEmpty(row.arrid, "relations.arrid"), nonEmpty(row.artid, "relations.artid")),
650+ },
651+ lines: {
652+ required: ["id_line", "name_line", "shortname_line", "transportmode", "transportsubmode", "status"],
653+ insert: (row) => line.run(canonicalLineId(row.id_line, "lines.id_line"), nonEmpty(row.name_line, "lines.name_line"), optionalText(row.shortname_line), nonEmpty(row.transportmode, "lines.transportmode"), optionalText(row.transportsubmode), nonEmpty(row.status, "lines.status")),
654+ },
655+ arretsLignes: {
656+ required: ["id", "stop_id"],
657+ insert: (row) => arretLigne.run(canonicalLineId(row.id, "arrets_lignes.id"), nonEmpty(row.stop_id, "arrets_lignes.stop_id")),
658+ },
659+ };
660+}
661+
662+function runBatched(database: DatabaseSync, rows: Iterable<Readonly<Record<string, string>>>, spec: CsvSpec): void {
663+ let pending = 0;
664+ database.exec("BEGIN IMMEDIATE");
665+ try {
666+ for (const row of rows) {
667+ spec.insert(row);
668+ pending += 1;
669+ if (pending === IMPORT_TRANSACTION_ROWS) {
670+ database.exec("COMMIT; BEGIN IMMEDIATE");
671+ pending = 0;
672+ }
673+ }
674+ database.exec("COMMIT");
675+ } catch (error) {
676+ database.exec("ROLLBACK");
677+ throw error;
678+ }
679+}
680+
681+function stringRecord(record: CsvRecord, required: readonly string[]): Readonly<Record<string, string>> {
682+ const result: Record<string, string> = {};
683+ for (const key of required) {
684+ if (!Object.hasOwn(record, key)) fail("MISSING_COLUMN", `record is missing ${key}`);
685+ result[key] = String(record[key] ?? "");
686+ }
687+ return result;
688+}
689+
690+function loadRecordSources(database: DatabaseSync, sources: CatalogRecordSourceSet): void {
691+ const specs = prepareSpecs(database);
692+ for (const key of Object.keys(specs) as (keyof CatalogRecordSourceSet)[]) {
693+ const spec = specs[key];
694+ const rows = sources[key].map((row) => stringRecord(row, spec.required));
695+ runBatched(database, rows, spec);
696+ }
697+}
698+
699+class Utf8Decoder extends Transform {
700+ readonly #decoder = new TextDecoder("utf-8", { fatal: true });
701+
702+ override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
703+ try {
704+ this.push(this.#decoder.decode(chunk, { stream: true }));
705+ callback();
706+ } catch (error) {
707+ callback(error as Error);
708+ }
709+ }
710+
711+ override _flush(callback: (error?: Error | null) => void): void {
712+ try {
713+ this.push(this.#decoder.decode());
714+ callback();
715+ } catch (error) {
716+ callback(error as Error);
717+ }
718+ }
719+}
720+
721+class ByteLimit extends Transform {
722+ #bytes = 0;
723+ readonly #limit: number;
724+ readonly #onBytes: (bytes: number) => void;
725+
726+ constructor(limit: number, onBytes: (bytes: number) => void) {
727+ super();
728+ this.#limit = limit;
729+ this.#onBytes = onBytes;
730+ }
731+
732+ override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
733+ this.#bytes += chunk.byteLength;
734+ if (this.#bytes > this.#limit) {
735+ callback(new CatalogImportError("ZIP_ENTRY_TOO_LARGE", "selected GTFS entry exceeds the 8 GiB limit"));
736+ return;
737+ }
738+ try {
739+ this.#onBytes(chunk.byteLength);
740+ this.push(chunk);
741+ callback();
742+ } catch (error) {
743+ callback(error as Error);
744+ }
745+ }
746+}
747+
748+async function loadCsvStream(
749+ database: DatabaseSync,
750+ input: Readable,
751+ delimiter: "," | ";",
752+ spec: CsvSpec,
753+): Promise<number> {
754+ const decoder = new Utf8Decoder();
755+ const parser = parse({
756+ bom: true,
757+ columns: false,
758+ delimiter,
759+ max_record_size: MAX_CSV_RECORD_BYTES,
760+ relax_column_count: false,
761+ relax_quotes: false,
762+ skip_empty_lines: true,
763+ });
764+ const destroyParser = (error: Error): void => parser.destroy(error);
765+ input.once("error", destroyParser);
766+ decoder.once("error", destroyParser);
767+ input.pipe(decoder).pipe(parser);
768+
769+ let indexes: Readonly<Record<string, number>> | undefined;
770+ let pending = 0;
771+ let rows = 0;
772+ database.exec("BEGIN IMMEDIATE");
773+ try {
774+ for await (const value of parser) {
775+ if (!Array.isArray(value) || !value.every((field) => typeof field === "string")) {
776+ fail("INVALID_CSV", "CSV parser returned a non-string record");
777+ }
778+ const fields = value as string[];
779+ if (indexes === undefined) {
780+ const header = fields.map((field) => field.trim());
781+ if (header.length === 0 || header.some((field) => field.length === 0)) fail("INVALID_HEADER", "CSV header is empty");
782+ if (new Set(header).size !== header.length) fail("DUPLICATE_HEADER", "CSV contains duplicate columns");
783+ const mapped: Record<string, number> = {};
784+ for (const required of spec.required) {
785+ const index = header.indexOf(required);
786+ if (index === -1) fail("MISSING_COLUMN", `CSV is missing required column ${required}`);
787+ mapped[required] = index;
788+ }
789+ indexes = mapped;
790+ continue;
791+ }
792+ const row: Record<string, string> = {};
793+ for (const [key, index] of Object.entries(indexes)) row[key] = fields[index] ?? "";
794+ spec.insert(row);
795+ rows += 1;
796+ pending += 1;
797+ if (pending === IMPORT_TRANSACTION_ROWS) {
798+ database.exec("COMMIT; BEGIN IMMEDIATE");
799+ pending = 0;
800+ }
801+ }
802+ if (indexes === undefined) fail("INVALID_HEADER", "CSV source is empty");
803+ database.exec("COMMIT");
804+ return rows;
805+ } catch (error) {
806+ input.destroy();
807+ database.exec("ROLLBACK");
808+ if (error instanceof CatalogImportError) throw error;
809+ fail("CSV_PARSE_FAILED", "CSV source could not be parsed", error);
810+ }
811+}
812+
813+function assertSourceFile(path: string): void {
814+ const size = statSync(path).size;
815+ if (size > MAX_COMPRESSED_DOWNLOAD_BYTES) fail("SOURCE_TOO_LARGE", `${basename(path)} exceeds the 2 GiB source limit`);
816+}
817+
818+function openZip(path: string): Promise<ZipFileLike> {
819+ const { promise, resolve: resolveZip, reject } = Promise.withResolvers<ZipFileLike>();
820+ yauzl.open(path, {
821+ autoClose: false,
822+ decodeStrings: true,
823+ lazyEntries: true,
824+ strictFileNames: true,
825+ validateEntrySizes: true,
826+ }, (error: Error | null, zipFile?: ZipFileLike) => {
827+ if (error || zipFile === undefined) reject(error ?? new Error("ZIP did not open"));
828+ else resolveZip(zipFile);
829+ });
830+ return promise;
831+}
832+
833+function nextZipEntry(zip: ZipFileLike): Promise<ZipEntryLike | undefined> {
834+ const { promise, resolve: resolveEntry, reject } = Promise.withResolvers<ZipEntryLike | undefined>();
835+ const cleanup = (): void => {
836+ zip.removeListener("entry", onEntry as (...args: never[]) => void);
837+ zip.removeListener("end", onEnd as (...args: never[]) => void);
838+ zip.removeListener("error", onError as (...args: never[]) => void);
839+ };
840+ const onEntry = (entry: ZipEntryLike): void => {
841+ cleanup();
842+ resolveEntry(entry);
843+ };
844+ const onEnd = (): void => {
845+ cleanup();
846+ resolveEntry(undefined);
847+ };
848+ const onError = (error: unknown): void => {
849+ cleanup();
850+ reject(error);
851+ };
852+ zip.once("entry", onEntry);
853+ zip.once("end", onEnd);
854+ zip.once("error", onError);
855+ zip.readEntry();
856+ return promise;
857+}
858+
859+function openZipEntry(zip: ZipFileLike, entry: ZipEntryLike): Promise<Readable> {
860+ const { promise, resolve: resolveStream, reject } = Promise.withResolvers<Readable>();
861+ zip.openReadStream(entry, (error, stream) => {
862+ if (error || stream === undefined) reject(error ?? new Error("ZIP entry did not open"));
863+ else resolveStream(stream);
864+ });
865+ return promise;
866+}
867+
868+function gtfsKey(name: string): keyof CatalogRecordSourceSet | undefined {
869+ switch (name) {
870+ case "routes.txt": return "routes";
871+ case "trips.txt": return "trips";
872+ case "stop_times.txt": return "stopTimes";
873+ case "stops.txt": return "stops";
874+ case "agency.txt": return "agency";
875+ case "object_codes_extension.txt":
876+ case "objects_codes_extension.txt": return "objectCodes";
877+ default: return undefined;
878+ }
879+}
880+
881+async function loadGtfsZip(database: DatabaseSync, path: string, specs: Readonly<Record<keyof CatalogRecordSourceSet, CsvSpec>>): Promise<void> {
882+ assertSourceFile(path);
883+ let zip: ZipFileLike | undefined;
884+ const seen = new Set<string>();
885+ let selectedBytes = 0;
886+ try {
887+ zip = await openZip(path);
888+ while (true) {
889+ const entry = await nextZipEntry(zip);
890+ if (entry === undefined) break;
891+ const normalizedName = entry.fileName.replaceAll("\\", "/").split("/").at(-1)!.toLowerCase();
892+ const key = gtfsKey(normalizedName);
893+ if (key === undefined) continue;
894+ if ((entry.generalPurposeBitFlag & 1) !== 0) fail("ENCRYPTED_ZIP_ENTRY", `selected GTFS entry ${normalizedName} is encrypted`);
895+ const conceptualName = Object.hasOwn(CODE_ENTRY_NAME, normalizedName) ? "codes-extension" : normalizedName;
896+ if (seen.has(conceptualName)) fail("DUPLICATE_ZIP_ENTRY", `GTFS contains duplicate required entry ${normalizedName}`);
897+ seen.add(conceptualName);
898+ if (!Number.isSafeInteger(entry.uncompressedSize) || entry.uncompressedSize < 0 || entry.uncompressedSize > MAX_SELECTED_ENTRY_BYTES) {
899+ fail("ZIP_ENTRY_TOO_LARGE", `selected GTFS entry ${normalizedName} exceeds its byte limit`);
900+ }
901+ const stream = await openZipEntry(zip, entry);
902+ const limiter = new ByteLimit(MAX_SELECTED_ENTRY_BYTES, (bytes) => {
903+ selectedBytes += bytes;
904+ if (selectedBytes > MAX_SELECTED_ARCHIVE_BYTES) {
905+ fail("ZIP_AGGREGATE_TOO_LARGE", "selected GTFS entries exceed the 16 GiB aggregate limit");
906+ }
907+ });
908+ stream.once("error", (error) => limiter.destroy(error));
909+ await loadCsvStream(database, stream.pipe(limiter), ",", specs[key]);
910+ }
911+ for (const name of REQUIRED_GTFS_ENTRIES) {
912+ if (!seen.has(name)) fail("MISSING_ZIP_ENTRY", `GTFS is missing required entry ${name}`);
913+ }
914+ if (!seen.has("codes-extension")) fail("MISSING_ZIP_ENTRY", "GTFS is missing its codes-extension entry");
915+ } catch (error) {
916+ if (error instanceof CatalogImportError) throw error;
917+ fail("GTFS_ZIP_FAILED", "GTFS ZIP could not be imported", error);
918+ } finally {
919+ zip?.close();
920+ }
921+}
922+
923+async function loadFileSources(database: DatabaseSync, sources: CatalogFileSourceSet): Promise<void> {
924+ const specs = prepareSpecs(database);
925+ await loadGtfsZip(database, sources.gtfsZipPath, specs);
926+ const csvSources: readonly [keyof CatalogRecordSourceSet, CsvFileSource][] = [
927+ ["perimeter", sources.perimeter],
928+ ["arrets", sources.arrets],
929+ ["zones", sources.zones],
930+ ["relations", sources.relations],
931+ ["lines", sources.lines],
932+ ["arretsLignes", sources.arretsLignes],
933+ ];
934+ for (const [key, source] of csvSources) {
935+ assertSourceFile(source.path);
936+ const rows = await loadCsvStream(database, createReadStream(source.path), source.delimiter ?? ";", specs[key]);
937+ if (source.expectedRows !== undefined && rows !== source.expectedRows) {
938+ fail("SOURCE_ROW_COUNT", `${key} row count ${rows} does not match metadata count ${source.expectedRows}`);
939+ }
940+ }
941+}
942+
943+function createIndexes(database: DatabaseSync): void {
944+ database.exec(`
945+ CREATE INDEX stage_trips_route ON stage_trips(route_id);
946+ CREATE INDEX stage_stop_times_trip ON stage_stop_times(trip_id, stop_sequence);
947+ CREATE INDEX stage_stop_times_stop ON stage_stop_times(stop_id);
948+ CREATE INDEX stage_codes_object ON stage_codes(object_id, object_code);
949+ CREATE INDEX stage_codes_code ON stage_codes(object_code, object_id);
950+ CREATE INDEX stage_perimeter_line ON stage_perimeter(id_line, monitoring_ref);
951+ CREATE INDEX stage_arrets_zda ON stage_arrets(zdaid);
952+ CREATE INDEX stage_relations_art ON stage_relations(artid, zdaid);
953+ CREATE INDEX stage_arrets_lignes_stop ON stage_arrets_lignes(stop_id, route_id);
954+ `);
955+}
956+
957+function deriveCandidates(database: DatabaseSync): void {
958+ database.exec(`
959+ CREATE TABLE stage_line_modes (
960+ id_line TEXT PRIMARY KEY,
961+ mode TEXT NOT NULL,
962+ line_label TEXT NOT NULL
963+ ) STRICT;
964+
965+ INSERT INTO stage_line_modes
966+ SELECT id_line,
967+ CASE
968+ WHEN lower(trim(transport_mode)) = 'bus' THEN 'BUS'
969+ WHEN lower(trim(transport_mode)) = 'metro' THEN 'METRO'
970+ WHEN lower(trim(transport_mode)) = 'tram' THEN 'TRAM'
971+ WHEN lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) = 'local' THEN 'RER'
972+ WHEN lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) = 'suburbanrailway' THEN 'TRANSILIEN'
973+ END,
974+ coalesce(nullif(trim(shortname_line), ''), trim(name_line))
975+ FROM stage_lines
976+ WHERE lower(trim(status)) = 'active'
977+ AND (
978+ lower(trim(transport_mode)) IN ('bus', 'metro', 'tram')
979+ OR (lower(trim(transport_mode)) = 'rail' AND lower(trim(transport_submode)) IN ('local', 'suburbanrailway'))
980+ );
981+
982+ CREATE TABLE stage_stop_zda (
983+ stop_id TEXT PRIMARY KEY,
984+ zdaid TEXT,
985+ rail_zdaid TEXT NOT NULL,
986+ resolution TEXT NOT NULL
987+ ) STRICT;
988+
989+ INSERT INTO stage_stop_zda (stop_id, zdaid, rail_zdaid, resolution)
990+ SELECT
991+ stop.stop_id,
992+ coalesce(area.zdaid, relation.zdaid),
993+ CASE
994+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:monomodalStopPlace:%'
995+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), length('IDFM:monomodalStopPlace:') + 1)
996+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:%'
997+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), 6)
998+ ELSE coalesce(nullif(stop.parent_station, ''), stop.stop_id)
999+ END,
1000+ CASE WHEN area.arrid IS NULL THEN 'relations' ELSE 'arrets' END
1001+ FROM stage_stops AS stop
1002+ LEFT JOIN stage_arrets AS area
1003+ ON area.arrid = CASE
1004+ WHEN coalesce(nullif(stop.parent_station, ''), stop.stop_id) LIKE 'IDFM:%'
1005+ THEN substr(coalesce(nullif(stop.parent_station, ''), stop.stop_id), 6)
1006+ ELSE coalesce(nullif(stop.parent_station, ''), stop.stop_id)
1007+ END
1008+ LEFT JOIN (
1009+ SELECT artid, min(zdaid) AS zdaid
1010+ FROM stage_relations
1011+ GROUP BY artid
1012+ ) AS relation
1013+ ON relation.artid = CASE
1014+ WHEN stop.stop_id LIKE 'IDFM:%' THEN substr(stop.stop_id, 6)
1015+ ELSE stop.stop_id
1016+ END;
1017+
1018+ CREATE TABLE stage_terminals AS
1019+ SELECT stop_time.trip_id, stop_time.stop_id, stop.stop_name
1020+ FROM stage_stop_times AS stop_time
1021+ JOIN stage_stops AS stop ON stop.stop_id = stop_time.stop_id
1022+ JOIN (
1023+ SELECT trip_id, max(stop_sequence) AS stop_sequence
1024+ FROM stage_stop_times
1025+ GROUP BY trip_id
1026+ ) AS terminal
1027+ ON terminal.trip_id = stop_time.trip_id
1028+ AND terminal.stop_sequence = stop_time.stop_sequence;
1029+ CREATE UNIQUE INDEX stage_terminals_trip ON stage_terminals(trip_id);
1030+
1031+ CREATE TABLE stage_candidates AS
1032+ WITH joined AS (
1033+ SELECT DISTINCT
1034+ line_mode.mode AS mode,
1035+ CASE
1036+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN stop_zda.rail_zdaid
1037+ ELSE stop_zda.zdaid
1038+ END AS zdaid,
1039+ zone.zdaname AS stop_label,
1040+ zone.zdatown AS locality_label,
1041+ line_mode.id_line AS id_line,
1042+ line_mode.line_label AS line_label,
1043+ CASE
1044+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN terminal.stop_name
1045+ ELSE coalesce(nullif(trim(trip.trip_headsign), ''), terminal.stop_name)
1046+ END AS destination_label,
1047+ perimeter.monitoring_ref AS monitoring_ref,
1048+ perimeter.line AS line_ref,
1049+ trip.direction_id AS direction_id,
1050+ destination_code.object_code AS destination_ref,
1051+ stop_zda.resolution AS resolution
1052+ FROM stage_stop_times AS current_stop_time
1053+ JOIN stage_trips AS trip ON trip.trip_id = current_stop_time.trip_id
1054+ JOIN stage_routes AS route ON route.route_id = CASE WHEN trip.route_id LIKE 'IDFM:%' THEN substr(trip.route_id, 6) ELSE trip.route_id END
1055+ JOIN stage_line_modes AS line_mode ON line_mode.id_line = route.route_id
1056+ JOIN stage_stops AS current_stop ON current_stop.stop_id = current_stop_time.stop_id
1057+ JOIN stage_stop_zda AS stop_zda ON stop_zda.stop_id = current_stop.stop_id
1058+ JOIN stage_terminals AS terminal
1059+ ON terminal.trip_id = current_stop_time.trip_id
1060+ JOIN stage_codes AS destination_code
1061+ ON destination_code.object_id = terminal.stop_id
1062+ AND lower(trim(destination_code.object_system)) = 'source'
1063+ JOIN stage_arrets_lignes AS attached
1064+ ON attached.route_id = route.route_id
1065+ AND attached.stop_id IN (current_stop.stop_id, current_stop.parent_station)
1066+ JOIN stage_perimeter AS perimeter
1067+ ON perimeter.id_line = line_mode.id_line
1068+ AND (
1069+ EXISTS (
1070+ SELECT 1
1071+ FROM stage_codes AS monitoring_code
1072+ WHERE monitoring_code.object_id IN (current_stop.stop_id, current_stop.parent_station)
1073+ AND monitoring_code.object_code = perimeter.monitoring_ref
1074+ )
1075+ OR instr(perimeter.monitoring_ref, ':' || stop_zda.rail_zdaid || ':') > 0
1076+ )
1077+ JOIN stage_zones AS zone
1078+ ON zone.zdaid = CASE
1079+ WHEN line_mode.mode IN ('RER', 'TRANSILIEN') THEN stop_zda.rail_zdaid
1080+ ELSE stop_zda.zdaid
1081+ END
1082+ WHERE current_stop_time.pickup_type <> 1
1083+ AND EXISTS (
1084+ SELECT 1 FROM stage_stop_times AS later
1085+ WHERE later.trip_id = current_stop_time.trip_id
1086+ AND later.stop_sequence > current_stop_time.stop_sequence
1087+ )
1088+ AND trim(trip.direction_id) <> ''
1089+ AND trim(destination_code.object_code) <> ''
1090+ )
1091+ SELECT mode, zdaid, min(stop_label) AS stop_label, min(locality_label) AS locality_label,
1092+ id_line, min(line_label) AS line_label, min(destination_label) AS destination_label,
1093+ monitoring_ref, line_ref, direction_id, destination_ref,
1094+ max(CASE WHEN resolution = 'arrets' THEN 1 ELSE 0 END) AS via_arrets
1095+ FROM joined
1096+ WHERE zdaid IS NOT NULL AND trim(zdaid) <> ''
1097+ GROUP BY mode, zdaid, monitoring_ref, id_line, line_ref, direction_id, destination_ref;
1098+ `);
1099+}
1100+
1101+function insertFinalCatalog(database: DatabaseSync, sourceRevision: string, createdAt: string): PlaceResolutionCounts {
1102+ const metadata = database.prepare("INSERT INTO metadata VALUES (?, ?)");
1103+ metadata.run("schema_version", String(SCHEMA_VERSION));
1104+ metadata.run("catalog_version", String(CATALOG_VERSION));
1105+ metadata.run("source_revision", sourceRevision);
1106+ metadata.run("created_at", createdAt);
1107+
1108+ const insertPlace = database.prepare("INSERT INTO places VALUES (?, ?, ?, ?, ?)");
1109+ const insertSearch = database.prepare("INSERT INTO place_search(search_text, place_id) VALUES (?, ?)");
1110+ const insertService = database.prepare("INSERT INTO services VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
1111+ const selectPlace = database.prepare("SELECT canonical_tuple FROM places WHERE place_id = ?");
1112+ const selectService = database.prepare("SELECT place_id, canonical_tuple FROM services WHERE service_id = ?");
1113+
1114+ const places = database.prepare(`
1115+ SELECT mode, zdaid, min(stop_label) AS stop_label, min(locality_label) AS locality_label,
1116+ max(via_arrets) AS via_arrets
1117+ FROM stage_candidates
1118+ GROUP BY mode, zdaid
1119+ ORDER BY mode, zdaid
1120+ `).all() as QueryRow[];
1121+
1122+ const placeResolution: PlaceResolutionCounts = { arrets: 0, relations: 0, direct: 0 };
1123+
1124+ database.exec("BEGIN IMMEDIATE");
1125+ try {
1126+ for (const row of places) {
1127+ const mode = transportMode(row.mode);
1128+ const zdaId = nonEmpty(row.zdaid, "candidate.zdaid");
1129+ const stopLabel = nonEmpty(row.stop_label, "candidate.stop_label");
1130+ const localityLabel = optionalText(row.locality_label);
1131+ const identity = createPlaceIdentity(mode, zdaId);
1132+ const collision = selectPlace.get(identity.placeId) as QueryRow | undefined;
1133+ if (collision !== undefined && collision.canonical_tuple !== identity.canonicalTuple) {
1134+ fail("HASH_COLLISION", "place ID collision detected");
1135+ }
1136+ if (collision === undefined) {
1137+ const searchText = normalizeCatalogSearchText(`${stopLabel} ${localityLabel}`);
1138+ if (searchText.length === 0) fail("CATALOG_SEARCH", "place label produces empty search text");
1139+ insertPlace.run(identity.placeId, stopLabel, localityLabel || null, mode, identity.canonicalTuple);
1140+ insertSearch.run(searchText, identity.placeId);
1141+ }
1142+ if (mode === "RER" || mode === "TRANSILIEN") placeResolution.direct += 1;
1143+ else if (Number(row.via_arrets) === 1) placeResolution.arrets += 1;
1144+ else placeResolution.relations += 1;
1145+ }
1146+
1147+ const candidates = database.prepare(`
1148+ SELECT mode, zdaid, stop_label, line_label, destination_label,
1149+ id_line, monitoring_ref, line_ref, direction_id, destination_ref
1150+ FROM stage_candidates
1151+ ORDER BY mode, zdaid, id_line, monitoring_ref, direction_id, destination_ref
1152+ `).iterate() as Iterable<QueryRow>;
1153+ for (const row of candidates) {
1154+ const mode = transportMode(row.mode);
1155+ const place = createPlaceIdentity(mode, nonEmpty(row.zdaid, "candidate.zdaid"));
1156+ const service = createServiceIdentity({
1157+ mode,
1158+ lineId: nonEmpty(row.id_line, "candidate.id_line"),
1159+ monitoringRef: nonEmpty(row.monitoring_ref, "candidate.monitoring_ref"),
1160+ directionId: nonEmpty(row.direction_id, "candidate.direction_id"),
1161+ destinationRef: nonEmpty(row.destination_ref, "candidate.destination_ref"),
1162+ });
1163+ const collision = selectService.get(service.serviceId) as QueryRow | undefined;
1164+ if (collision !== undefined && collision.canonical_tuple !== service.canonicalTuple) {
1165+ fail("HASH_COLLISION", "service ID collision detected");
1166+ }
1167+ if (collision !== undefined && collision.place_id !== place.placeId) {
1168+ fail("INVALID_SERVICE_PLACE", "one semantic service resolved to multiple places");
1169+ }
1170+ if (collision === undefined) {
1171+ insertService.run(
1172+ service.serviceId,
1173+ place.placeId,
1174+ nonEmpty(row.stop_label, "candidate.stop_label"),
1175+ nonEmpty(row.line_label, "candidate.line_label"),
1176+ nonEmpty(row.destination_label, "candidate.destination_label"),
1177+ nonEmpty(row.monitoring_ref, "candidate.monitoring_ref"),
1178+ nonEmpty(row.line_ref, "candidate.line_ref"),
1179+ nonEmpty(row.direction_id, "candidate.direction_id"),
1180+ nonEmpty(row.destination_ref, "candidate.destination_ref"),
1181+ service.canonicalTuple,
1182+ );
1183+ }
1184+ }
1185+ database.exec("COMMIT");
1186+ } catch (error) {
1187+ database.exec("ROLLBACK");
1188+ throw error;
1189+ }
1190+ return placeResolution;
1191+}
1192+
1193+function validateByteBounds(database: DatabaseSync): void {
1194+ const invalidPlace = database.prepare(`
1195+ SELECT place_id FROM places
1196+ WHERE length(CAST(place_id AS BLOB)) > ?
1197+ OR length(CAST(stop_label AS BLOB)) > ?
1198+ OR (locality_label IS NOT NULL AND length(CAST(locality_label AS BLOB)) > ?)
1199+ LIMIT 1
1200+ `).get(LIMITS.idUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes);
1201+ if (invalidPlace !== undefined) fail("CATALOG_LIMIT", "place exceeds a shared UTF-8 byte limit");
1202+
1203+ const invalidService = database.prepare(`
1204+ SELECT service_id FROM services
1205+ WHERE length(CAST(service_id AS BLOB)) > ?
1206+ OR length(CAST(stop_label AS BLOB)) > ?
1207+ OR length(CAST(line_label AS BLOB)) > ?
1208+ OR length(CAST(destination_label AS BLOB)) > ?
1209+ LIMIT 1
1210+ `).get(LIMITS.idUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes, LIMITS.labelUtf8Bytes);
1211+ if (invalidService !== undefined) fail("CATALOG_LIMIT", "service exceeds a shared UTF-8 byte limit");
1212+}
1213+
1214+function exactColumns(database: DatabaseSync, table: string, expected: readonly string[]): void {
1215+ const rows = database.prepare(`PRAGMA table_info(${table})`).all() as QueryRow[];
1216+ const actual = rows.map((row) => String(row.name));
1217+ if (actual.length !== expected.length || actual.some((name, index) => name !== expected[index])) {
1218+ fail("CATALOG_SCHEMA", `${table} has an unexpected column contract`);
1219+ }
1220+}
1221+
1222+function metadataValue(database: DatabaseSync, key: string): string | undefined {
1223+ const row = database.prepare("SELECT value FROM metadata WHERE key = ?").get(key) as QueryRow | undefined;
1224+ return row === undefined ? undefined : String(row.value);
1225+}
1226+
1227+function validateOpenCatalog(database: DatabaseSync): CatalogBuildResult {
1228+ database.exec("PRAGMA foreign_keys = ON");
1229+ exactColumns(database, "metadata", ["key", "value"]);
1230+ exactColumns(database, "places", ["place_id", "stop_label", "locality_label", "mode", "canonical_tuple"]);
1231+ exactColumns(database, "services", ["service_id", "place_id", "stop_label", "line_label", "destination_label", "monitoring_ref", "line_ref", "direction_id", "destination_ref", "canonical_tuple"]);
1232+ exactColumns(database, "place_search", ["search_text", "place_id"]);
1233+
1234+ const sourceRevision = metadataValue(database, "source_revision");
1235+ const createdAt = metadataValue(database, "created_at");
1236+ if (metadataValue(database, "schema_version") !== String(SCHEMA_VERSION)
1237+ || metadataValue(database, "catalog_version") !== String(CATALOG_VERSION)
1238+ || sourceRevision === undefined || sourceRevision.trim() === ""
1239+ || createdAt === undefined) {
1240+ fail("CATALOG_METADATA", "catalog metadata is incomplete or incompatible");
1241+ }
1242+ isoTimestamp(createdAt, "metadata.created_at");
1243+
1244+ const integrity = database.prepare("PRAGMA integrity_check").all() as QueryRow[];
1245+ if (integrity.length !== 1 || String(integrity[0]?.integrity_check) !== "ok") fail("CATALOG_INTEGRITY", "SQLite integrity_check failed");
1246+ if ((database.prepare("PRAGMA foreign_key_check").all() as QueryRow[]).length !== 0) fail("CATALOG_FOREIGN_KEY", "SQLite foreign_key_check failed");
1247+ validateByteBounds(database);
1248+
1249+ const missingSearch = database.prepare(`
1250+ SELECT 1 FROM places AS place
1251+ LEFT JOIN place_search AS search ON search.place_id = place.place_id
1252+ WHERE search.place_id IS NULL
1253+ LIMIT 1
1254+ `).get();
1255+ const extraSearch = database.prepare(`
1256+ SELECT 1 FROM place_search AS search
1257+ LEFT JOIN places AS place ON place.place_id = search.place_id
1258+ WHERE place.place_id IS NULL
1259+ LIMIT 1
1260+ `).get();
1261+ if (missingSearch !== undefined || extraSearch !== undefined) fail("CATALOG_SEARCH", "FTS place index does not match places");
1262+ const invalidPlaceService = database.prepare(`
1263+ SELECT 1
1264+ FROM places AS place
1265+ LEFT JOIN services AS service ON service.place_id = place.place_id
1266+ WHERE service.service_id IS NULL OR service.stop_label <> place.stop_label
1267+ LIMIT 1
1268+ `).get();
1269+ if (invalidPlaceService !== undefined) {
1270+ fail("CATALOG_PLACE_SERVICE", "catalog place is empty or has inconsistent service labels");
1271+ }
1272+ if (database.prepare("SELECT 1 FROM place_search WHERE trim(search_text) = '' LIMIT 1").get() !== undefined) {
1273+ fail("CATALOG_SEARCH", "FTS place index contains empty search text");
1274+ }
1275+
1276+ const unresolved = database.prepare(`
1277+ SELECT 1 FROM services
1278+ WHERE trim(monitoring_ref) = '' OR trim(line_ref) = '' OR trim(direction_id) = '' OR trim(destination_ref) = ''
1279+ LIMIT 1
1280+ `).get();
1281+ if (unresolved !== undefined) fail("CATALOG_UNRESOLVED", "catalog contains an unresolvable offered service");
1282+
1283+ for (const row of database.prepare("SELECT place_id, canonical_tuple FROM places").iterate() as Iterable<QueryRow>) {
1284+ if (opaqueId("plc_", String(row.canonical_tuple)) !== row.place_id) fail("HASH_COLLISION", "place ID does not match its canonical tuple");
1285+ }
1286+ for (const row of database.prepare("SELECT service_id, canonical_tuple FROM services").iterate() as Iterable<QueryRow>) {
1287+ if (opaqueId("svc_", String(row.canonical_tuple)) !== row.service_id) fail("HASH_COLLISION", "service ID does not match its canonical tuple");
1288+ }
1289+
1290+ const counts = Object.fromEntries(TRANSPORT_MODE.map((mode) => [mode, 0])) as Record<TransportMode, number>;
1291+ for (const row of database.prepare(`
1292+ SELECT place.mode AS mode, count(service.service_id) AS service_count
1293+ FROM places AS place
1294+ JOIN services AS service ON service.place_id = place.place_id
1295+ GROUP BY place.mode
1296+ `).all() as QueryRow[]) {
1297+ const mode = transportMode(row.mode);
1298+ counts[mode] = Number(row.service_count);
1299+ }
1300+ for (const mode of TRANSPORT_MODE) {
1301+ if (counts[mode] < 1) fail("MODE_COVERAGE", `catalog has no resolvable ${mode} service`);
1302+ }
1303+
1304+ const placeCount = Number((database.prepare("SELECT count(*) AS count FROM places").get() as QueryRow).count);
1305+ const serviceCount = Number((database.prepare("SELECT count(*) AS count FROM services").get() as QueryRow).count);
1306+ return { candidatePath: "", sourceRevision, createdAt, placeCount, serviceCount, countsByMode: counts };
1307+}
1308+
1309+function validateStagedPerimeter(database: DatabaseSync): void {
1310+ const missing = database.prepare(`
1311+ SELECT 1 FROM stage_candidates AS candidate
1312+ WHERE NOT EXISTS (
1313+ SELECT 1 FROM stage_perimeter AS perimeter
1314+ WHERE perimeter.line = candidate.line_ref
1315+ AND perimeter.monitoring_ref = candidate.monitoring_ref
1316+ )
1317+ LIMIT 1
1318+ `).get();
1319+ if (missing !== undefined) fail("PERIMETER_MISMATCH", "offered service lacks its exact perimeter pair");
1320+}
1321+
1322+function validateCandidateSemanticTuples(database: DatabaseSync): void {
1323+ const ambiguous = database.prepare(`
1324+ SELECT 1
1325+ FROM stage_candidates
1326+ GROUP BY mode, id_line, monitoring_ref, direction_id, destination_ref
1327+ HAVING count(*) <> 1 OR count(DISTINCT zdaid) <> 1
1328+ LIMIT 1
1329+ `).get();
1330+ if (ambiguous !== undefined) {
1331+ fail("AMBIGUOUS_SERVICE", "a semantic service tuple resolves to multiple candidate rows or places");
1332+ }
1333+}
1334+
1335+function dropStaging(database: DatabaseSync): void {
1336+ database.exec(`
1337+ DROP TABLE stage_candidates;
1338+ DROP TABLE stage_terminals;
1339+ DROP TABLE stage_stop_zda;
1340+ DROP TABLE stage_line_modes;
1341+ DROP TABLE stage_arrets_lignes;
1342+ DROP TABLE stage_lines;
1343+ DROP TABLE stage_relations;
1344+ DROP TABLE stage_zones;
1345+ DROP TABLE stage_arrets;
1346+ DROP TABLE stage_perimeter;
1347+ DROP TABLE stage_codes;
1348+ DROP TABLE stage_agency;
1349+ DROP TABLE stage_stops;
1350+ DROP TABLE stage_stop_times;
1351+ DROP TABLE stage_trips;
1352+ DROP TABLE stage_routes;
1353+ PRAGMA optimize;
1354+ `);
1355+}
1356+
1357+async function build(
1358+ candidatePath: string,
1359+ sourceRevisionValue: string,
1360+ createdAtValue: string | undefined,
1361+ load: (database: DatabaseSync) => void | Promise<void>,
1362+): Promise<CatalogBuildResult> {
1363+ const candidate = resolve(candidatePath);
1364+ const sourceRevision = nonEmpty(sourceRevisionValue, "sourceRevision");
1365+ const createdAt = isoTimestamp(createdAtValue ?? new Date().toISOString(), "createdAt");
1366+ if (existsSync(candidate)) fail("CANDIDATE_EXISTS", "catalog candidate path already exists");
1367+ mkdirSync(dirname(candidate), { recursive: true });
1368+
1369+ let database: DatabaseSync | undefined;
1370+ try {
1371+ database = new DatabaseSync(candidate);
1372+ createSchema(database);
1373+ await load(database);
1374+ createIndexes(database);
1375+ deriveCandidates(database);
1376+ validateStagedPerimeter(database);
1377+ validateCandidateSemanticTuples(database);
1378+ const placeResolution = insertFinalCatalog(database, sourceRevision, createdAt);
1379+ dropStaging(database);
1380+ const result = validateOpenCatalog(database);
1381+ database.close();
1382+ database = undefined;
1383+ fsyncFile(candidate);
1384+ return { ...result, candidatePath: candidate, placeResolution };
1385+ } catch (error) {
1386+ try {
1387+ database?.close();
1388+ } finally {
1389+ removeCandidateFiles(candidate);
1390+ }
1391+ if (error instanceof CatalogImportError) throw error;
1392+ fail("CATALOG_BUILD_FAILED", "catalog candidate build failed", error);
1393+ }
1394+}
1395+
1396+export function buildCatalogCandidate(options: BuildCatalogOptions): Promise<CatalogBuildResult> {
1397+ return build(options.candidatePath, options.sourceRevision, options.createdAt, (database) => loadFileSources(database, options.sources));
1398+}
1399+
1400+export function buildCatalogCandidateFromRecords(options: BuildCatalogFromRecordsOptions): Promise<CatalogBuildResult> {
1401+ return build(options.candidatePath, options.sourceRevision, options.createdAt, (database) => loadRecordSources(database, options.sources));
1402+}
1403+
1404+export function validateCatalogCandidate(path: string): CatalogBuildResult {
1405+ const candidate = resolve(path);
1406+ let database: DatabaseSync | undefined;
1407+ try {
1408+ database = new DatabaseSync(candidate, { readOnly: true });
1409+ const result = validateOpenCatalog(database);
1410+ return { ...result, candidatePath: candidate };
1411+ } catch (error) {
1412+ if (error instanceof CatalogImportError) throw error;
1413+ fail("CATALOG_VALIDATION_FAILED", "catalog candidate validation failed", error);
1414+ } finally {
1415+ database?.close();
1416+ }
1417+}
1418+
1419+export function activateCatalogCandidate(candidatePath: string, activePath: string): CatalogBuildResult {
1420+ const candidate = resolve(candidatePath);
1421+ const active = resolve(activePath);
1422+ if (candidate === active) fail("INVALID_ACTIVATION", "candidate and active paths must differ");
1423+ mkdirSync(dirname(active), { recursive: true });
1424+ let result: CatalogBuildResult;
1425+ try {
1426+ result = validateCatalogCandidate(candidate);
1427+ if (statSync(candidate).dev !== statSync(dirname(active)).dev) {
1428+ fail("CROSS_DEVICE_ACTIVATION", "catalog candidate and active directory must share a filesystem");
1429+ }
1430+ fsyncFile(candidate);
1431+ } catch (error) {
1432+ removeCandidateFiles(candidate);
1433+ throw error;
1434+ }
1435+ try {
1436+ renameSync(candidate, active);
1437+ } catch (error) {
1438+ removeCandidateFiles(candidate);
1439+ throw error;
1440+ }
1441+ try {
1442+ fsyncDirectory(dirname(active));
1443+ } catch {
1444+ // The rename already replaced the active path; never unlink either file.
1445+ // Revalidate what is actually active and report the applied activation.
1446+ const reapplied = validateCatalogCandidate(active);
1447+ return { ...reapplied, candidatePath: active, revalidatedAfterRenameFailure: true };
1448+ }
1449+ return { ...result, candidatePath: active };
1450+}
1451+
1452+export async function refreshCatalog(options: BuildCatalogOptions & { readonly activePath: string }): Promise<CatalogBuildResult> {
1453+ await buildCatalogCandidate(options);
1454+ return activateCatalogCandidate(options.candidatePath, options.activePath);
1455+}
1456+
1457+export async function refreshCatalogFromRecords(
1458+ options: BuildCatalogFromRecordsOptions & { readonly activePath: string },
1459+): Promise<CatalogBuildResult> {
1460+ await buildCatalogCandidateFromRecords(options);
1461+ return activateCatalogCandidate(options.candidatePath, options.activePath);
1462+}
added packages/backend/src/catalog.ts +686 -0
new file mode 100644
@@ -0,0 +1,686 @@
1+import { DatabaseSync, type StatementSync } from "node:sqlite";
2+import {
3+ CATALOG_ROUTE,
4+ LIMITS,
5+ SCHEMA_VERSION,
6+ TRANSPORT_MODE,
7+ type CatalogErrorResult,
8+ type PlaceSearchItem,
9+ type PlaceSearchResult,
10+ type ServiceOption,
11+ type ServiceOptionsResult,
12+ type TransportMode,
13+} from "../../contracts/src/index.ts";
14+
15+import { normalizeCatalogSearchText } from "./catalog-import.ts";
16+
17+export type CatalogServiceResolution =
18+ | {
19+ readonly status: "RESOLVED";
20+ readonly monitoringRef: string;
21+ readonly lineRef: string;
22+ readonly directionId: string;
23+ readonly destinationRef: string;
24+ readonly destinationLabel: string;
25+ }
26+ | { readonly status: "UNRESOLVED"; readonly code: "INVALID_SERVICE" };
27+
28+export interface CatalogReader {
29+ searchPlaces(query: string): PlaceSearchItem[];
30+ listServices(placeId: string): ServiceOption[] | undefined;
31+ resolveService(serviceId: string): CatalogServiceResolution;
32+}
33+
34+export interface CatalogHandlerResponse {
35+ readonly status: 200 | 400 | 404 | 405 | 503;
36+ readonly body: PlaceSearchResult | ServiceOptionsResult | CatalogErrorResult;
37+}
38+
39+interface PlaceRow {
40+ readonly placeId: string;
41+ readonly stopLabel: string;
42+ readonly localityLabel: string | null;
43+ readonly mode: TransportMode;
44+}
45+
46+interface ServiceRow {
47+ readonly serviceId: string;
48+ readonly stopLabel: string;
49+ readonly lineLabel: string;
50+ readonly destinationLabel: string;
51+}
52+
53+interface ResolutionRow {
54+ readonly monitoringRef: string;
55+ readonly lineRef: string;
56+ readonly directionId: string;
57+ readonly destinationRef: string;
58+ readonly destinationLabel: string;
59+}
60+
61+interface MetadataRow {
62+ readonly key: string;
63+ readonly value: string;
64+}
65+
66+interface SchemaRow {
67+ readonly name: string;
68+ readonly sql: string | null;
69+}
70+
71+interface ColumnRow {
72+ readonly cid: number;
73+ readonly name: string;
74+ readonly type: string;
75+ readonly notNull: number;
76+ readonly primaryKey: number;
77+}
78+
79+interface ExpectedColumn {
80+ readonly name: string;
81+ readonly type: string;
82+ readonly notNull?: boolean;
83+ readonly primaryKey?: number;
84+}
85+
86+
87+const CLOSE_READER = Symbol("closeCatalogReader");
88+const OPAQUE_ID_BYTES = 47;
89+const CATALOG_SCHEMA_VERSION = "1";
90+const CATALOG_FORMAT_VERSION = "1";
91+const REQUIRED_METADATA = [
92+ "schema_version",
93+ "catalog_version",
94+ "source_revision",
95+ "created_at",
96+] as const;
97+const EXPECTED_TABLES: Readonly<Record<string, true>> = {
98+ metadata: true,
99+ places: true,
100+ services: true,
101+ place_search: true,
102+ place_search_data: true,
103+ place_search_idx: true,
104+ place_search_content: true,
105+ place_search_docsize: true,
106+ place_search_config: true,
107+};
108+const TABLE_COLUMNS: Readonly<Record<string, readonly ExpectedColumn[]>> = {
109+ metadata: [
110+ { name: "key", type: "TEXT", primaryKey: 1 },
111+ { name: "value", type: "TEXT", notNull: true },
112+ ],
113+ places: [
114+ { name: "place_id", type: "TEXT", primaryKey: 1 },
115+ { name: "stop_label", type: "TEXT", notNull: true },
116+ { name: "locality_label", type: "TEXT" },
117+ { name: "mode", type: "TEXT", notNull: true },
118+ { name: "canonical_tuple", type: "TEXT", notNull: true },
119+ ],
120+ services: [
121+ { name: "service_id", type: "TEXT", primaryKey: 1 },
122+ { name: "place_id", type: "TEXT", notNull: true },
123+ { name: "stop_label", type: "TEXT", notNull: true },
124+ { name: "line_label", type: "TEXT", notNull: true },
125+ { name: "destination_label", type: "TEXT", notNull: true },
126+ { name: "monitoring_ref", type: "TEXT", notNull: true },
127+ { name: "line_ref", type: "TEXT", notNull: true },
128+ { name: "direction_id", type: "TEXT", notNull: true },
129+ { name: "destination_ref", type: "TEXT", notNull: true },
130+ { name: "canonical_tuple", type: "TEXT", notNull: true },
131+ ],
132+ place_search: [
133+ { name: "search_text", type: "" },
134+ { name: "place_id", type: "" },
135+ ],
136+};
137+
138+class CatalogUnavailableError extends Error {
139+ constructor() {
140+ super("No complete catalog is active");
141+ this.name = "CatalogUnavailableError";
142+ }
143+}
144+
145+class InvalidCatalogQueryError extends RangeError {
146+ constructor() {
147+ super("Catalog query is outside the supported bounds");
148+ this.name = "InvalidCatalogQueryError";
149+ }
150+}
151+
152+function invalidCandidate(reason: string): never {
153+ throw new Error(`Invalid catalog candidate: ${reason}`);
154+}
155+
156+function rows<T>(statement: StatementSync, ...parameters: (string | number)[]): T[] {
157+ return statement.all(...parameters) as unknown as T[];
158+}
159+
160+function row<T>(statement: StatementSync, ...parameters: (string | number)[]): T | undefined {
161+ return statement.get(...parameters) as unknown as T | undefined;
162+}
163+
164+function validateColumns(database: DatabaseSync): void {
165+ const statement = database.prepare(`
166+ SELECT
167+ cid,
168+ name,
169+ type,
170+ "notnull" AS "notNull",
171+ pk AS "primaryKey"
172+ FROM pragma_table_info(?)
173+ ORDER BY cid
174+ `);
175+
176+ for (const [table, expected] of Object.entries(TABLE_COLUMNS)) {
177+ const actual = rows<ColumnRow>(statement, table);
178+ if (actual.length !== expected.length) invalidCandidate(`${table} columns`);
179+
180+ for (let index = 0; index < expected.length; index += 1) {
181+ const actualColumn = actual[index];
182+ const expectedColumn = expected[index];
183+ if (
184+ actualColumn === undefined
185+ || expectedColumn === undefined
186+ || actualColumn.cid !== index
187+ || actualColumn.name !== expectedColumn.name
188+ || actualColumn.type.toUpperCase() !== expectedColumn.type
189+ || (expectedColumn.notNull === true && actualColumn.notNull !== 1)
190+ || (expectedColumn.primaryKey !== undefined
191+ && actualColumn.primaryKey !== expectedColumn.primaryKey)
192+ ) {
193+ invalidCandidate(`${table} columns`);
194+ }
195+ }
196+ }
197+}
198+
199+function validateSchema(database: DatabaseSync): void {
200+ const schema = rows<SchemaRow>(database.prepare(`
201+ SELECT name, sql
202+ FROM sqlite_schema
203+ WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
204+ ORDER BY name
205+ `));
206+ const names = new Set(schema.map((entry) => entry.name));
207+
208+ if (schema.some((entry) => EXPECTED_TABLES[entry.name] !== true)) {
209+ invalidCandidate("unexpected table");
210+ }
211+ for (const name of Object.keys(EXPECTED_TABLES)) {
212+ if (!names.has(name)) invalidCandidate(`missing ${name}`);
213+ }
214+
215+ const searchSql = schema.find((entry) => entry.name === "place_search")?.sql;
216+ if (
217+ typeof searchSql !== "string"
218+ || !/\bUSING\s+fts5\s*\(/iu.test(searchSql)
219+ || !/\bplace_id\s+UNINDEXED\b/iu.test(searchSql)
220+ ) {
221+ invalidCandidate("place_search definition");
222+ }
223+
224+ validateColumns(database);
225+
226+ const uniqueIndex = database.prepare(`
227+ SELECT indexes.name
228+ FROM pragma_index_list(?) AS indexes
229+ JOIN pragma_index_info(indexes.name) AS columns
230+ WHERE indexes."unique" = 1
231+ GROUP BY indexes.name
232+ HAVING COUNT(*) = 1 AND MAX(columns.name) = ?
233+ LIMIT 1
234+ `);
235+ if (row(uniqueIndex, "places", "canonical_tuple") === undefined) {
236+ invalidCandidate("places canonical uniqueness");
237+ }
238+ if (row(uniqueIndex, "services", "canonical_tuple") === undefined) {
239+ invalidCandidate("services canonical uniqueness");
240+ }
241+
242+ const foreignKeys = rows<{ targetTable: string; sourceColumn: string; targetColumn: string }>(
243+ database.prepare(`
244+ SELECT
245+ "table" AS targetTable,
246+ "from" AS sourceColumn,
247+ "to" AS targetColumn
248+ FROM pragma_foreign_key_list('services')
249+ `),
250+ );
251+ if (
252+ foreignKeys.length !== 1
253+ || foreignKeys[0]?.targetTable !== "places"
254+ || foreignKeys[0].sourceColumn !== "place_id"
255+ || foreignKeys[0].targetColumn !== "place_id"
256+ ) {
257+ invalidCandidate("services foreign key");
258+ }
259+}
260+
261+function validateMetadata(database: DatabaseSync): void {
262+ const statement = database.prepare(`
263+ SELECT key, value
264+ FROM metadata
265+ WHERE key IN (?, ?, ?, ?)
266+ `);
267+ const metadata = new Map(
268+ rows<MetadataRow>(statement, ...REQUIRED_METADATA).map((entry) => [entry.key, entry.value]),
269+ );
270+
271+ if (metadata.size !== REQUIRED_METADATA.length) invalidCandidate("required metadata");
272+ if (metadata.get("schema_version") !== CATALOG_SCHEMA_VERSION) {
273+ invalidCandidate("schema version");
274+ }
275+ if (metadata.get("catalog_version") !== CATALOG_FORMAT_VERSION) {
276+ invalidCandidate("catalog version");
277+ }
278+ if (metadata.get("source_revision")?.trim() === "") {
279+ invalidCandidate("source revision");
280+ }
281+ if (metadata.get("created_at")?.trim() === "") invalidCandidate("creation time");
282+}
283+
284+function validateIntegrity(database: DatabaseSync): void {
285+ const integrity = row<{ integrityCheck: string }>(database.prepare(`
286+ SELECT integrity_check AS integrityCheck
287+ FROM pragma_integrity_check
288+ `));
289+ if (integrity?.integrityCheck !== "ok") invalidCandidate("integrity check");
290+ if (database.prepare("PRAGMA foreign_key_check").get() !== undefined) {
291+ invalidCandidate("foreign key check");
292+ }
293+}
294+
295+function validateData(database: DatabaseSync): void {
296+ const invalidPlace = database.prepare(`
297+ SELECT 1
298+ FROM places
299+ WHERE
300+ typeof(place_id) <> 'text'
301+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
302+ OR substr(place_id, 1, 4) <> 'plc_'
303+ OR substr(place_id, 5) GLOB '*[^A-Za-z0-9_-]*'
304+ OR length(CAST(stop_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
305+ OR (locality_label IS NOT NULL
306+ AND length(CAST(locality_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes})
307+ OR mode NOT IN ('BUS', 'METRO', 'TRAM', 'RER', 'TRANSILIEN')
308+ OR length(CAST(canonical_tuple AS BLOB)) < 1
309+ LIMIT 1
310+ `).get();
311+ if (invalidPlace !== undefined) invalidCandidate("place data");
312+
313+ const invalidService = database.prepare(`
314+ SELECT 1
315+ FROM services
316+ WHERE
317+ typeof(service_id) <> 'text'
318+ OR length(CAST(service_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
319+ OR substr(service_id, 1, 4) <> 'svc_'
320+ OR substr(service_id, 5) GLOB '*[^A-Za-z0-9_-]*'
321+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
322+ OR length(CAST(stop_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
323+ OR length(CAST(line_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
324+ OR length(CAST(destination_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
325+ OR length(CAST(monitoring_ref AS BLOB)) < 1
326+ OR length(CAST(line_ref AS BLOB)) < 1
327+ OR length(CAST(direction_id AS BLOB)) < 1
328+ OR length(CAST(destination_ref AS BLOB)) < 1
329+ OR length(CAST(canonical_tuple AS BLOB)) < 1
330+ LIMIT 1
331+ `).get();
332+ if (invalidService !== undefined) invalidCandidate("service data");
333+
334+ if (database.prepare(`
335+ SELECT 1
336+ FROM places
337+ WHERE NOT EXISTS (
338+ SELECT 1 FROM services WHERE services.place_id = places.place_id
339+ )
340+ LIMIT 1
341+ `).get() !== undefined) {
342+ invalidCandidate("place without service");
343+ }
344+
345+ if (database.prepare(`
346+ SELECT 1
347+ FROM services
348+ JOIN places ON places.place_id = services.place_id
349+ WHERE services.stop_label <> places.stop_label
350+ LIMIT 1
351+ `).get() !== undefined) {
352+ invalidCandidate("inconsistent stop label");
353+ }
354+
355+
356+ if (database.prepare(`
357+ SELECT 1
358+ FROM places
359+ GROUP BY canonical_tuple
360+ HAVING COUNT(*) <> 1
361+ LIMIT 1
362+ `).get() !== undefined || database.prepare(`
363+ SELECT 1
364+ FROM services
365+ GROUP BY canonical_tuple
366+ HAVING COUNT(*) <> 1
367+ LIMIT 1
368+ `).get() !== undefined) {
369+ invalidCandidate("canonical collision");
370+ }
371+
372+ if (database.prepare(`
373+ SELECT 1
374+ FROM place_search
375+ WHERE
376+ length(CAST(search_text AS BLOB)) < 1
377+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
378+ LIMIT 1
379+ `).get() !== undefined) {
380+ invalidCandidate("search data");
381+ }
382+
383+ if (database.prepare(`
384+ SELECT 1
385+ FROM place_search
386+ GROUP BY place_id
387+ HAVING COUNT(*) <> 1
388+ LIMIT 1
389+ `).get() !== undefined || database.prepare(`
390+ SELECT 1
391+ FROM places
392+ LEFT JOIN place_search ON place_search.place_id = places.place_id
393+ WHERE place_search.place_id IS NULL
394+ LIMIT 1
395+ `).get() !== undefined || database.prepare(`
396+ SELECT 1
397+ FROM place_search
398+ LEFT JOIN places ON places.place_id = place_search.place_id
399+ WHERE places.place_id IS NULL
400+ LIMIT 1
401+ `).get() !== undefined) {
402+ invalidCandidate("search consistency");
403+ }
404+
405+ const coverage = new Set(rows<{ mode: TransportMode }>(database.prepare(`
406+ SELECT places.mode AS mode
407+ FROM places
408+ JOIN services ON services.place_id = places.place_id
409+ GROUP BY places.mode
410+ `)).map((entry) => entry.mode));
411+ if (TRANSPORT_MODE.some((mode) => !coverage.has(mode))) {
412+ invalidCandidate("transport mode coverage");
413+ }
414+}
415+
416+function validateCandidate(database: DatabaseSync): void {
417+ validateIntegrity(database);
418+ validateSchema(database);
419+ validateMetadata(database);
420+ validateData(database);
421+}
422+
423+function normalizedMatchQuery(query: string): string {
424+ const trimmed = query.trim();
425+ const codePoints = [...trimmed].length;
426+ if (
427+ codePoints < LIMITS.catalogQueryMinCharacters
428+ || codePoints > LIMITS.catalogQueryMaxCharacters
429+ ) {
430+ throw new InvalidCatalogQueryError();
431+ }
432+
433+ const normalized = normalizeCatalogSearchText(trimmed);
434+ if (normalized === "") throw new InvalidCatalogQueryError();
435+
436+ return normalized
437+ .split(" ")
438+ .map((token) => `"${token.replaceAll('"', '""')}"*`)
439+ .join(" AND ");
440+}
441+
442+function placeItem(entry: PlaceRow): PlaceSearchItem {
443+ const item: PlaceSearchItem = {
444+ placeId: entry.placeId,
445+ stopLabel: entry.stopLabel,
446+ mode: entry.mode,
447+ };
448+ if (entry.localityLabel !== null) item.localityLabel = entry.localityLabel;
449+ return item;
450+}
451+
452+export class SqliteCatalogReader implements CatalogReader {
453+ readonly #database: DatabaseSync;
454+ readonly #searchPlacesStatement: StatementSync;
455+ readonly #placeExistsStatement: StatementSync;
456+ readonly #listServicesStatement: StatementSync;
457+ readonly #resolveServiceStatement: StatementSync;
458+ #closed = false;
459+
460+ private constructor(database: DatabaseSync) {
461+ this.#database = database;
462+ this.#searchPlacesStatement = database.prepare(`
463+ SELECT
464+ places.place_id AS placeId,
465+ places.stop_label AS stopLabel,
466+ places.locality_label AS localityLabel,
467+ places.mode AS mode
468+ FROM place_search
469+ JOIN places ON places.place_id = place_search.place_id
470+ WHERE place_search MATCH ?
471+ ORDER BY
472+ bm25(place_search),
473+ places.stop_label COLLATE NOCASE,
474+ places.place_id
475+ LIMIT ?
476+ `);
477+ this.#placeExistsStatement = database.prepare(`
478+ SELECT 1
479+ FROM places
480+ WHERE place_id = ?
481+ LIMIT 1
482+ `);
483+ this.#listServicesStatement = database.prepare(`
484+ SELECT
485+ service_id AS serviceId,
486+ stop_label AS stopLabel,
487+ line_label AS lineLabel,
488+ destination_label AS destinationLabel
489+ FROM services
490+ WHERE place_id = ?
491+ ORDER BY
492+ line_label COLLATE NOCASE,
493+ destination_label COLLATE NOCASE,
494+ service_id
495+ `);
496+ this.#resolveServiceStatement = database.prepare(`
497+ SELECT
498+ monitoring_ref AS monitoringRef,
499+ line_ref AS lineRef,
500+ direction_id AS directionId,
501+ destination_ref AS destinationRef,
502+ destination_label AS destinationLabel
503+ FROM services
504+ WHERE service_id = ?
505+ LIMIT 1
506+ `);
507+ }
508+
509+ static open(path: string): SqliteCatalogReader {
510+ const database = new DatabaseSync(path, {
511+ readOnly: true,
512+ enableForeignKeyConstraints: true,
513+ enableDoubleQuotedStringLiterals: false,
514+ allowExtension: false,
515+ defensive: true,
516+ });
517+
518+ try {
519+ validateCandidate(database);
520+ return new SqliteCatalogReader(database);
521+ } catch (error) {
522+ try {
523+ database.close();
524+ } catch {
525+ // Preserve the candidate validation/open error.
526+ }
527+ throw error;
528+ }
529+ }
530+
531+ searchPlaces(query: string): PlaceSearchItem[] {
532+ const matchQuery = normalizedMatchQuery(query);
533+ return rows<PlaceRow>(
534+ this.#searchPlacesStatement,
535+ matchQuery,
536+ LIMITS.catalogSearchResults,
537+ ).map(placeItem);
538+ }
539+
540+ listServices(placeId: string): ServiceOption[] | undefined {
541+ if (row(this.#placeExistsStatement, placeId) === undefined) return undefined;
542+ return rows<ServiceRow>(this.#listServicesStatement, placeId).map((entry) => ({
543+ serviceId: entry.serviceId,
544+ stopLabel: entry.stopLabel,
545+ lineLabel: entry.lineLabel,
546+ destinationLabel: entry.destinationLabel,
547+ }));
548+ }
549+
550+ resolveService(serviceId: string): CatalogServiceResolution {
551+ const resolution = row<ResolutionRow>(this.#resolveServiceStatement, serviceId);
552+ if (resolution === undefined) return { status: "UNRESOLVED", code: "INVALID_SERVICE" };
553+ return { status: "RESOLVED", ...resolution };
554+ }
555+
556+ [CLOSE_READER](): void {
557+ if (this.#closed) return;
558+ this.#closed = true;
559+ this.#database.close();
560+ }
561+}
562+
563+export class CatalogManager implements CatalogReader {
564+ #reader: SqliteCatalogReader | undefined;
565+
566+ reload(path: string): void {
567+ const candidate = SqliteCatalogReader.open(path);
568+ const previous = this.#reader;
569+ this.#reader = candidate;
570+ if (previous !== undefined) {
571+ try {
572+ previous[CLOSE_READER]();
573+ } catch {
574+ // The validated replacement is already active; an old close failure cannot roll it back.
575+ }
576+ }
577+ }
578+
579+ searchPlaces(query: string): PlaceSearchItem[] {
580+ return this.#activeReader().searchPlaces(query);
581+ }
582+
583+ listServices(placeId: string): ServiceOption[] | undefined {
584+ return this.#activeReader().listServices(placeId);
585+ }
586+
587+ resolveService(serviceId: string): CatalogServiceResolution {
588+ return this.#activeReader().resolveService(serviceId);
589+ }
590+
591+ #activeReader(): SqliteCatalogReader {
592+ if (this.#reader === undefined) throw new CatalogUnavailableError();
593+ return this.#reader;
594+ }
595+}
596+
597+function catalogError(code: CatalogErrorResult["code"]): CatalogErrorResult {
598+ return { schemaVersion: SCHEMA_VERSION, code };
599+}
600+
601+function response(
602+ status: CatalogHandlerResponse["status"],
603+ body: CatalogHandlerResponse["body"],
604+): CatalogHandlerResponse {
605+ return { status, body };
606+}
607+
608+function requestUrl(value: string): URL | undefined {
609+ try {
610+ return new URL(value, "http://catalog.invalid");
611+ } catch {
612+ return undefined;
613+ }
614+}
615+
616+function servicePlaceId(pathname: string): string | undefined {
617+ const prefix = `${CATALOG_ROUTE.places}/`;
618+ const suffix = "/services";
619+ if (!pathname.startsWith(prefix) || !pathname.endsWith(suffix)) return undefined;
620+
621+ const encoded = pathname.slice(prefix.length, -suffix.length);
622+ if (encoded === "" || encoded.includes("/")) return undefined;
623+ try {
624+ const placeId = decodeURIComponent(encoded);
625+ const bytes = new TextEncoder().encode(placeId).byteLength;
626+ return bytes >= 1 && bytes <= LIMITS.idUtf8Bytes ? placeId : undefined;
627+ } catch {
628+ return undefined;
629+ }
630+}
631+
632+function validSearchParameters(url: URL): string | undefined {
633+ const entries = [...url.searchParams.entries()];
634+ if (entries.length !== 1 || entries[0]?.[0] !== "q") return undefined;
635+ const query = entries[0][1];
636+ try {
637+ normalizedMatchQuery(query);
638+ return query;
639+ } catch {
640+ return undefined;
641+ }
642+}
643+
644+export function handleCatalogRequest(
645+ request: { readonly method: string; readonly url: string },
646+ dependencies: { readonly catalog: CatalogManager },
647+): CatalogHandlerResponse {
648+ const url = requestUrl(request.url);
649+ if (url === undefined) return response(400, catalogError("INVALID_QUERY"));
650+
651+ if (url.pathname === CATALOG_ROUTE.places) {
652+ if (request.method !== "GET") return response(405, catalogError("METHOD_NOT_ALLOWED"));
653+ const query = validSearchParameters(url);
654+ if (query === undefined) return response(400, catalogError("INVALID_QUERY"));
655+
656+ try {
657+ return response(200, {
658+ schemaVersion: SCHEMA_VERSION,
659+ places: dependencies.catalog.searchPlaces(query),
660+ });
661+ } catch (error) {
662+ if (error instanceof InvalidCatalogQueryError) {
663+ return response(400, catalogError("INVALID_QUERY"));
664+ }
665+ return response(503, catalogError("CATALOG_UNAVAILABLE"));
666+ }
667+ }
668+
669+ const placeId = servicePlaceId(url.pathname);
670+ if (placeId !== undefined) {
671+ if (request.method !== "GET") return response(405, catalogError("METHOD_NOT_ALLOWED"));
672+ if ([...url.searchParams].length !== 0) {
673+ return response(400, catalogError("INVALID_QUERY"));
674+ }
675+
676+ try {
677+ const services = dependencies.catalog.listServices(placeId);
678+ if (services === undefined) return response(404, catalogError("PLACE_NOT_FOUND"));
679+ return response(200, { schemaVersion: SCHEMA_VERSION, placeId, services });
680+ } catch {
681+ return response(503, catalogError("CATALOG_UNAVAILABLE"));
682+ }
683+ }
684+
685+ return response(404, catalogError("PLACE_NOT_FOUND"));
686+}
new file mode 100644
@@ -0,0 +1,686 @@
1+import { DatabaseSync, type StatementSync } from "node:sqlite";
2+import {
3+ CATALOG_ROUTE,
4+ LIMITS,
5+ SCHEMA_VERSION,
6+ TRANSPORT_MODE,
7+ type CatalogErrorResult,
8+ type PlaceSearchItem,
9+ type PlaceSearchResult,
10+ type ServiceOption,
11+ type ServiceOptionsResult,
12+ type TransportMode,
13+} from "../../contracts/src/index.ts";
14+
15+import { normalizeCatalogSearchText } from "./catalog-import.ts";
16+
17+export type CatalogServiceResolution =
18+ | {
19+ readonly status: "RESOLVED";
20+ readonly monitoringRef: string;
21+ readonly lineRef: string;
22+ readonly directionId: string;
23+ readonly destinationRef: string;
24+ readonly destinationLabel: string;
25+ }
26+ | { readonly status: "UNRESOLVED"; readonly code: "INVALID_SERVICE" };
27+
28+export interface CatalogReader {
29+ searchPlaces(query: string): PlaceSearchItem[];
30+ listServices(placeId: string): ServiceOption[] | undefined;
31+ resolveService(serviceId: string): CatalogServiceResolution;
32+}
33+
34+export interface CatalogHandlerResponse {
35+ readonly status: 200 | 400 | 404 | 405 | 503;
36+ readonly body: PlaceSearchResult | ServiceOptionsResult | CatalogErrorResult;
37+}
38+
39+interface PlaceRow {
40+ readonly placeId: string;
41+ readonly stopLabel: string;
42+ readonly localityLabel: string | null;
43+ readonly mode: TransportMode;
44+}
45+
46+interface ServiceRow {
47+ readonly serviceId: string;
48+ readonly stopLabel: string;
49+ readonly lineLabel: string;
50+ readonly destinationLabel: string;
51+}
52+
53+interface ResolutionRow {
54+ readonly monitoringRef: string;
55+ readonly lineRef: string;
56+ readonly directionId: string;
57+ readonly destinationRef: string;
58+ readonly destinationLabel: string;
59+}
60+
61+interface MetadataRow {
62+ readonly key: string;
63+ readonly value: string;
64+}
65+
66+interface SchemaRow {
67+ readonly name: string;
68+ readonly sql: string | null;
69+}
70+
71+interface ColumnRow {
72+ readonly cid: number;
73+ readonly name: string;
74+ readonly type: string;
75+ readonly notNull: number;
76+ readonly primaryKey: number;
77+}
78+
79+interface ExpectedColumn {
80+ readonly name: string;
81+ readonly type: string;
82+ readonly notNull?: boolean;
83+ readonly primaryKey?: number;
84+}
85+
86+
87+const CLOSE_READER = Symbol("closeCatalogReader");
88+const OPAQUE_ID_BYTES = 47;
89+const CATALOG_SCHEMA_VERSION = "1";
90+const CATALOG_FORMAT_VERSION = "1";
91+const REQUIRED_METADATA = [
92+ "schema_version",
93+ "catalog_version",
94+ "source_revision",
95+ "created_at",
96+] as const;
97+const EXPECTED_TABLES: Readonly<Record<string, true>> = {
98+ metadata: true,
99+ places: true,
100+ services: true,
101+ place_search: true,
102+ place_search_data: true,
103+ place_search_idx: true,
104+ place_search_content: true,
105+ place_search_docsize: true,
106+ place_search_config: true,
107+};
108+const TABLE_COLUMNS: Readonly<Record<string, readonly ExpectedColumn[]>> = {
109+ metadata: [
110+ { name: "key", type: "TEXT", primaryKey: 1 },
111+ { name: "value", type: "TEXT", notNull: true },
112+ ],
113+ places: [
114+ { name: "place_id", type: "TEXT", primaryKey: 1 },
115+ { name: "stop_label", type: "TEXT", notNull: true },
116+ { name: "locality_label", type: "TEXT" },
117+ { name: "mode", type: "TEXT", notNull: true },
118+ { name: "canonical_tuple", type: "TEXT", notNull: true },
119+ ],
120+ services: [
121+ { name: "service_id", type: "TEXT", primaryKey: 1 },
122+ { name: "place_id", type: "TEXT", notNull: true },
123+ { name: "stop_label", type: "TEXT", notNull: true },
124+ { name: "line_label", type: "TEXT", notNull: true },
125+ { name: "destination_label", type: "TEXT", notNull: true },
126+ { name: "monitoring_ref", type: "TEXT", notNull: true },
127+ { name: "line_ref", type: "TEXT", notNull: true },
128+ { name: "direction_id", type: "TEXT", notNull: true },
129+ { name: "destination_ref", type: "TEXT", notNull: true },
130+ { name: "canonical_tuple", type: "TEXT", notNull: true },
131+ ],
132+ place_search: [
133+ { name: "search_text", type: "" },
134+ { name: "place_id", type: "" },
135+ ],
136+};
137+
138+class CatalogUnavailableError extends Error {
139+ constructor() {
140+ super("No complete catalog is active");
141+ this.name = "CatalogUnavailableError";
142+ }
143+}
144+
145+class InvalidCatalogQueryError extends RangeError {
146+ constructor() {
147+ super("Catalog query is outside the supported bounds");
148+ this.name = "InvalidCatalogQueryError";
149+ }
150+}
151+
152+function invalidCandidate(reason: string): never {
153+ throw new Error(`Invalid catalog candidate: ${reason}`);
154+}
155+
156+function rows<T>(statement: StatementSync, ...parameters: (string | number)[]): T[] {
157+ return statement.all(...parameters) as unknown as T[];
158+}
159+
160+function row<T>(statement: StatementSync, ...parameters: (string | number)[]): T | undefined {
161+ return statement.get(...parameters) as unknown as T | undefined;
162+}
163+
164+function validateColumns(database: DatabaseSync): void {
165+ const statement = database.prepare(`
166+ SELECT
167+ cid,
168+ name,
169+ type,
170+ "notnull" AS "notNull",
171+ pk AS "primaryKey"
172+ FROM pragma_table_info(?)
173+ ORDER BY cid
174+ `);
175+
176+ for (const [table, expected] of Object.entries(TABLE_COLUMNS)) {
177+ const actual = rows<ColumnRow>(statement, table);
178+ if (actual.length !== expected.length) invalidCandidate(`${table} columns`);
179+
180+ for (let index = 0; index < expected.length; index += 1) {
181+ const actualColumn = actual[index];
182+ const expectedColumn = expected[index];
183+ if (
184+ actualColumn === undefined
185+ || expectedColumn === undefined
186+ || actualColumn.cid !== index
187+ || actualColumn.name !== expectedColumn.name
188+ || actualColumn.type.toUpperCase() !== expectedColumn.type
189+ || (expectedColumn.notNull === true && actualColumn.notNull !== 1)
190+ || (expectedColumn.primaryKey !== undefined
191+ && actualColumn.primaryKey !== expectedColumn.primaryKey)
192+ ) {
193+ invalidCandidate(`${table} columns`);
194+ }
195+ }
196+ }
197+}
198+
199+function validateSchema(database: DatabaseSync): void {
200+ const schema = rows<SchemaRow>(database.prepare(`
201+ SELECT name, sql
202+ FROM sqlite_schema
203+ WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
204+ ORDER BY name
205+ `));
206+ const names = new Set(schema.map((entry) => entry.name));
207+
208+ if (schema.some((entry) => EXPECTED_TABLES[entry.name] !== true)) {
209+ invalidCandidate("unexpected table");
210+ }
211+ for (const name of Object.keys(EXPECTED_TABLES)) {
212+ if (!names.has(name)) invalidCandidate(`missing ${name}`);
213+ }
214+
215+ const searchSql = schema.find((entry) => entry.name === "place_search")?.sql;
216+ if (
217+ typeof searchSql !== "string"
218+ || !/\bUSING\s+fts5\s*\(/iu.test(searchSql)
219+ || !/\bplace_id\s+UNINDEXED\b/iu.test(searchSql)
220+ ) {
221+ invalidCandidate("place_search definition");
222+ }
223+
224+ validateColumns(database);
225+
226+ const uniqueIndex = database.prepare(`
227+ SELECT indexes.name
228+ FROM pragma_index_list(?) AS indexes
229+ JOIN pragma_index_info(indexes.name) AS columns
230+ WHERE indexes."unique" = 1
231+ GROUP BY indexes.name
232+ HAVING COUNT(*) = 1 AND MAX(columns.name) = ?
233+ LIMIT 1
234+ `);
235+ if (row(uniqueIndex, "places", "canonical_tuple") === undefined) {
236+ invalidCandidate("places canonical uniqueness");
237+ }
238+ if (row(uniqueIndex, "services", "canonical_tuple") === undefined) {
239+ invalidCandidate("services canonical uniqueness");
240+ }
241+
242+ const foreignKeys = rows<{ targetTable: string; sourceColumn: string; targetColumn: string }>(
243+ database.prepare(`
244+ SELECT
245+ "table" AS targetTable,
246+ "from" AS sourceColumn,
247+ "to" AS targetColumn
248+ FROM pragma_foreign_key_list('services')
249+ `),
250+ );
251+ if (
252+ foreignKeys.length !== 1
253+ || foreignKeys[0]?.targetTable !== "places"
254+ || foreignKeys[0].sourceColumn !== "place_id"
255+ || foreignKeys[0].targetColumn !== "place_id"
256+ ) {
257+ invalidCandidate("services foreign key");
258+ }
259+}
260+
261+function validateMetadata(database: DatabaseSync): void {
262+ const statement = database.prepare(`
263+ SELECT key, value
264+ FROM metadata
265+ WHERE key IN (?, ?, ?, ?)
266+ `);
267+ const metadata = new Map(
268+ rows<MetadataRow>(statement, ...REQUIRED_METADATA).map((entry) => [entry.key, entry.value]),
269+ );
270+
271+ if (metadata.size !== REQUIRED_METADATA.length) invalidCandidate("required metadata");
272+ if (metadata.get("schema_version") !== CATALOG_SCHEMA_VERSION) {
273+ invalidCandidate("schema version");
274+ }
275+ if (metadata.get("catalog_version") !== CATALOG_FORMAT_VERSION) {
276+ invalidCandidate("catalog version");
277+ }
278+ if (metadata.get("source_revision")?.trim() === "") {
279+ invalidCandidate("source revision");
280+ }
281+ if (metadata.get("created_at")?.trim() === "") invalidCandidate("creation time");
282+}
283+
284+function validateIntegrity(database: DatabaseSync): void {
285+ const integrity = row<{ integrityCheck: string }>(database.prepare(`
286+ SELECT integrity_check AS integrityCheck
287+ FROM pragma_integrity_check
288+ `));
289+ if (integrity?.integrityCheck !== "ok") invalidCandidate("integrity check");
290+ if (database.prepare("PRAGMA foreign_key_check").get() !== undefined) {
291+ invalidCandidate("foreign key check");
292+ }
293+}
294+
295+function validateData(database: DatabaseSync): void {
296+ const invalidPlace = database.prepare(`
297+ SELECT 1
298+ FROM places
299+ WHERE
300+ typeof(place_id) <> 'text'
301+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
302+ OR substr(place_id, 1, 4) <> 'plc_'
303+ OR substr(place_id, 5) GLOB '*[^A-Za-z0-9_-]*'
304+ OR length(CAST(stop_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
305+ OR (locality_label IS NOT NULL
306+ AND length(CAST(locality_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes})
307+ OR mode NOT IN ('BUS', 'METRO', 'TRAM', 'RER', 'TRANSILIEN')
308+ OR length(CAST(canonical_tuple AS BLOB)) < 1
309+ LIMIT 1
310+ `).get();
311+ if (invalidPlace !== undefined) invalidCandidate("place data");
312+
313+ const invalidService = database.prepare(`
314+ SELECT 1
315+ FROM services
316+ WHERE
317+ typeof(service_id) <> 'text'
318+ OR length(CAST(service_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
319+ OR substr(service_id, 1, 4) <> 'svc_'
320+ OR substr(service_id, 5) GLOB '*[^A-Za-z0-9_-]*'
321+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
322+ OR length(CAST(stop_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
323+ OR length(CAST(line_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
324+ OR length(CAST(destination_label AS BLOB)) NOT BETWEEN 1 AND ${LIMITS.labelUtf8Bytes}
325+ OR length(CAST(monitoring_ref AS BLOB)) < 1
326+ OR length(CAST(line_ref AS BLOB)) < 1
327+ OR length(CAST(direction_id AS BLOB)) < 1
328+ OR length(CAST(destination_ref AS BLOB)) < 1
329+ OR length(CAST(canonical_tuple AS BLOB)) < 1
330+ LIMIT 1
331+ `).get();
332+ if (invalidService !== undefined) invalidCandidate("service data");
333+
334+ if (database.prepare(`
335+ SELECT 1
336+ FROM places
337+ WHERE NOT EXISTS (
338+ SELECT 1 FROM services WHERE services.place_id = places.place_id
339+ )
340+ LIMIT 1
341+ `).get() !== undefined) {
342+ invalidCandidate("place without service");
343+ }
344+
345+ if (database.prepare(`
346+ SELECT 1
347+ FROM services
348+ JOIN places ON places.place_id = services.place_id
349+ WHERE services.stop_label <> places.stop_label
350+ LIMIT 1
351+ `).get() !== undefined) {
352+ invalidCandidate("inconsistent stop label");
353+ }
354+
355+
356+ if (database.prepare(`
357+ SELECT 1
358+ FROM places
359+ GROUP BY canonical_tuple
360+ HAVING COUNT(*) <> 1
361+ LIMIT 1
362+ `).get() !== undefined || database.prepare(`
363+ SELECT 1
364+ FROM services
365+ GROUP BY canonical_tuple
366+ HAVING COUNT(*) <> 1
367+ LIMIT 1
368+ `).get() !== undefined) {
369+ invalidCandidate("canonical collision");
370+ }
371+
372+ if (database.prepare(`
373+ SELECT 1
374+ FROM place_search
375+ WHERE
376+ length(CAST(search_text AS BLOB)) < 1
377+ OR length(CAST(place_id AS BLOB)) <> ${OPAQUE_ID_BYTES}
378+ LIMIT 1
379+ `).get() !== undefined) {
380+ invalidCandidate("search data");
381+ }
382+
383+ if (database.prepare(`
384+ SELECT 1
385+ FROM place_search
386+ GROUP BY place_id
387+ HAVING COUNT(*) <> 1
388+ LIMIT 1
389+ `).get() !== undefined || database.prepare(`
390+ SELECT 1
391+ FROM places
392+ LEFT JOIN place_search ON place_search.place_id = places.place_id
393+ WHERE place_search.place_id IS NULL
394+ LIMIT 1
395+ `).get() !== undefined || database.prepare(`
396+ SELECT 1
397+ FROM place_search
398+ LEFT JOIN places ON places.place_id = place_search.place_id
399+ WHERE places.place_id IS NULL
400+ LIMIT 1
401+ `).get() !== undefined) {
402+ invalidCandidate("search consistency");
403+ }
404+
405+ const coverage = new Set(rows<{ mode: TransportMode }>(database.prepare(`
406+ SELECT places.mode AS mode
407+ FROM places
408+ JOIN services ON services.place_id = places.place_id
409+ GROUP BY places.mode
410+ `)).map((entry) => entry.mode));
411+ if (TRANSPORT_MODE.some((mode) => !coverage.has(mode))) {
412+ invalidCandidate("transport mode coverage");
413+ }
414+}
415+
416+function validateCandidate(database: DatabaseSync): void {
417+ validateIntegrity(database);
418+ validateSchema(database);
419+ validateMetadata(database);
420+ validateData(database);
421+}
422+
423+function normalizedMatchQuery(query: string): string {
424+ const trimmed = query.trim();
425+ const codePoints = [...trimmed].length;
426+ if (
427+ codePoints < LIMITS.catalogQueryMinCharacters
428+ || codePoints > LIMITS.catalogQueryMaxCharacters
429+ ) {
430+ throw new InvalidCatalogQueryError();
431+ }
432+
433+ const normalized = normalizeCatalogSearchText(trimmed);
434+ if (normalized === "") throw new InvalidCatalogQueryError();
435+
436+ return normalized
437+ .split(" ")
438+ .map((token) => `"${token.replaceAll('"', '""')}"*`)
439+ .join(" AND ");
440+}
441+
442+function placeItem(entry: PlaceRow): PlaceSearchItem {
443+ const item: PlaceSearchItem = {
444+ placeId: entry.placeId,
445+ stopLabel: entry.stopLabel,
446+ mode: entry.mode,
447+ };
448+ if (entry.localityLabel !== null) item.localityLabel = entry.localityLabel;
449+ return item;
450+}
451+
452+export class SqliteCatalogReader implements CatalogReader {
453+ readonly #database: DatabaseSync;
454+ readonly #searchPlacesStatement: StatementSync;
455+ readonly #placeExistsStatement: StatementSync;
456+ readonly #listServicesStatement: StatementSync;
457+ readonly #resolveServiceStatement: StatementSync;
458+ #closed = false;
459+
460+ private constructor(database: DatabaseSync) {
461+ this.#database = database;
462+ this.#searchPlacesStatement = database.prepare(`
463+ SELECT
464+ places.place_id AS placeId,
465+ places.stop_label AS stopLabel,
466+ places.locality_label AS localityLabel,
467+ places.mode AS mode
468+ FROM place_search
469+ JOIN places ON places.place_id = place_search.place_id
470+ WHERE place_search MATCH ?
471+ ORDER BY
472+ bm25(place_search),
473+ places.stop_label COLLATE NOCASE,
474+ places.place_id
475+ LIMIT ?
476+ `);
477+ this.#placeExistsStatement = database.prepare(`
478+ SELECT 1
479+ FROM places
480+ WHERE place_id = ?
481+ LIMIT 1
482+ `);
483+ this.#listServicesStatement = database.prepare(`
484+ SELECT
485+ service_id AS serviceId,
486+ stop_label AS stopLabel,
487+ line_label AS lineLabel,
488+ destination_label AS destinationLabel
489+ FROM services
490+ WHERE place_id = ?
491+ ORDER BY
492+ line_label COLLATE NOCASE,
493+ destination_label COLLATE NOCASE,
494+ service_id
495+ `);
496+ this.#resolveServiceStatement = database.prepare(`
497+ SELECT
498+ monitoring_ref AS monitoringRef,
499+ line_ref AS lineRef,
500+ direction_id AS directionId,
501+ destination_ref AS destinationRef,
502+ destination_label AS destinationLabel
503+ FROM services
504+ WHERE service_id = ?
505+ LIMIT 1
506+ `);
507+ }
508+
509+ static open(path: string): SqliteCatalogReader {
510+ const database = new DatabaseSync(path, {
511+ readOnly: true,
512+ enableForeignKeyConstraints: true,
513+ enableDoubleQuotedStringLiterals: false,
514+ allowExtension: false,
515+ defensive: true,
516+ });
517+
518+ try {
519+ validateCandidate(database);
520+ return new SqliteCatalogReader(database);
521+ } catch (error) {
522+ try {
523+ database.close();
524+ } catch {
525+ // Preserve the candidate validation/open error.
526+ }
527+ throw error;
528+ }
529+ }
530+
531+ searchPlaces(query: string): PlaceSearchItem[] {
532+ const matchQuery = normalizedMatchQuery(query);
533+ return rows<PlaceRow>(
534+ this.#searchPlacesStatement,
535+ matchQuery,
536+ LIMITS.catalogSearchResults,
537+ ).map(placeItem);
538+ }
539+
540+ listServices(placeId: string): ServiceOption[] | undefined {
541+ if (row(this.#placeExistsStatement, placeId) === undefined) return undefined;
542+ return rows<ServiceRow>(this.#listServicesStatement, placeId).map((entry) => ({
543+ serviceId: entry.serviceId,
544+ stopLabel: entry.stopLabel,
545+ lineLabel: entry.lineLabel,
546+ destinationLabel: entry.destinationLabel,
547+ }));
548+ }
549+
550+ resolveService(serviceId: string): CatalogServiceResolution {
551+ const resolution = row<ResolutionRow>(this.#resolveServiceStatement, serviceId);
552+ if (resolution === undefined) return { status: "UNRESOLVED", code: "INVALID_SERVICE" };
553+ return { status: "RESOLVED", ...resolution };
554+ }
555+
556+ [CLOSE_READER](): void {
557+ if (this.#closed) return;
558+ this.#closed = true;
559+ this.#database.close();
560+ }
561+}
562+
563+export class CatalogManager implements CatalogReader {
564+ #reader: SqliteCatalogReader | undefined;
565+
566+ reload(path: string): void {
567+ const candidate = SqliteCatalogReader.open(path);
568+ const previous = this.#reader;
569+ this.#reader = candidate;
570+ if (previous !== undefined) {
571+ try {
572+ previous[CLOSE_READER]();
573+ } catch {
574+ // The validated replacement is already active; an old close failure cannot roll it back.
575+ }
576+ }
577+ }
578+
579+ searchPlaces(query: string): PlaceSearchItem[] {
580+ return this.#activeReader().searchPlaces(query);
581+ }
582+
583+ listServices(placeId: string): ServiceOption[] | undefined {
584+ return this.#activeReader().listServices(placeId);
585+ }
586+
587+ resolveService(serviceId: string): CatalogServiceResolution {
588+ return this.#activeReader().resolveService(serviceId);
589+ }
590+
591+ #activeReader(): SqliteCatalogReader {
592+ if (this.#reader === undefined) throw new CatalogUnavailableError();
593+ return this.#reader;
594+ }
595+}
596+
597+function catalogError(code: CatalogErrorResult["code"]): CatalogErrorResult {
598+ return { schemaVersion: SCHEMA_VERSION, code };
599+}
600+
601+function response(
602+ status: CatalogHandlerResponse["status"],
603+ body: CatalogHandlerResponse["body"],
604+): CatalogHandlerResponse {
605+ return { status, body };
606+}
607+
608+function requestUrl(value: string): URL | undefined {
609+ try {
610+ return new URL(value, "http://catalog.invalid");
611+ } catch {
612+ return undefined;
613+ }
614+}
615+
616+function servicePlaceId(pathname: string): string | undefined {
617+ const prefix = `${CATALOG_ROUTE.places}/`;
618+ const suffix = "/services";
619+ if (!pathname.startsWith(prefix) || !pathname.endsWith(suffix)) return undefined;
620+
621+ const encoded = pathname.slice(prefix.length, -suffix.length);
622+ if (encoded === "" || encoded.includes("/")) return undefined;
623+ try {
624+ const placeId = decodeURIComponent(encoded);
625+ const bytes = new TextEncoder().encode(placeId).byteLength;
626+ return bytes >= 1 && bytes <= LIMITS.idUtf8Bytes ? placeId : undefined;
627+ } catch {
628+ return undefined;
629+ }
630+}
631+
632+function validSearchParameters(url: URL): string | undefined {
633+ const entries = [...url.searchParams.entries()];
634+ if (entries.length !== 1 || entries[0]?.[0] !== "q") return undefined;
635+ const query = entries[0][1];
636+ try {
637+ normalizedMatchQuery(query);
638+ return query;
639+ } catch {
640+ return undefined;
641+ }
642+}
643+
644+export function handleCatalogRequest(
645+ request: { readonly method: string; readonly url: string },
646+ dependencies: { readonly catalog: CatalogManager },
647+): CatalogHandlerResponse {
648+ const url = requestUrl(request.url);
649+ if (url === undefined) return response(400, catalogError("INVALID_QUERY"));
650+
651+ if (url.pathname === CATALOG_ROUTE.places) {
652+ if (request.method !== "GET") return response(405, catalogError("METHOD_NOT_ALLOWED"));
653+ const query = validSearchParameters(url);
654+ if (query === undefined) return response(400, catalogError("INVALID_QUERY"));
655+
656+ try {
657+ return response(200, {
658+ schemaVersion: SCHEMA_VERSION,
659+ places: dependencies.catalog.searchPlaces(query),
660+ });
661+ } catch (error) {
662+ if (error instanceof InvalidCatalogQueryError) {
663+ return response(400, catalogError("INVALID_QUERY"));
664+ }
665+ return response(503, catalogError("CATALOG_UNAVAILABLE"));
666+ }
667+ }
668+
669+ const placeId = servicePlaceId(url.pathname);
670+ if (placeId !== undefined) {
671+ if (request.method !== "GET") return response(405, catalogError("METHOD_NOT_ALLOWED"));
672+ if ([...url.searchParams].length !== 0) {
673+ return response(400, catalogError("INVALID_QUERY"));
674+ }
675+
676+ try {
677+ const services = dependencies.catalog.listServices(placeId);
678+ if (services === undefined) return response(404, catalogError("PLACE_NOT_FOUND"));
679+ return response(200, { schemaVersion: SCHEMA_VERSION, placeId, services });
680+ } catch {
681+ return response(503, catalogError("CATALOG_UNAVAILABLE"));
682+ }
683+ }
684+
685+ return response(404, catalogError("PLACE_NOT_FOUND"));
686+}
modified packages/backend/src/index.ts +9 -0
@@ -13,6 +13,15 @@ import {
1313 type ErrorResult,
1414 } from "../../contracts/src/index.ts";
1515
16+export {
17+ CatalogManager,
18+ SqliteCatalogReader,
19+ handleCatalogRequest,
20+ type CatalogHandlerResponse,
21+ type CatalogReader,
22+ type CatalogServiceResolution,
23+} from "./catalog.ts";
24+
1625 export const REDACTED_SECRET = "[REDACTED]" as const;
1726
1827 export const BACKEND_ERROR_CODE = {
@@ -13,6 +13,15 @@ import {
13 type ErrorResult,13 type ErrorResult,
14 } from "../../contracts/src/index.ts";14 } from "../../contracts/src/index.ts";
15 15
16+export {
17+ CatalogManager,
18+ SqliteCatalogReader,
19+ handleCatalogRequest,
20+ type CatalogHandlerResponse,
21+ type CatalogReader,
22+ type CatalogServiceResolution,
23+} from "./catalog.ts";
24+
16 export const REDACTED_SECRET = "[REDACTED]" as const;25 export const REDACTED_SECRET = "[REDACTED]" as const;
17 26
18 export const BACKEND_ERROR_CODE = {27 export const BACKEND_ERROR_CODE = {
added packages/backend/test/catalog-http.test.ts +158 -0
new file mode 100644
@@ -0,0 +1,158 @@
1+import test from "node:test";
2+import assert from "node:assert/strict";
3+import { mkdtempSync, rmSync } from "node:fs";
4+import { tmpdir } from "node:os";
5+import { join } from "node:path";
6+import { DatabaseSync } from "node:sqlite";
7+import {
8+ CATALOG_ROUTE,
9+ LIMITS,
10+ SCHEMA_VERSION,
11+ isCatalogErrorResult,
12+ isPlaceSearchResult,
13+ isServiceOptionsResult,
14+ type TransportMode,
15+} from "../../contracts/src/index.ts";
16+import { CatalogManager, handleCatalogRequest } from "../src/catalog.ts";
17+
18+interface FixturePlace {
19+ readonly id: string;
20+ readonly label: string;
21+ readonly locality: string;
22+ readonly mode: TransportMode;
23+ readonly serviceId: string;
24+ readonly line: string;
25+ readonly destination: string;
26+ readonly monitoringRef: string;
27+ readonly lineRef: string;
28+}
29+
30+const PLACE_ID = `plc_${"a".repeat(43)}`;
31+const SERVICE_ID = `svc_${"f".repeat(43)}`;
32+const PERSONAL_KEY = "test-only-personal-credential";
33+const temporaryDirectories: string[] = [];
34+
35+const FIXTURE: readonly FixturePlace[] = [
36+ { id: PLACE_ID, label: "Châtelet", locality: "Paris", mode: "METRO", serviceId: SERVICE_ID, line: "4", destination: "Bagneux – Lucie Aubrac", monitoringRef: "raw-monitoring-metro-chatelet", lineRef: "raw-line-metro-4" },
37+ { id: `plc_${"b".repeat(43)}`, label: "République", locality: "Paris", mode: "BUS", serviceId: `svc_${"g".repeat(43)}`, line: "96", destination: "Porte des Lilas", monitoringRef: "raw-monitoring-bus-republique", lineRef: "raw-line-bus-96" },
38+ { id: `plc_${"c".repeat(43)}`, label: "Porte de Versailles", locality: "Paris", mode: "TRAM", serviceId: `svc_${"h".repeat(43)}`, line: "T2", destination: "Pont de Bezons", monitoringRef: "raw-monitoring-tram-versailles", lineRef: "raw-line-tram-t2" },
39+ { id: `plc_${"d".repeat(43)}`, label: "Châtelet – Les Halles", locality: "Paris", mode: "RER", serviceId: `svc_${"i".repeat(43)}`, line: "B", destination: "Aéroport Charles de Gaulle 2", monitoringRef: "raw-monitoring-rer-chatelet", lineRef: "raw-line-rer-b" },
40+ { id: `plc_${"e".repeat(43)}`, label: "Saint-Lazare", locality: "Paris", mode: "TRANSILIEN", serviceId: `svc_${"j".repeat(43)}`, line: "L", destination: "Versailles Rive Droite", monitoringRef: "raw-monitoring-train-saint-lazare", lineRef: "raw-line-train-l" },
41+];
42+
43+function normalizedSearchText(place: FixturePlace): string {
44+ return `${place.label} ${place.locality} ${place.mode}`.normalize("NFKD")
45+ .replace(/\p{M}+/gu, "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ")
46+ .trim().replace(/\s+/gu, " ");
47+}
48+
49+function fixtureManager(): CatalogManager {
50+ const directory = mkdtempSync(join(tmpdir(), "lapin-catalog-http-"));
51+ temporaryDirectories.push(directory);
52+ const path = join(directory, "catalog.sqlite");
53+ const database = new DatabaseSync(path);
54+ database.exec(`
55+ PRAGMA foreign_keys = ON;
56+ CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL);
57+ CREATE TABLE places(place_id TEXT PRIMARY KEY, stop_label TEXT NOT NULL, locality_label TEXT, mode TEXT NOT NULL, canonical_tuple TEXT NOT NULL UNIQUE);
58+ CREATE TABLE services(service_id TEXT PRIMARY KEY, place_id TEXT NOT NULL REFERENCES places(place_id), stop_label TEXT NOT NULL, line_label TEXT NOT NULL, destination_label TEXT NOT NULL, monitoring_ref TEXT NOT NULL, line_ref TEXT NOT NULL, direction_id TEXT NOT NULL, destination_ref TEXT NOT NULL, canonical_tuple TEXT NOT NULL UNIQUE);
59+ CREATE VIRTUAL TABLE place_search USING fts5(search_text, place_id UNINDEXED);
60+ `);
61+ const metadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
62+ metadata.run("schema_version", "1"); metadata.run("catalog_version", "1");
63+ metadata.run("source_revision", "http-fixture-revision");
64+ metadata.run("created_at", "2026-08-30T12:00:00.000Z");
65+ const place = database.prepare("INSERT INTO places VALUES (?, ?, ?, ?, ?)");
66+ const search = database.prepare("INSERT INTO place_search(search_text, place_id) VALUES (?, ?)");
67+ const service = database.prepare("INSERT INTO services VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
68+ FIXTURE.forEach((entry, index) => {
69+ place.run(entry.id, entry.label, entry.locality, entry.mode, `place:v1|${index}:${entry.id}`);
70+ search.run(normalizedSearchText(entry), entry.id);
71+ service.run(entry.serviceId, entry.id, entry.label, entry.line, entry.destination, entry.monitoringRef, entry.lineRef, `direction-${index}`, `raw-terminal-${index}`, `service:v1|${index}:${entry.serviceId}`);
72+ });
73+ database.close();
74+ const manager = new CatalogManager();
75+ manager.reload(path);
76+ return manager;
77+}
78+
79+function request(catalog: CatalogManager, method: string, url: string) {
80+ return handleCatalogRequest({ method, url }, { catalog });
81+}
82+
83+test.afterEach(() => {
84+ while (temporaryDirectories.length > 0) {
85+ const directory = temporaryDirectories.pop();
86+ if (directory !== undefined) rmSync(directory, { recursive: true, force: true });
87+ }
88+});
89+
90+test("GET routes return exact public response bodies without source identifiers or credentials", () => {
91+ const catalog = fixtureManager();
92+ const places = request(catalog, "GET", `${CATALOG_ROUTE.places}?q=chatelet`);
93+ assert.equal(places.status, 200);
94+ assert.ok(isPlaceSearchResult(places.body));
95+ assert.deepEqual(places.body, { schemaVersion: SCHEMA_VERSION, places: [
96+ { placeId: PLACE_ID, stopLabel: "Châtelet", localityLabel: "Paris", mode: "METRO" },
97+ { placeId: FIXTURE[3]!.id, stopLabel: "Châtelet – Les Halles", localityLabel: "Paris", mode: "RER" },
98+ ] });
99+ const services = request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID));
100+ assert.equal(services.status, 200);
101+ assert.ok(isServiceOptionsResult(services.body));
102+ assert.deepEqual(services.body, { schemaVersion: SCHEMA_VERSION, placeId: PLACE_ID, services: [
103+ { serviceId: SERVICE_ID, stopLabel: "Châtelet", lineLabel: "4", destinationLabel: "Bagneux – Lucie Aubrac" },
104+ ] });
105+ const publicJson = JSON.stringify([places.body, services.body]);
106+ for (const forbidden of [PERSONAL_KEY, "raw-monitoring-metro-chatelet", "raw-line-metro-4", "raw-terminal-0", "direction-0"]) {
107+ assert.equal(publicJson.includes(forbidden), false);
108+ }
109+});
110+
111+test("search rejects malformed parameters and accepts exact code-point boundaries", () => {
112+ const catalog = fixtureManager();
113+ for (const url of [CATALOG_ROUTE.places, `${CATALOG_ROUTE.places}?q=`, `${CATALOG_ROUTE.places}?q=a`, `${CATALOG_ROUTE.places}?q=${"a".repeat(101)}`, `${CATALOG_ROUTE.places}?q=--`, `${CATALOG_ROUTE.places}?q=ch&q=metro`, `${CATALOG_ROUTE.places}?q=ch&mode=METRO`]) {
114+ const result = request(catalog, "GET", url);
115+ assert.equal(result.status, 400);
116+ assert.deepEqual(result.body, { schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY" });
117+ assert.ok(isCatalogErrorResult(result.body));
118+ }
119+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=ch`).status, 200);
120+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=${"z".repeat(100)}`).status, 200);
121+ const route = CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID);
122+ assert.deepEqual(request(catalog, "GET", `${route}?raw=forbidden`), {
123+ status: 400, body: { schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY" },
124+ });
125+});
126+
127+test("handler distinguishes method, place, route, and unavailable failures", () => {
128+ const catalog = fixtureManager();
129+ const route = CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID);
130+ assert.deepEqual(request(catalog, "POST", CATALOG_ROUTE.places), { status: 405, body: { schemaVersion: SCHEMA_VERSION, code: "METHOD_NOT_ALLOWED" } });
131+ assert.deepEqual(request(catalog, "DELETE", route), { status: 405, body: { schemaVersion: SCHEMA_VERSION, code: "METHOD_NOT_ALLOWED" } });
132+ assert.deepEqual(request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", "plc_unknown")), { status: 404, body: { schemaVersion: SCHEMA_VERSION, code: "PLACE_NOT_FOUND" } });
133+ assert.deepEqual(request(catalog, "GET", "/api/catalog/unknown"), { status: 404, body: { schemaVersion: SCHEMA_VERSION, code: "PLACE_NOT_FOUND" } });
134+ const unavailable = new CatalogManager();
135+ assert.deepEqual(request(unavailable, "GET", `${CATALOG_ROUTE.places}?q=ch`), { status: 503, body: { schemaVersion: SCHEMA_VERSION, code: "CATALOG_UNAVAILABLE" } });
136+ assert.deepEqual(request(unavailable, "GET", route), { status: 503, body: { schemaVersion: SCHEMA_VERSION, code: "CATALOG_UNAVAILABLE" } });
137+});
138+
139+test("catalog browsing never invokes fetch or a PRIM source", { concurrency: false }, () => {
140+ const catalog = fixtureManager();
141+ const originalFetch = globalThis.fetch;
142+ let fetchCalls = 0;
143+ globalThis.fetch = (() => { fetchCalls += 1; throw new Error("network request"); }) as typeof fetch;
144+ try {
145+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=chatelet`).status, 200);
146+ assert.equal(request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID)).status, 200);
147+ assert.equal(fetchCalls, 0);
148+ } finally {
149+ globalThis.fetch = originalFetch;
150+ }
151+});
152+
153+test("success collections remain within the shared result bound", () => {
154+ const result = request(fixtureManager(), "GET", `${CATALOG_ROUTE.places}?q=ch`);
155+ assert.equal(result.status, 200);
156+ if (!isPlaceSearchResult(result.body)) assert.fail("expected place search result");
157+ assert.ok(result.body.places.length <= LIMITS.catalogSearchResults);
158+});
new file mode 100644
@@ -0,0 +1,158 @@
1+import test from "node:test";
2+import assert from "node:assert/strict";
3+import { mkdtempSync, rmSync } from "node:fs";
4+import { tmpdir } from "node:os";
5+import { join } from "node:path";
6+import { DatabaseSync } from "node:sqlite";
7+import {
8+ CATALOG_ROUTE,
9+ LIMITS,
10+ SCHEMA_VERSION,
11+ isCatalogErrorResult,
12+ isPlaceSearchResult,
13+ isServiceOptionsResult,
14+ type TransportMode,
15+} from "../../contracts/src/index.ts";
16+import { CatalogManager, handleCatalogRequest } from "../src/catalog.ts";
17+
18+interface FixturePlace {
19+ readonly id: string;
20+ readonly label: string;
21+ readonly locality: string;
22+ readonly mode: TransportMode;
23+ readonly serviceId: string;
24+ readonly line: string;
25+ readonly destination: string;
26+ readonly monitoringRef: string;
27+ readonly lineRef: string;
28+}
29+
30+const PLACE_ID = `plc_${"a".repeat(43)}`;
31+const SERVICE_ID = `svc_${"f".repeat(43)}`;
32+const PERSONAL_KEY = "test-only-personal-credential";
33+const temporaryDirectories: string[] = [];
34+
35+const FIXTURE: readonly FixturePlace[] = [
36+ { id: PLACE_ID, label: "Châtelet", locality: "Paris", mode: "METRO", serviceId: SERVICE_ID, line: "4", destination: "Bagneux – Lucie Aubrac", monitoringRef: "raw-monitoring-metro-chatelet", lineRef: "raw-line-metro-4" },
37+ { id: `plc_${"b".repeat(43)}`, label: "République", locality: "Paris", mode: "BUS", serviceId: `svc_${"g".repeat(43)}`, line: "96", destination: "Porte des Lilas", monitoringRef: "raw-monitoring-bus-republique", lineRef: "raw-line-bus-96" },
38+ { id: `plc_${"c".repeat(43)}`, label: "Porte de Versailles", locality: "Paris", mode: "TRAM", serviceId: `svc_${"h".repeat(43)}`, line: "T2", destination: "Pont de Bezons", monitoringRef: "raw-monitoring-tram-versailles", lineRef: "raw-line-tram-t2" },
39+ { id: `plc_${"d".repeat(43)}`, label: "Châtelet – Les Halles", locality: "Paris", mode: "RER", serviceId: `svc_${"i".repeat(43)}`, line: "B", destination: "Aéroport Charles de Gaulle 2", monitoringRef: "raw-monitoring-rer-chatelet", lineRef: "raw-line-rer-b" },
40+ { id: `plc_${"e".repeat(43)}`, label: "Saint-Lazare", locality: "Paris", mode: "TRANSILIEN", serviceId: `svc_${"j".repeat(43)}`, line: "L", destination: "Versailles Rive Droite", monitoringRef: "raw-monitoring-train-saint-lazare", lineRef: "raw-line-train-l" },
41+];
42+
43+function normalizedSearchText(place: FixturePlace): string {
44+ return `${place.label} ${place.locality} ${place.mode}`.normalize("NFKD")
45+ .replace(/\p{M}+/gu, "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ")
46+ .trim().replace(/\s+/gu, " ");
47+}
48+
49+function fixtureManager(): CatalogManager {
50+ const directory = mkdtempSync(join(tmpdir(), "lapin-catalog-http-"));
51+ temporaryDirectories.push(directory);
52+ const path = join(directory, "catalog.sqlite");
53+ const database = new DatabaseSync(path);
54+ database.exec(`
55+ PRAGMA foreign_keys = ON;
56+ CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL);
57+ CREATE TABLE places(place_id TEXT PRIMARY KEY, stop_label TEXT NOT NULL, locality_label TEXT, mode TEXT NOT NULL, canonical_tuple TEXT NOT NULL UNIQUE);
58+ CREATE TABLE services(service_id TEXT PRIMARY KEY, place_id TEXT NOT NULL REFERENCES places(place_id), stop_label TEXT NOT NULL, line_label TEXT NOT NULL, destination_label TEXT NOT NULL, monitoring_ref TEXT NOT NULL, line_ref TEXT NOT NULL, direction_id TEXT NOT NULL, destination_ref TEXT NOT NULL, canonical_tuple TEXT NOT NULL UNIQUE);
59+ CREATE VIRTUAL TABLE place_search USING fts5(search_text, place_id UNINDEXED);
60+ `);
61+ const metadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
62+ metadata.run("schema_version", "1"); metadata.run("catalog_version", "1");
63+ metadata.run("source_revision", "http-fixture-revision");
64+ metadata.run("created_at", "2026-08-30T12:00:00.000Z");
65+ const place = database.prepare("INSERT INTO places VALUES (?, ?, ?, ?, ?)");
66+ const search = database.prepare("INSERT INTO place_search(search_text, place_id) VALUES (?, ?)");
67+ const service = database.prepare("INSERT INTO services VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
68+ FIXTURE.forEach((entry, index) => {
69+ place.run(entry.id, entry.label, entry.locality, entry.mode, `place:v1|${index}:${entry.id}`);
70+ search.run(normalizedSearchText(entry), entry.id);
71+ service.run(entry.serviceId, entry.id, entry.label, entry.line, entry.destination, entry.monitoringRef, entry.lineRef, `direction-${index}`, `raw-terminal-${index}`, `service:v1|${index}:${entry.serviceId}`);
72+ });
73+ database.close();
74+ const manager = new CatalogManager();
75+ manager.reload(path);
76+ return manager;
77+}
78+
79+function request(catalog: CatalogManager, method: string, url: string) {
80+ return handleCatalogRequest({ method, url }, { catalog });
81+}
82+
83+test.afterEach(() => {
84+ while (temporaryDirectories.length > 0) {
85+ const directory = temporaryDirectories.pop();
86+ if (directory !== undefined) rmSync(directory, { recursive: true, force: true });
87+ }
88+});
89+
90+test("GET routes return exact public response bodies without source identifiers or credentials", () => {
91+ const catalog = fixtureManager();
92+ const places = request(catalog, "GET", `${CATALOG_ROUTE.places}?q=chatelet`);
93+ assert.equal(places.status, 200);
94+ assert.ok(isPlaceSearchResult(places.body));
95+ assert.deepEqual(places.body, { schemaVersion: SCHEMA_VERSION, places: [
96+ { placeId: PLACE_ID, stopLabel: "Châtelet", localityLabel: "Paris", mode: "METRO" },
97+ { placeId: FIXTURE[3]!.id, stopLabel: "Châtelet – Les Halles", localityLabel: "Paris", mode: "RER" },
98+ ] });
99+ const services = request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID));
100+ assert.equal(services.status, 200);
101+ assert.ok(isServiceOptionsResult(services.body));
102+ assert.deepEqual(services.body, { schemaVersion: SCHEMA_VERSION, placeId: PLACE_ID, services: [
103+ { serviceId: SERVICE_ID, stopLabel: "Châtelet", lineLabel: "4", destinationLabel: "Bagneux – Lucie Aubrac" },
104+ ] });
105+ const publicJson = JSON.stringify([places.body, services.body]);
106+ for (const forbidden of [PERSONAL_KEY, "raw-monitoring-metro-chatelet", "raw-line-metro-4", "raw-terminal-0", "direction-0"]) {
107+ assert.equal(publicJson.includes(forbidden), false);
108+ }
109+});
110+
111+test("search rejects malformed parameters and accepts exact code-point boundaries", () => {
112+ const catalog = fixtureManager();
113+ for (const url of [CATALOG_ROUTE.places, `${CATALOG_ROUTE.places}?q=`, `${CATALOG_ROUTE.places}?q=a`, `${CATALOG_ROUTE.places}?q=${"a".repeat(101)}`, `${CATALOG_ROUTE.places}?q=--`, `${CATALOG_ROUTE.places}?q=ch&q=metro`, `${CATALOG_ROUTE.places}?q=ch&mode=METRO`]) {
114+ const result = request(catalog, "GET", url);
115+ assert.equal(result.status, 400);
116+ assert.deepEqual(result.body, { schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY" });
117+ assert.ok(isCatalogErrorResult(result.body));
118+ }
119+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=ch`).status, 200);
120+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=${"z".repeat(100)}`).status, 200);
121+ const route = CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID);
122+ assert.deepEqual(request(catalog, "GET", `${route}?raw=forbidden`), {
123+ status: 400, body: { schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY" },
124+ });
125+});
126+
127+test("handler distinguishes method, place, route, and unavailable failures", () => {
128+ const catalog = fixtureManager();
129+ const route = CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID);
130+ assert.deepEqual(request(catalog, "POST", CATALOG_ROUTE.places), { status: 405, body: { schemaVersion: SCHEMA_VERSION, code: "METHOD_NOT_ALLOWED" } });
131+ assert.deepEqual(request(catalog, "DELETE", route), { status: 405, body: { schemaVersion: SCHEMA_VERSION, code: "METHOD_NOT_ALLOWED" } });
132+ assert.deepEqual(request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", "plc_unknown")), { status: 404, body: { schemaVersion: SCHEMA_VERSION, code: "PLACE_NOT_FOUND" } });
133+ assert.deepEqual(request(catalog, "GET", "/api/catalog/unknown"), { status: 404, body: { schemaVersion: SCHEMA_VERSION, code: "PLACE_NOT_FOUND" } });
134+ const unavailable = new CatalogManager();
135+ assert.deepEqual(request(unavailable, "GET", `${CATALOG_ROUTE.places}?q=ch`), { status: 503, body: { schemaVersion: SCHEMA_VERSION, code: "CATALOG_UNAVAILABLE" } });
136+ assert.deepEqual(request(unavailable, "GET", route), { status: 503, body: { schemaVersion: SCHEMA_VERSION, code: "CATALOG_UNAVAILABLE" } });
137+});
138+
139+test("catalog browsing never invokes fetch or a PRIM source", { concurrency: false }, () => {
140+ const catalog = fixtureManager();
141+ const originalFetch = globalThis.fetch;
142+ let fetchCalls = 0;
143+ globalThis.fetch = (() => { fetchCalls += 1; throw new Error("network request"); }) as typeof fetch;
144+ try {
145+ assert.equal(request(catalog, "GET", `${CATALOG_ROUTE.places}?q=chatelet`).status, 200);
146+ assert.equal(request(catalog, "GET", CATALOG_ROUTE.placeServices.replace(":placeId", PLACE_ID)).status, 200);
147+ assert.equal(fetchCalls, 0);
148+ } finally {
149+ globalThis.fetch = originalFetch;
150+ }
151+});
152+
153+test("success collections remain within the shared result bound", () => {
154+ const result = request(fixtureManager(), "GET", `${CATALOG_ROUTE.places}?q=ch`);
155+ assert.equal(result.status, 200);
156+ if (!isPlaceSearchResult(result.body)) assert.fail("expected place search result");
157+ assert.ok(result.body.places.length <= LIMITS.catalogSearchResults);
158+});
added packages/backend/test/catalog-import.test.ts +185 -0
new file mode 100644
@@ -0,0 +1,185 @@
1+import test from "node:test";
2+import type { TestContext } from "node:test";
3+import assert from "node:assert/strict";
4+import { createHash } from "node:crypto";
5+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
6+import { tmpdir } from "node:os";
7+import { join } from "node:path";
8+import { TRANSPORT_MODE } from "../../contracts/src/index.ts";
9+import type { TransportMode } from "../../contracts/src/index.ts";
10+import { CatalogManager } from "../src/catalog.ts";
11+import {
12+ activateCatalogCandidate,
13+ buildCatalogCandidateFromRecords,
14+ createPlaceIdentity,
15+ createServiceIdentity,
16+ downloadCatalogSource,
17+ refreshCatalogFromRecords,
18+} from "../src/catalog-import.ts";
19+import type {
20+ CatalogRecordSourceSet,
21+ CsvRecord,
22+} from "../src/catalog-import.ts";
23+
24+interface Fixture {
25+ readonly schemaVersion: number;
26+ readonly fixtureVersion: string;
27+ readonly dataClassification: string;
28+ readonly attribution: readonly { readonly dataset: string; readonly url: string; readonly retrievedAt: string; readonly license: string }[];
29+ readonly sources: Readonly<Record<keyof CatalogRecordSourceSet, readonly Record<string, unknown>[]>>;
30+}
31+
32+const fixtureUrl = new URL("../../../fixtures/catalog/idfm-v1.json", import.meta.url);
33+const fixtureText = readFileSync(fixtureUrl, "utf8");
34+const fixture = JSON.parse(fixtureText) as Fixture;
35+
36+function sourcesFor(revisionId: string): CatalogRecordSourceSet {
37+ return Object.fromEntries(Object.entries(fixture.sources).map(([name, records]) => [
38+ name,
39+ records.filter((record) => !Array.isArray(record.revisions) || record.revisions.includes(revisionId)).map((record) => {
40+ const { revisions: _revisions, ...sourceRecord } = record;
41+ return sourceRecord as CsvRecord;
42+ }),
43+ ])) as unknown as CatalogRecordSourceSet;
44+}
45+
46+function workspace(t: TestContext): string {
47+ const directory = mkdtempSync(join(tmpdir(), "lapin-fute-catalog-import-"));
48+ t.after(() => rmSync(directory, { recursive: true, force: true }));
49+ return directory;
50+}
51+
52+function place(manager: CatalogManager, query: string, mode: TransportMode, locality?: string) {
53+ const found = manager.searchPlaces(query).find((entry) => entry.mode === mode && (locality === undefined || entry.localityLabel === locality));
54+ assert.ok(found, `expected ${mode} place for ${query}`);
55+ return found;
56+}
57+
58+function service(manager: CatalogManager, placeId: string, destination: string) {
59+ const found = manager.listServices(placeId)?.find((entry) => entry.destinationLabel === destination);
60+ assert.ok(found, `expected service to ${destination}`);
61+ return found;
62+}
63+
64+test("fixture is attributed and contains neither credentials nor personal responses", () => {
65+ assert.equal(fixture.schemaVersion, 1);
66+ assert.equal(fixture.fixtureVersion, "idfm-catalog-v1");
67+ assert.match(fixture.dataClassification, /fixture-only/u);
68+ assert.equal(fixture.attribution.length, 7);
69+ for (const source of fixture.attribution) {
70+ assert.equal(new URL(source.url).hostname, "data.iledefrance-mobilites.fr");
71+ assert.ok(Number.isFinite(Date.parse(source.retrievedAt)));
72+ assert.ok(source.license.length > 0);
73+ }
74+ assert.doesNotMatch(fixtureText, /IDFM_DATASET_TOKEN|PRIM_API_KEY|"apikey"|StopMonitoringDelivery|MonitoredStopVisit/iu);
75+});
76+
77+test("opaque IDs use stable full SHA-256 identities and distinguish opposite directions", () => {
78+ const firstPlace = createPlaceIdentity("METRO", "ZDA-METRO");
79+ assert.deepEqual(firstPlace, createPlaceIdentity("METRO", "ZDA-METRO"));
80+ assert.match(firstPlace.placeId, /^plc_[A-Za-z0-9_-]{43}$/u);
81+ const east = createServiceIdentity({ mode: "METRO", lineId: "C200", monitoringRef: "STIF:StopPoint:Q:METRO-EAST:", directionId: "0", destinationRef: "DEST:METRO-SOUTH" });
82+ const west = createServiceIdentity({ mode: "METRO", lineId: "C200", monitoringRef: "STIF:StopPoint:Q:METRO-WEST:", directionId: "1", destinationRef: "DEST:METRO-NORTH" });
83+ assert.match(east.serviceId, /^svc_[A-Za-z0-9_-]{43}$/u);
84+ assert.notEqual(east.serviceId, west.serviceId);
85+ assert.match(east.canonicalTuple, /^lapin-fute:service:v1\|/u);
86+});
87+
88+test("import covers five modes, gates perimeter and unsupported modes, preserves homonyms, and uses rail terminals", async (t) => {
89+ const directory = workspace(t);
90+ const candidate = join(directory, "candidate.sqlite");
91+ const active = join(directory, "catalog.sqlite");
92+ const result = await buildCatalogCandidateFromRecords({ candidatePath: candidate, sourceRevision: "fixture-2026-08-a", createdAt: "2026-08-30T00:00:00.000Z", sources: sourcesFor("fixture-2026-08-a") });
93+ assert.equal(result.placeCount, 6);
94+ assert.equal(result.serviceCount, 8);
95+ assert.deepEqual(result.countsByMode, { BUS: 2, METRO: 2, TRAM: 1, RER: 2, TRANSILIEN: 1 });
96+ for (const mode of TRANSPORT_MODE) assert.ok(result.countsByMode[mode] > 0);
97+
98+ activateCatalogCandidate(candidate, active);
99+ const manager = new CatalogManager();
100+ manager.reload(active);
101+ const homonyms = manager.searchPlaces("gare du nord").filter((entry) => entry.mode === "BUS");
102+ assert.equal(homonyms.length, 2);
103+ assert.notEqual(homonyms[0]?.placeId, homonyms[1]?.placeId);
104+ assert.deepEqual(new Set(homonyms.map((entry) => entry.localityLabel)), new Set(["Paris", "Saint-Denis"]));
105+
106+ const metro = place(manager, "chatelet", "METRO");
107+ const metroServices = manager.listServices(metro.placeId) ?? [];
108+ assert.equal(metroServices.length, 2);
109+ assert.notEqual(metroServices[0]?.serviceId, metroServices[1]?.serviceId);
110+ const rer = place(manager, "chatelet les halles", "RER");
111+ const east = service(manager, rer.placeId, "Marne-la-Vallée Chessy");
112+ const west = service(manager, rer.placeId, "Saint-Germain-en-Laye");
113+ const eastResolution = manager.resolveService(east.serviceId);
114+ const westResolution = manager.resolveService(west.serviceId);
115+ assert.equal(eastResolution.status, "RESOLVED");
116+ assert.equal(westResolution.status, "RESOLVED");
117+ if (eastResolution.status === "RESOLVED" && westResolution.status === "RESOLVED") {
118+ assert.equal(eastResolution.monitoringRef, westResolution.monitoringRef);
119+ assert.notEqual(eastResolution.directionId, westResolution.directionId);
120+ assert.notEqual(eastResolution.destinationRef, westResolution.destinationRef);
121+ assert.notEqual(eastResolution.destinationLabel, "NORA");
122+ assert.notEqual(westResolution.destinationLabel, "ZEBU");
123+ }
124+ assert.equal(manager.searchPlaces("montmartre").length, 0);
125+ const bus = place(manager, "gare du nord", "BUS", "Paris");
126+ const busServices = manager.listServices(bus.placeId) ?? [];
127+ assert.deepEqual(busServices.map((entry) => entry.destinationLabel), ["Hôpital Européen"]);
128+ assert.equal(busServices.some((entry) => entry.destinationLabel === "Wrong perimeter destination"), false);
129+ assert.doesNotMatch(JSON.stringify([...manager.searchPlaces("chatelet"), ...metroServices, ...busServices]), /STIF:|IDFM:|DEST:/u);
130+});
131+
132+test("activation preserves stable services, removes stale IDs, and failed replacement leaves the active file untouched", async (t) => {
133+ const directory = workspace(t);
134+ const active = join(directory, "catalog.sqlite");
135+ const first = join(directory, "first.sqlite");
136+ await buildCatalogCandidateFromRecords({ candidatePath: first, sourceRevision: "fixture-2026-08-a", sources: sourcesFor("fixture-2026-08-a") });
137+ activateCatalogCandidate(first, active);
138+ const manager = new CatalogManager();
139+ manager.reload(active);
140+ const busId = service(manager, place(manager, "gare du nord", "BUS", "Paris").placeId, "Hôpital Européen").serviceId;
141+ const removedId = service(manager, place(manager, "chatelet", "METRO").placeId, "Porte de Clignancourt").serviceId;
142+
143+ const second = join(directory, "second.sqlite");
144+ await buildCatalogCandidateFromRecords({ candidatePath: second, sourceRevision: "fixture-2026-08-b", sources: sourcesFor("fixture-2026-08-b") });
145+ activateCatalogCandidate(second, active);
146+ assert.equal(manager.resolveService(removedId).status, "RESOLVED");
147+ manager.reload(active);
148+ assert.equal(service(manager, place(manager, "gare du nord", "BUS", "Paris").placeId, "Hôpital Européen").serviceId, busId);
149+ assert.deepEqual(manager.resolveService(removedId), { status: "UNRESOLVED", code: "INVALID_SERVICE" });
150+
151+ const before = createHash("sha256").update(readFileSync(active)).digest("hex");
152+ const invalid = join(directory, "invalid.sqlite");
153+ await assert.rejects(refreshCatalogFromRecords({ candidatePath: invalid, activePath: active, sourceRevision: "fixture-invalid-no-transilien", sources: sourcesFor("fixture-invalid-no-transilien") }), /no resolvable TRANSILIEN/u);
154+ assert.equal(existsSync(invalid), false);
155+ assert.equal(createHash("sha256").update(readFileSync(active)).digest("hex"), before);
156+ assert.equal(manager.resolveService(busId).status, "RESOLVED");
157+});
158+
159+test("restricted source download streams bytes and puts its token only in Authorization", async (t) => {
160+ const directory = workspace(t);
161+ const destination = join(directory, "source.csv");
162+ const bytes = new TextEncoder().encode("id;name\n1;Châtelet\n");
163+ let requested = "";
164+ let authorization: string | null = null;
165+ let redirect: RequestRedirect | undefined;
166+ const fetchSource = (async (input: string | URL | Request, init?: RequestInit) => {
167+ requested = String(input);
168+ authorization = new Headers(init?.headers).get("Authorization");
169+ redirect = init?.redirect;
170+ return new Response(bytes, { status: 200, headers: { "content-type": "text/csv; charset=utf-8" } });
171+ }) as typeof globalThis.fetch;
172+ const result = await downloadCatalogSource({
173+ source: { dataset: "restricted-fixture", url: "https://data.iledefrance-mobilites.fr/source.csv", retrievedAt: "2026-08-30T00:00:00.000Z", license: "Licence Mobilité", restricted: true },
174+ destinationPath: destination,
175+ expectedContentTypes: ["text/csv"],
176+ datasetToken: "operator-test-value",
177+ fetch: fetchSource,
178+ });
179+ assert.equal(requested.includes("operator-test-value"), false);
180+ assert.equal(authorization, "apikey operator-test-value");
181+ assert.equal(redirect, "manual");
182+ assert.equal(result.bytes, bytes.byteLength);
183+ assert.equal(result.sha256, createHash("sha256").update(bytes).digest("hex"));
184+ assert.deepEqual(readFileSync(destination), Buffer.from(bytes));
185+});
new file mode 100644
@@ -0,0 +1,185 @@
1+import test from "node:test";
2+import type { TestContext } from "node:test";
3+import assert from "node:assert/strict";
4+import { createHash } from "node:crypto";
5+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
6+import { tmpdir } from "node:os";
7+import { join } from "node:path";
8+import { TRANSPORT_MODE } from "../../contracts/src/index.ts";
9+import type { TransportMode } from "../../contracts/src/index.ts";
10+import { CatalogManager } from "../src/catalog.ts";
11+import {
12+ activateCatalogCandidate,
13+ buildCatalogCandidateFromRecords,
14+ createPlaceIdentity,
15+ createServiceIdentity,
16+ downloadCatalogSource,
17+ refreshCatalogFromRecords,
18+} from "../src/catalog-import.ts";
19+import type {
20+ CatalogRecordSourceSet,
21+ CsvRecord,
22+} from "../src/catalog-import.ts";
23+
24+interface Fixture {
25+ readonly schemaVersion: number;
26+ readonly fixtureVersion: string;
27+ readonly dataClassification: string;
28+ readonly attribution: readonly { readonly dataset: string; readonly url: string; readonly retrievedAt: string; readonly license: string }[];
29+ readonly sources: Readonly<Record<keyof CatalogRecordSourceSet, readonly Record<string, unknown>[]>>;
30+}
31+
32+const fixtureUrl = new URL("../../../fixtures/catalog/idfm-v1.json", import.meta.url);
33+const fixtureText = readFileSync(fixtureUrl, "utf8");
34+const fixture = JSON.parse(fixtureText) as Fixture;
35+
36+function sourcesFor(revisionId: string): CatalogRecordSourceSet {
37+ return Object.fromEntries(Object.entries(fixture.sources).map(([name, records]) => [
38+ name,
39+ records.filter((record) => !Array.isArray(record.revisions) || record.revisions.includes(revisionId)).map((record) => {
40+ const { revisions: _revisions, ...sourceRecord } = record;
41+ return sourceRecord as CsvRecord;
42+ }),
43+ ])) as unknown as CatalogRecordSourceSet;
44+}
45+
46+function workspace(t: TestContext): string {
47+ const directory = mkdtempSync(join(tmpdir(), "lapin-fute-catalog-import-"));
48+ t.after(() => rmSync(directory, { recursive: true, force: true }));
49+ return directory;
50+}
51+
52+function place(manager: CatalogManager, query: string, mode: TransportMode, locality?: string) {
53+ const found = manager.searchPlaces(query).find((entry) => entry.mode === mode && (locality === undefined || entry.localityLabel === locality));
54+ assert.ok(found, `expected ${mode} place for ${query}`);
55+ return found;
56+}
57+
58+function service(manager: CatalogManager, placeId: string, destination: string) {
59+ const found = manager.listServices(placeId)?.find((entry) => entry.destinationLabel === destination);
60+ assert.ok(found, `expected service to ${destination}`);
61+ return found;
62+}
63+
64+test("fixture is attributed and contains neither credentials nor personal responses", () => {
65+ assert.equal(fixture.schemaVersion, 1);
66+ assert.equal(fixture.fixtureVersion, "idfm-catalog-v1");
67+ assert.match(fixture.dataClassification, /fixture-only/u);
68+ assert.equal(fixture.attribution.length, 7);
69+ for (const source of fixture.attribution) {
70+ assert.equal(new URL(source.url).hostname, "data.iledefrance-mobilites.fr");
71+ assert.ok(Number.isFinite(Date.parse(source.retrievedAt)));
72+ assert.ok(source.license.length > 0);
73+ }
74+ assert.doesNotMatch(fixtureText, /IDFM_DATASET_TOKEN|PRIM_API_KEY|"apikey"|StopMonitoringDelivery|MonitoredStopVisit/iu);
75+});
76+
77+test("opaque IDs use stable full SHA-256 identities and distinguish opposite directions", () => {
78+ const firstPlace = createPlaceIdentity("METRO", "ZDA-METRO");
79+ assert.deepEqual(firstPlace, createPlaceIdentity("METRO", "ZDA-METRO"));
80+ assert.match(firstPlace.placeId, /^plc_[A-Za-z0-9_-]{43}$/u);
81+ const east = createServiceIdentity({ mode: "METRO", lineId: "C200", monitoringRef: "STIF:StopPoint:Q:METRO-EAST:", directionId: "0", destinationRef: "DEST:METRO-SOUTH" });
82+ const west = createServiceIdentity({ mode: "METRO", lineId: "C200", monitoringRef: "STIF:StopPoint:Q:METRO-WEST:", directionId: "1", destinationRef: "DEST:METRO-NORTH" });
83+ assert.match(east.serviceId, /^svc_[A-Za-z0-9_-]{43}$/u);
84+ assert.notEqual(east.serviceId, west.serviceId);
85+ assert.match(east.canonicalTuple, /^lapin-fute:service:v1\|/u);
86+});
87+
88+test("import covers five modes, gates perimeter and unsupported modes, preserves homonyms, and uses rail terminals", async (t) => {
89+ const directory = workspace(t);
90+ const candidate = join(directory, "candidate.sqlite");
91+ const active = join(directory, "catalog.sqlite");
92+ const result = await buildCatalogCandidateFromRecords({ candidatePath: candidate, sourceRevision: "fixture-2026-08-a", createdAt: "2026-08-30T00:00:00.000Z", sources: sourcesFor("fixture-2026-08-a") });
93+ assert.equal(result.placeCount, 6);
94+ assert.equal(result.serviceCount, 8);
95+ assert.deepEqual(result.countsByMode, { BUS: 2, METRO: 2, TRAM: 1, RER: 2, TRANSILIEN: 1 });
96+ for (const mode of TRANSPORT_MODE) assert.ok(result.countsByMode[mode] > 0);
97+
98+ activateCatalogCandidate(candidate, active);
99+ const manager = new CatalogManager();
100+ manager.reload(active);
101+ const homonyms = manager.searchPlaces("gare du nord").filter((entry) => entry.mode === "BUS");
102+ assert.equal(homonyms.length, 2);
103+ assert.notEqual(homonyms[0]?.placeId, homonyms[1]?.placeId);
104+ assert.deepEqual(new Set(homonyms.map((entry) => entry.localityLabel)), new Set(["Paris", "Saint-Denis"]));
105+
106+ const metro = place(manager, "chatelet", "METRO");
107+ const metroServices = manager.listServices(metro.placeId) ?? [];
108+ assert.equal(metroServices.length, 2);
109+ assert.notEqual(metroServices[0]?.serviceId, metroServices[1]?.serviceId);
110+ const rer = place(manager, "chatelet les halles", "RER");
111+ const east = service(manager, rer.placeId, "Marne-la-Vallée Chessy");
112+ const west = service(manager, rer.placeId, "Saint-Germain-en-Laye");
113+ const eastResolution = manager.resolveService(east.serviceId);
114+ const westResolution = manager.resolveService(west.serviceId);
115+ assert.equal(eastResolution.status, "RESOLVED");
116+ assert.equal(westResolution.status, "RESOLVED");
117+ if (eastResolution.status === "RESOLVED" && westResolution.status === "RESOLVED") {
118+ assert.equal(eastResolution.monitoringRef, westResolution.monitoringRef);
119+ assert.notEqual(eastResolution.directionId, westResolution.directionId);
120+ assert.notEqual(eastResolution.destinationRef, westResolution.destinationRef);
121+ assert.notEqual(eastResolution.destinationLabel, "NORA");
122+ assert.notEqual(westResolution.destinationLabel, "ZEBU");
123+ }
124+ assert.equal(manager.searchPlaces("montmartre").length, 0);
125+ const bus = place(manager, "gare du nord", "BUS", "Paris");
126+ const busServices = manager.listServices(bus.placeId) ?? [];
127+ assert.deepEqual(busServices.map((entry) => entry.destinationLabel), ["Hôpital Européen"]);
128+ assert.equal(busServices.some((entry) => entry.destinationLabel === "Wrong perimeter destination"), false);
129+ assert.doesNotMatch(JSON.stringify([...manager.searchPlaces("chatelet"), ...metroServices, ...busServices]), /STIF:|IDFM:|DEST:/u);
130+});
131+
132+test("activation preserves stable services, removes stale IDs, and failed replacement leaves the active file untouched", async (t) => {
133+ const directory = workspace(t);
134+ const active = join(directory, "catalog.sqlite");
135+ const first = join(directory, "first.sqlite");
136+ await buildCatalogCandidateFromRecords({ candidatePath: first, sourceRevision: "fixture-2026-08-a", sources: sourcesFor("fixture-2026-08-a") });
137+ activateCatalogCandidate(first, active);
138+ const manager = new CatalogManager();
139+ manager.reload(active);
140+ const busId = service(manager, place(manager, "gare du nord", "BUS", "Paris").placeId, "Hôpital Européen").serviceId;
141+ const removedId = service(manager, place(manager, "chatelet", "METRO").placeId, "Porte de Clignancourt").serviceId;
142+
143+ const second = join(directory, "second.sqlite");
144+ await buildCatalogCandidateFromRecords({ candidatePath: second, sourceRevision: "fixture-2026-08-b", sources: sourcesFor("fixture-2026-08-b") });
145+ activateCatalogCandidate(second, active);
146+ assert.equal(manager.resolveService(removedId).status, "RESOLVED");
147+ manager.reload(active);
148+ assert.equal(service(manager, place(manager, "gare du nord", "BUS", "Paris").placeId, "Hôpital Européen").serviceId, busId);
149+ assert.deepEqual(manager.resolveService(removedId), { status: "UNRESOLVED", code: "INVALID_SERVICE" });
150+
151+ const before = createHash("sha256").update(readFileSync(active)).digest("hex");
152+ const invalid = join(directory, "invalid.sqlite");
153+ await assert.rejects(refreshCatalogFromRecords({ candidatePath: invalid, activePath: active, sourceRevision: "fixture-invalid-no-transilien", sources: sourcesFor("fixture-invalid-no-transilien") }), /no resolvable TRANSILIEN/u);
154+ assert.equal(existsSync(invalid), false);
155+ assert.equal(createHash("sha256").update(readFileSync(active)).digest("hex"), before);
156+ assert.equal(manager.resolveService(busId).status, "RESOLVED");
157+});
158+
159+test("restricted source download streams bytes and puts its token only in Authorization", async (t) => {
160+ const directory = workspace(t);
161+ const destination = join(directory, "source.csv");
162+ const bytes = new TextEncoder().encode("id;name\n1;Châtelet\n");
163+ let requested = "";
164+ let authorization: string | null = null;
165+ let redirect: RequestRedirect | undefined;
166+ const fetchSource = (async (input: string | URL | Request, init?: RequestInit) => {
167+ requested = String(input);
168+ authorization = new Headers(init?.headers).get("Authorization");
169+ redirect = init?.redirect;
170+ return new Response(bytes, { status: 200, headers: { "content-type": "text/csv; charset=utf-8" } });
171+ }) as typeof globalThis.fetch;
172+ const result = await downloadCatalogSource({
173+ source: { dataset: "restricted-fixture", url: "https://data.iledefrance-mobilites.fr/source.csv", retrievedAt: "2026-08-30T00:00:00.000Z", license: "Licence Mobilité", restricted: true },
174+ destinationPath: destination,
175+ expectedContentTypes: ["text/csv"],
176+ datasetToken: "operator-test-value",
177+ fetch: fetchSource,
178+ });
179+ assert.equal(requested.includes("operator-test-value"), false);
180+ assert.equal(authorization, "apikey operator-test-value");
181+ assert.equal(redirect, "manual");
182+ assert.equal(result.bytes, bytes.byteLength);
183+ assert.equal(result.sha256, createHash("sha256").update(bytes).digest("hex"));
184+ assert.deepEqual(readFileSync(destination), Buffer.from(bytes));
185+});
added packages/backend/test/catalog.test.ts +467 -0
new file mode 100644
@@ -0,0 +1,467 @@
1+import test from "node:test";
2+import assert from "node:assert/strict";
3+import { createHash } from "node:crypto";
4+import { mkdtempSync, rmSync } from "node:fs";
5+import { tmpdir } from "node:os";
6+import { join } from "node:path";
7+import { DatabaseSync } from "node:sqlite";
8+import { LIMITS, type TransportMode } from "../../contracts/src/index.ts";
9+import { CatalogManager, SqliteCatalogReader } from "../src/catalog.ts";
10+
11+interface PlaceSeed {
12+ readonly key: string;
13+ readonly mode: TransportMode;
14+ readonly sourceCode: string;
15+ readonly stopLabel: string;
16+ readonly localityLabel?: string;
17+}
18+
19+interface ServiceSeed {
20+ readonly key: string;
21+ readonly placeKey: string;
22+ readonly mode: TransportMode;
23+ readonly stopLabel: string;
24+ readonly lineLabel: string;
25+ readonly destinationLabel: string;
26+ readonly monitoringRef: string;
27+ readonly lineRef: string;
28+ readonly directionId: string;
29+ readonly destinationRef: string;
30+}
31+
32+interface CatalogSeed {
33+ readonly revision?: string;
34+ readonly places?: readonly PlaceSeed[];
35+ readonly services?: readonly ServiceSeed[];
36+ readonly omitServiceKeys?: readonly string[];
37+ readonly catalogVersion?: string;
38+ readonly omitSearchFor?: string;
39+}
40+
41+interface BuiltCatalog {
42+ readonly path: string;
43+ readonly placeIds: Readonly<Record<string, string>>;
44+ readonly serviceIds: Readonly<Record<string, string>>;
45+}
46+
47+const temporaryDirectories: string[] = [];
48+
49+const BASE_PLACES: readonly PlaceSeed[] = [
50+ {
51+ key: "bus",
52+ mode: "BUS",
53+ sourceCode: "zda-bus-republique",
54+ stopLabel: "République",
55+ localityLabel: "Paris",
56+ },
57+ {
58+ key: "metro",
59+ mode: "METRO",
60+ sourceCode: "zda-metro-chatelet",
61+ stopLabel: "Châtelet",
62+ localityLabel: "Paris",
63+ },
64+ {
65+ key: "tram",
66+ mode: "TRAM",
67+ sourceCode: "zda-tram-versailles",
68+ stopLabel: "Porte de Versailles",
69+ localityLabel: "Paris",
70+ },
71+ {
72+ key: "rer",
73+ mode: "RER",
74+ sourceCode: "zda-rer-chatelet-halles",
75+ stopLabel: "Châtelet – Les Halles",
76+ localityLabel: "Paris",
77+ },
78+ {
79+ key: "transilien",
80+ mode: "TRANSILIEN",
81+ sourceCode: "zda-train-saint-lazare",
82+ stopLabel: "Saint-Lazare",
83+ localityLabel: "Paris",
84+ },
85+];
86+
87+const BASE_SERVICES: readonly ServiceSeed[] = [
88+ {
89+ key: "bus-east",
90+ placeKey: "bus",
91+ mode: "BUS",
92+ stopLabel: "République",
93+ lineLabel: "96",
94+ destinationLabel: "Porte des Lilas",
95+ monitoringRef: "raw-monitoring-bus-east",
96+ lineRef: "raw-line-bus-96",
97+ directionId: "east",
98+ destinationRef: "terminal-bus-east",
99+ },
100+ {
101+ key: "metro-south",
102+ placeKey: "metro",
103+ mode: "METRO",
104+ stopLabel: "Châtelet",
105+ lineLabel: "4",
106+ destinationLabel: "Bagneux – Lucie Aubrac",
107+ monitoringRef: "raw-monitoring-metro-south",
108+ lineRef: "raw-line-metro-4",
109+ directionId: "south",
110+ destinationRef: "terminal-metro-south",
111+ },
112+ {
113+ key: "tram-west",
114+ placeKey: "tram",
115+ mode: "TRAM",
116+ stopLabel: "Porte de Versailles",
117+ lineLabel: "T2",
118+ destinationLabel: "Porte de Versailles",
119+ monitoringRef: "raw-monitoring-tram-west",
120+ lineRef: "raw-line-tram-t2",
121+ directionId: "west",
122+ destinationRef: "terminal-tram-west",
123+ },
124+ {
125+ key: "rer-north",
126+ placeKey: "rer",
127+ mode: "RER",
128+ stopLabel: "Châtelet – Les Halles",
129+ lineLabel: "B",
130+ destinationLabel: "Aéroport Charles de Gaulle 2",
131+ monitoringRef: "raw-shared-rer-station",
132+ lineRef: "raw-line-rer-b",
133+ directionId: "north",
134+ destinationRef: "terminal-rer-north",
135+ },
136+ {
137+ key: "train-west",
138+ placeKey: "transilien",
139+ mode: "TRANSILIEN",
140+ stopLabel: "Saint-Lazare",
141+ lineLabel: "L",
142+ destinationLabel: "Versailles Rive Droite",
143+ monitoringRef: "raw-monitoring-train-west",
144+ lineRef: "raw-line-train-l",
145+ directionId: "west",
146+ destinationRef: "terminal-train-west",
147+ },
148+];
149+
150+function canonicalTuple(namespace: string, values: readonly string[]): string {
151+ const framed = values.map((value) => `${Buffer.byteLength(value, "utf8")}:${value}`).join("|");
152+ return `${namespace}:v1|${framed}`;
153+}
154+
155+function opaqueId(prefix: "plc_" | "svc_", canonical: string): string {
156+ return prefix + createHash("sha256").update(canonical).digest("base64url");
157+}
158+
159+function placeIdentity(place: PlaceSeed): { id: string; canonical: string } {
160+ const canonical = canonicalTuple("place", [place.mode, place.sourceCode]);
161+ return { id: opaqueId("plc_", canonical), canonical };
162+}
163+
164+function serviceIdentity(service: ServiceSeed): { id: string; canonical: string } {
165+ const canonical = canonicalTuple("service", [
166+ service.mode,
167+ service.lineRef,
168+ service.monitoringRef,
169+ service.directionId,
170+ service.destinationRef,
171+ ]);
172+ return { id: opaqueId("svc_", canonical), canonical };
173+}
174+
175+function searchText(place: PlaceSeed): string {
176+ return [place.stopLabel, place.localityLabel, place.mode]
177+ .filter((value): value is string => value !== undefined)
178+ .join(" ")
179+ .normalize("NFKD")
180+ .replace(/\p{M}+/gu, "")
181+ .toLowerCase()
182+ .replace(/[^\p{L}\p{N}]+/gu, " ")
183+ .trim()
184+ .replace(/\s+/gu, " ");
185+}
186+
187+function buildCatalog(seed: CatalogSeed = {}): BuiltCatalog {
188+ const directory = mkdtempSync(join(tmpdir(), "lapin-catalog-runtime-"));
189+ temporaryDirectories.push(directory);
190+ const path = join(directory, "catalog.sqlite");
191+ const database = new DatabaseSync(path);
192+ database.exec(`
193+ PRAGMA foreign_keys = ON;
194+ CREATE TABLE metadata(
195+ key TEXT PRIMARY KEY,
196+ value TEXT NOT NULL
197+ );
198+ CREATE TABLE places(
199+ place_id TEXT PRIMARY KEY,
200+ stop_label TEXT NOT NULL,
201+ locality_label TEXT,
202+ mode TEXT NOT NULL,
203+ canonical_tuple TEXT NOT NULL UNIQUE
204+ );
205+ CREATE TABLE services(
206+ service_id TEXT PRIMARY KEY,
207+ place_id TEXT NOT NULL REFERENCES places(place_id),
208+ stop_label TEXT NOT NULL,
209+ line_label TEXT NOT NULL,
210+ destination_label TEXT NOT NULL,
211+ monitoring_ref TEXT NOT NULL,
212+ line_ref TEXT NOT NULL,
213+ direction_id TEXT NOT NULL,
214+ destination_ref TEXT NOT NULL,
215+ canonical_tuple TEXT NOT NULL UNIQUE
216+ );
217+ CREATE VIRTUAL TABLE place_search USING fts5(search_text, place_id UNINDEXED);
218+ `);
219+
220+ const metadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
221+ metadata.run("schema_version", "1");
222+ metadata.run("catalog_version", seed.catalogVersion ?? "1");
223+ metadata.run("source_revision", seed.revision ?? "fixture-revision-a");
224+ metadata.run("created_at", "2026-08-30T12:00:00.000Z");
225+
226+ const places = [...BASE_PLACES, ...(seed.places ?? [])];
227+ const omittedServiceKeys = new Set(seed.omitServiceKeys ?? []);
228+ const services = [
229+ ...BASE_SERVICES.filter((service) => !omittedServiceKeys.has(service.key)),
230+ ...(seed.services ?? []),
231+ ];
232+ const placeIds: Record<string, string> = {};
233+ const serviceIds: Record<string, string> = {};
234+ const insertPlace = database.prepare(`
235+ INSERT INTO places(place_id, stop_label, locality_label, mode, canonical_tuple)
236+ VALUES (?, ?, ?, ?, ?)
237+ `);
238+ const insertSearch = database.prepare(
239+ "INSERT INTO place_search(search_text, place_id) VALUES (?, ?)",
240+ );
241+ for (const place of places) {
242+ const identity = placeIdentity(place);
243+ placeIds[place.key] = identity.id;
244+ insertPlace.run(
245+ identity.id,
246+ place.stopLabel,
247+ place.localityLabel ?? null,
248+ place.mode,
249+ identity.canonical,
250+ );
251+ if (place.key !== seed.omitSearchFor) insertSearch.run(searchText(place), identity.id);
252+ }
253+
254+ const insertService = database.prepare(`
255+ INSERT INTO services(
256+ service_id,
257+ place_id,
258+ stop_label,
259+ line_label,
260+ destination_label,
261+ monitoring_ref,
262+ line_ref,
263+ direction_id,
264+ destination_ref,
265+ canonical_tuple
266+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
267+ `);
268+ for (const service of services) {
269+ const identity = serviceIdentity(service);
270+ serviceIds[service.key] = identity.id;
271+ insertService.run(
272+ identity.id,
273+ placeIds[service.placeKey],
274+ service.stopLabel,
275+ service.lineLabel,
276+ service.destinationLabel,
277+ service.monitoringRef,
278+ service.lineRef,
279+ service.directionId,
280+ service.destinationRef,
281+ identity.canonical,
282+ );
283+ }
284+ database.close();
285+ return { path, placeIds, serviceIds };
286+}
287+
288+test.afterEach(() => {
289+ while (temporaryDirectories.length > 0) {
290+ const directory = temporaryDirectories.pop();
291+ if (directory !== undefined) rmSync(directory, { recursive: true, force: true });
292+ }
293+});
294+
295+test("search is accent tolerant, punctuation safe, and bounded by Unicode code points", () => {
296+ const catalog = buildCatalog();
297+ const reader = SqliteCatalogReader.open(catalog.path);
298+
299+ assert.deepEqual(
300+ reader.searchPlaces("chatelet").map((place) => place.placeId),
301+ [catalog.placeIds.metro, catalog.placeIds.rer],
302+ );
303+ assert.deepEqual(
304+ reader.searchPlaces(" ChÂtElEt---LES...HALLES ").map((place) => place.placeId),
305+ [catalog.placeIds.rer],
306+ );
307+ assert.throws(() => reader.searchPlaces("a"), RangeError);
308+ assert.throws(() => reader.searchPlaces("a".repeat(101)), RangeError);
309+ assert.throws(() => reader.searchPlaces("--"), RangeError);
310+ assert.doesNotThrow(() => reader.searchPlaces("😀a"));
311+ assert.doesNotThrow(() => reader.searchPlaces("z".repeat(100)));
312+});
313+
314+test("search preserves homonyms and applies deterministic ranking before the 20-row bound", () => {
315+ const homonyms: PlaceSeed[] = [
316+ {
317+ key: "homonym-b",
318+ mode: "BUS",
319+ sourceCode: "zda-homonym-b",
320+ stopLabel: "Place Centrale",
321+ localityLabel: "Saint-Denis",
322+ },
323+ {
324+ key: "homonym-a",
325+ mode: "BUS",
326+ sourceCode: "zda-homonym-a",
327+ stopLabel: "Place Centrale",
328+ localityLabel: "Saint-Denis",
329+ },
330+ ];
331+ const rankedPlaces: PlaceSeed[] = Array.from({ length: 25 }, (_, index) => ({
332+ key: `rank-${index}`,
333+ mode: "BUS" as const,
334+ sourceCode: `zda-rank-${index}`,
335+ stopLabel: `Alpha ${String(index).padStart(2, "0")}`,
336+ localityLabel: "Paris",
337+ }));
338+ const services: ServiceSeed[] = [...homonyms, ...rankedPlaces].map((place, index) => ({
339+ key: `extra-service-${index}`,
340+ placeKey: place.key,
341+ mode: place.mode,
342+ stopLabel: place.stopLabel,
343+ lineLabel: `X${index}`,
344+ destinationLabel: "Terminus",
345+ monitoringRef: `raw-monitoring-extra-${index}`,
346+ lineRef: `raw-line-extra-${index}`,
347+ directionId: "outbound",
348+ destinationRef: `terminal-extra-${index}`,
349+ }));
350+ const catalog = buildCatalog({ places: [...homonyms, ...rankedPlaces], services });
351+ const reader = SqliteCatalogReader.open(catalog.path);
352+
353+ const homonymResults = reader.searchPlaces("place centrale");
354+ assert.equal(homonymResults.length, 2);
355+ assert.deepEqual(
356+ homonymResults.map((place) => place.placeId),
357+ [catalog.placeIds["homonym-a"], catalog.placeIds["homonym-b"]].sort(),
358+ );
359+
360+ const ranked = reader.searchPlaces("alpha");
361+ assert.equal(ranked.length, LIMITS.catalogSearchResults);
362+ assert.deepEqual(
363+ ranked.map((place) => place.stopLabel),
364+ Array.from({ length: 20 }, (_, index) => `Alpha ${String(index).padStart(2, "0")}`),
365+ );
366+});
367+
368+test("service options return every valid direction without applying the place-search bound", () => {
369+ const extraServices: ServiceSeed[] = Array.from({ length: 25 }, (_, index) => ({
370+ key: `bus-option-${index}`,
371+ placeKey: "bus",
372+ mode: "BUS",
373+ stopLabel: "République",
374+ lineLabel: `X${String(index).padStart(2, "0")}`,
375+ destinationLabel: `Terminus ${index}`,
376+ monitoringRef: `raw-monitoring-bus-option-${index}`,
377+ lineRef: `raw-line-bus-option-${index}`,
378+ directionId: `direction-${index}`,
379+ destinationRef: `terminal-bus-option-${index}`,
380+ }));
381+ const catalog = buildCatalog({ services: extraServices });
382+ const reader = SqliteCatalogReader.open(catalog.path);
383+
384+ const options = reader.listServices(catalog.placeIds.bus);
385+ assert.equal(options?.length, extraServices.length + 1);
386+});
387+
388+test("reload preserves stable IDs, distinguishes directions, rejects invalid candidates, and removes stale services", () => {
389+ const southbound: ServiceSeed = {
390+ key: "rer-south",
391+ placeKey: "rer",
392+ mode: "RER",
393+ stopLabel: "Châtelet – Les Halles",
394+ lineLabel: "B",
395+ destinationLabel: "Saint-Rémy-lès-Chevreuse",
396+ monitoringRef: "raw-shared-rer-station",
397+ lineRef: "raw-line-rer-b",
398+ directionId: "south",
399+ destinationRef: "terminal-rer-south",
400+ };
401+ const first = buildCatalog({ services: [southbound] });
402+ const manager = new CatalogManager();
403+ manager.reload(first.path);
404+
405+ const northId = first.serviceIds["rer-north"];
406+ const southId = first.serviceIds["rer-south"];
407+ assert.equal(Buffer.byteLength(northId, "utf8"), 47);
408+ assert.ok(Buffer.byteLength(northId, "utf8") <= LIMITS.idUtf8Bytes);
409+ assert.notEqual(northId, southId);
410+ assert.deepEqual(manager.resolveService(northId), {
411+ status: "RESOLVED",
412+ monitoringRef: "raw-shared-rer-station",
413+ lineRef: "raw-line-rer-b",
414+ directionId: "north",
415+ destinationRef: "terminal-rer-north",
416+ destinationLabel: "Aéroport Charles de Gaulle 2",
417+ });
418+ assert.deepEqual(manager.resolveService(southId), {
419+ status: "RESOLVED",
420+ monitoringRef: "raw-shared-rer-station",
421+ lineRef: "raw-line-rer-b",
422+ directionId: "south",
423+ destinationRef: "terminal-rer-south",
424+ destinationLabel: "Saint-Rémy-lès-Chevreuse",
425+ });
426+
427+ const invalid = buildCatalog({ catalogVersion: "2" });
428+ assert.throws(() => manager.reload(invalid.path), /Invalid catalog candidate/u);
429+ assert.equal(manager.resolveService(southId).status, "RESOLVED");
430+
431+ const renamedNorth: ServiceSeed = {
432+ ...BASE_SERVICES.find((service) => service.key === "rer-north")!,
433+ destinationLabel: "Aéroport CDG 2",
434+ };
435+ const replacement = buildCatalog({
436+ revision: "fixture-revision-b",
437+ omitServiceKeys: ["rer-north"],
438+ services: [renamedNorth],
439+ });
440+ manager.reload(replacement.path);
441+
442+ assert.equal(replacement.serviceIds["rer-north"], northId);
443+ assert.deepEqual(manager.resolveService(northId), {
444+ status: "RESOLVED",
445+ monitoringRef: "raw-shared-rer-station",
446+ lineRef: "raw-line-rer-b",
447+ directionId: "north",
448+ destinationRef: "terminal-rer-north",
449+ destinationLabel: "Aéroport CDG 2",
450+ });
451+ assert.deepEqual(
452+ manager.resolveService(southId),
453+ { status: "UNRESOLVED", code: "INVALID_SERVICE" },
454+ );
455+ assert.equal(manager.listServices("plc_missing"), undefined);
456+ assert.deepEqual(Object.keys(manager.listServices(replacement.placeIds.metro)![0]!).sort(), [
457+ "destinationLabel",
458+ "lineLabel",
459+ "serviceId",
460+ "stopLabel",
461+ ]);
462+});
463+
464+test("candidate validation rejects incomplete FTS coverage", () => {
465+ const invalid = buildCatalog({ omitSearchFor: "metro" });
466+ assert.throws(() => SqliteCatalogReader.open(invalid.path), /search consistency/u);
467+});
new file mode 100644
@@ -0,0 +1,467 @@
1+import test from "node:test";
2+import assert from "node:assert/strict";
3+import { createHash } from "node:crypto";
4+import { mkdtempSync, rmSync } from "node:fs";
5+import { tmpdir } from "node:os";
6+import { join } from "node:path";
7+import { DatabaseSync } from "node:sqlite";
8+import { LIMITS, type TransportMode } from "../../contracts/src/index.ts";
9+import { CatalogManager, SqliteCatalogReader } from "../src/catalog.ts";
10+
11+interface PlaceSeed {
12+ readonly key: string;
13+ readonly mode: TransportMode;
14+ readonly sourceCode: string;
15+ readonly stopLabel: string;
16+ readonly localityLabel?: string;
17+}
18+
19+interface ServiceSeed {
20+ readonly key: string;
21+ readonly placeKey: string;
22+ readonly mode: TransportMode;
23+ readonly stopLabel: string;
24+ readonly lineLabel: string;
25+ readonly destinationLabel: string;
26+ readonly monitoringRef: string;
27+ readonly lineRef: string;
28+ readonly directionId: string;
29+ readonly destinationRef: string;
30+}
31+
32+interface CatalogSeed {
33+ readonly revision?: string;
34+ readonly places?: readonly PlaceSeed[];
35+ readonly services?: readonly ServiceSeed[];
36+ readonly omitServiceKeys?: readonly string[];
37+ readonly catalogVersion?: string;
38+ readonly omitSearchFor?: string;
39+}
40+
41+interface BuiltCatalog {
42+ readonly path: string;
43+ readonly placeIds: Readonly<Record<string, string>>;
44+ readonly serviceIds: Readonly<Record<string, string>>;
45+}
46+
47+const temporaryDirectories: string[] = [];
48+
49+const BASE_PLACES: readonly PlaceSeed[] = [
50+ {
51+ key: "bus",
52+ mode: "BUS",
53+ sourceCode: "zda-bus-republique",
54+ stopLabel: "République",
55+ localityLabel: "Paris",
56+ },
57+ {
58+ key: "metro",
59+ mode: "METRO",
60+ sourceCode: "zda-metro-chatelet",
61+ stopLabel: "Châtelet",
62+ localityLabel: "Paris",
63+ },
64+ {
65+ key: "tram",
66+ mode: "TRAM",
67+ sourceCode: "zda-tram-versailles",
68+ stopLabel: "Porte de Versailles",
69+ localityLabel: "Paris",
70+ },
71+ {
72+ key: "rer",
73+ mode: "RER",
74+ sourceCode: "zda-rer-chatelet-halles",
75+ stopLabel: "Châtelet – Les Halles",
76+ localityLabel: "Paris",
77+ },
78+ {
79+ key: "transilien",
80+ mode: "TRANSILIEN",
81+ sourceCode: "zda-train-saint-lazare",
82+ stopLabel: "Saint-Lazare",
83+ localityLabel: "Paris",
84+ },
85+];
86+
87+const BASE_SERVICES: readonly ServiceSeed[] = [
88+ {
89+ key: "bus-east",
90+ placeKey: "bus",
91+ mode: "BUS",
92+ stopLabel: "République",
93+ lineLabel: "96",
94+ destinationLabel: "Porte des Lilas",
95+ monitoringRef: "raw-monitoring-bus-east",
96+ lineRef: "raw-line-bus-96",
97+ directionId: "east",
98+ destinationRef: "terminal-bus-east",
99+ },
100+ {
101+ key: "metro-south",
102+ placeKey: "metro",
103+ mode: "METRO",
104+ stopLabel: "Châtelet",
105+ lineLabel: "4",
106+ destinationLabel: "Bagneux – Lucie Aubrac",
107+ monitoringRef: "raw-monitoring-metro-south",
108+ lineRef: "raw-line-metro-4",
109+ directionId: "south",
110+ destinationRef: "terminal-metro-south",
111+ },
112+ {
113+ key: "tram-west",
114+ placeKey: "tram",
115+ mode: "TRAM",
116+ stopLabel: "Porte de Versailles",
117+ lineLabel: "T2",
118+ destinationLabel: "Porte de Versailles",
119+ monitoringRef: "raw-monitoring-tram-west",
120+ lineRef: "raw-line-tram-t2",
121+ directionId: "west",
122+ destinationRef: "terminal-tram-west",
123+ },
124+ {
125+ key: "rer-north",
126+ placeKey: "rer",
127+ mode: "RER",
128+ stopLabel: "Châtelet – Les Halles",
129+ lineLabel: "B",
130+ destinationLabel: "Aéroport Charles de Gaulle 2",
131+ monitoringRef: "raw-shared-rer-station",
132+ lineRef: "raw-line-rer-b",
133+ directionId: "north",
134+ destinationRef: "terminal-rer-north",
135+ },
136+ {
137+ key: "train-west",
138+ placeKey: "transilien",
139+ mode: "TRANSILIEN",
140+ stopLabel: "Saint-Lazare",
141+ lineLabel: "L",
142+ destinationLabel: "Versailles Rive Droite",
143+ monitoringRef: "raw-monitoring-train-west",
144+ lineRef: "raw-line-train-l",
145+ directionId: "west",
146+ destinationRef: "terminal-train-west",
147+ },
148+];
149+
150+function canonicalTuple(namespace: string, values: readonly string[]): string {
151+ const framed = values.map((value) => `${Buffer.byteLength(value, "utf8")}:${value}`).join("|");
152+ return `${namespace}:v1|${framed}`;
153+}
154+
155+function opaqueId(prefix: "plc_" | "svc_", canonical: string): string {
156+ return prefix + createHash("sha256").update(canonical).digest("base64url");
157+}
158+
159+function placeIdentity(place: PlaceSeed): { id: string; canonical: string } {
160+ const canonical = canonicalTuple("place", [place.mode, place.sourceCode]);
161+ return { id: opaqueId("plc_", canonical), canonical };
162+}
163+
164+function serviceIdentity(service: ServiceSeed): { id: string; canonical: string } {
165+ const canonical = canonicalTuple("service", [
166+ service.mode,
167+ service.lineRef,
168+ service.monitoringRef,
169+ service.directionId,
170+ service.destinationRef,
171+ ]);
172+ return { id: opaqueId("svc_", canonical), canonical };
173+}
174+
175+function searchText(place: PlaceSeed): string {
176+ return [place.stopLabel, place.localityLabel, place.mode]
177+ .filter((value): value is string => value !== undefined)
178+ .join(" ")
179+ .normalize("NFKD")
180+ .replace(/\p{M}+/gu, "")
181+ .toLowerCase()
182+ .replace(/[^\p{L}\p{N}]+/gu, " ")
183+ .trim()
184+ .replace(/\s+/gu, " ");
185+}
186+
187+function buildCatalog(seed: CatalogSeed = {}): BuiltCatalog {
188+ const directory = mkdtempSync(join(tmpdir(), "lapin-catalog-runtime-"));
189+ temporaryDirectories.push(directory);
190+ const path = join(directory, "catalog.sqlite");
191+ const database = new DatabaseSync(path);
192+ database.exec(`
193+ PRAGMA foreign_keys = ON;
194+ CREATE TABLE metadata(
195+ key TEXT PRIMARY KEY,
196+ value TEXT NOT NULL
197+ );
198+ CREATE TABLE places(
199+ place_id TEXT PRIMARY KEY,
200+ stop_label TEXT NOT NULL,
201+ locality_label TEXT,
202+ mode TEXT NOT NULL,
203+ canonical_tuple TEXT NOT NULL UNIQUE
204+ );
205+ CREATE TABLE services(
206+ service_id TEXT PRIMARY KEY,
207+ place_id TEXT NOT NULL REFERENCES places(place_id),
208+ stop_label TEXT NOT NULL,
209+ line_label TEXT NOT NULL,
210+ destination_label TEXT NOT NULL,
211+ monitoring_ref TEXT NOT NULL,
212+ line_ref TEXT NOT NULL,
213+ direction_id TEXT NOT NULL,
214+ destination_ref TEXT NOT NULL,
215+ canonical_tuple TEXT NOT NULL UNIQUE
216+ );
217+ CREATE VIRTUAL TABLE place_search USING fts5(search_text, place_id UNINDEXED);
218+ `);
219+
220+ const metadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
221+ metadata.run("schema_version", "1");
222+ metadata.run("catalog_version", seed.catalogVersion ?? "1");
223+ metadata.run("source_revision", seed.revision ?? "fixture-revision-a");
224+ metadata.run("created_at", "2026-08-30T12:00:00.000Z");
225+
226+ const places = [...BASE_PLACES, ...(seed.places ?? [])];
227+ const omittedServiceKeys = new Set(seed.omitServiceKeys ?? []);
228+ const services = [
229+ ...BASE_SERVICES.filter((service) => !omittedServiceKeys.has(service.key)),
230+ ...(seed.services ?? []),
231+ ];
232+ const placeIds: Record<string, string> = {};
233+ const serviceIds: Record<string, string> = {};
234+ const insertPlace = database.prepare(`
235+ INSERT INTO places(place_id, stop_label, locality_label, mode, canonical_tuple)
236+ VALUES (?, ?, ?, ?, ?)
237+ `);
238+ const insertSearch = database.prepare(
239+ "INSERT INTO place_search(search_text, place_id) VALUES (?, ?)",
240+ );
241+ for (const place of places) {
242+ const identity = placeIdentity(place);
243+ placeIds[place.key] = identity.id;
244+ insertPlace.run(
245+ identity.id,
246+ place.stopLabel,
247+ place.localityLabel ?? null,
248+ place.mode,
249+ identity.canonical,
250+ );
251+ if (place.key !== seed.omitSearchFor) insertSearch.run(searchText(place), identity.id);
252+ }
253+
254+ const insertService = database.prepare(`
255+ INSERT INTO services(
256+ service_id,
257+ place_id,
258+ stop_label,
259+ line_label,
260+ destination_label,
261+ monitoring_ref,
262+ line_ref,
263+ direction_id,
264+ destination_ref,
265+ canonical_tuple
266+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
267+ `);
268+ for (const service of services) {
269+ const identity = serviceIdentity(service);
270+ serviceIds[service.key] = identity.id;
271+ insertService.run(
272+ identity.id,
273+ placeIds[service.placeKey],
274+ service.stopLabel,
275+ service.lineLabel,
276+ service.destinationLabel,
277+ service.monitoringRef,
278+ service.lineRef,
279+ service.directionId,
280+ service.destinationRef,
281+ identity.canonical,
282+ );
283+ }
284+ database.close();
285+ return { path, placeIds, serviceIds };
286+}
287+
288+test.afterEach(() => {
289+ while (temporaryDirectories.length > 0) {
290+ const directory = temporaryDirectories.pop();
291+ if (directory !== undefined) rmSync(directory, { recursive: true, force: true });
292+ }
293+});
294+
295+test("search is accent tolerant, punctuation safe, and bounded by Unicode code points", () => {
296+ const catalog = buildCatalog();
297+ const reader = SqliteCatalogReader.open(catalog.path);
298+
299+ assert.deepEqual(
300+ reader.searchPlaces("chatelet").map((place) => place.placeId),
301+ [catalog.placeIds.metro, catalog.placeIds.rer],
302+ );
303+ assert.deepEqual(
304+ reader.searchPlaces(" ChÂtElEt---LES...HALLES ").map((place) => place.placeId),
305+ [catalog.placeIds.rer],
306+ );
307+ assert.throws(() => reader.searchPlaces("a"), RangeError);
308+ assert.throws(() => reader.searchPlaces("a".repeat(101)), RangeError);
309+ assert.throws(() => reader.searchPlaces("--"), RangeError);
310+ assert.doesNotThrow(() => reader.searchPlaces("😀a"));
311+ assert.doesNotThrow(() => reader.searchPlaces("z".repeat(100)));
312+});
313+
314+test("search preserves homonyms and applies deterministic ranking before the 20-row bound", () => {
315+ const homonyms: PlaceSeed[] = [
316+ {
317+ key: "homonym-b",
318+ mode: "BUS",
319+ sourceCode: "zda-homonym-b",
320+ stopLabel: "Place Centrale",
321+ localityLabel: "Saint-Denis",
322+ },
323+ {
324+ key: "homonym-a",
325+ mode: "BUS",
326+ sourceCode: "zda-homonym-a",
327+ stopLabel: "Place Centrale",
328+ localityLabel: "Saint-Denis",
329+ },
330+ ];
331+ const rankedPlaces: PlaceSeed[] = Array.from({ length: 25 }, (_, index) => ({
332+ key: `rank-${index}`,
333+ mode: "BUS" as const,
334+ sourceCode: `zda-rank-${index}`,
335+ stopLabel: `Alpha ${String(index).padStart(2, "0")}`,
336+ localityLabel: "Paris",
337+ }));
338+ const services: ServiceSeed[] = [...homonyms, ...rankedPlaces].map((place, index) => ({
339+ key: `extra-service-${index}`,
340+ placeKey: place.key,
341+ mode: place.mode,
342+ stopLabel: place.stopLabel,
343+ lineLabel: `X${index}`,
344+ destinationLabel: "Terminus",
345+ monitoringRef: `raw-monitoring-extra-${index}`,
346+ lineRef: `raw-line-extra-${index}`,
347+ directionId: "outbound",
348+ destinationRef: `terminal-extra-${index}`,
349+ }));
350+ const catalog = buildCatalog({ places: [...homonyms, ...rankedPlaces], services });
351+ const reader = SqliteCatalogReader.open(catalog.path);
352+
353+ const homonymResults = reader.searchPlaces("place centrale");
354+ assert.equal(homonymResults.length, 2);
355+ assert.deepEqual(
356+ homonymResults.map((place) => place.placeId),
357+ [catalog.placeIds["homonym-a"], catalog.placeIds["homonym-b"]].sort(),
358+ );
359+
360+ const ranked = reader.searchPlaces("alpha");
361+ assert.equal(ranked.length, LIMITS.catalogSearchResults);
362+ assert.deepEqual(
363+ ranked.map((place) => place.stopLabel),
364+ Array.from({ length: 20 }, (_, index) => `Alpha ${String(index).padStart(2, "0")}`),
365+ );
366+});
367+
368+test("service options return every valid direction without applying the place-search bound", () => {
369+ const extraServices: ServiceSeed[] = Array.from({ length: 25 }, (_, index) => ({
370+ key: `bus-option-${index}`,
371+ placeKey: "bus",
372+ mode: "BUS",
373+ stopLabel: "République",
374+ lineLabel: `X${String(index).padStart(2, "0")}`,
375+ destinationLabel: `Terminus ${index}`,
376+ monitoringRef: `raw-monitoring-bus-option-${index}`,
377+ lineRef: `raw-line-bus-option-${index}`,
378+ directionId: `direction-${index}`,
379+ destinationRef: `terminal-bus-option-${index}`,
380+ }));
381+ const catalog = buildCatalog({ services: extraServices });
382+ const reader = SqliteCatalogReader.open(catalog.path);
383+
384+ const options = reader.listServices(catalog.placeIds.bus);
385+ assert.equal(options?.length, extraServices.length + 1);
386+});
387+
388+test("reload preserves stable IDs, distinguishes directions, rejects invalid candidates, and removes stale services", () => {
389+ const southbound: ServiceSeed = {
390+ key: "rer-south",
391+ placeKey: "rer",
392+ mode: "RER",
393+ stopLabel: "Châtelet – Les Halles",
394+ lineLabel: "B",
395+ destinationLabel: "Saint-Rémy-lès-Chevreuse",
396+ monitoringRef: "raw-shared-rer-station",
397+ lineRef: "raw-line-rer-b",
398+ directionId: "south",
399+ destinationRef: "terminal-rer-south",
400+ };
401+ const first = buildCatalog({ services: [southbound] });
402+ const manager = new CatalogManager();
403+ manager.reload(first.path);
404+
405+ const northId = first.serviceIds["rer-north"];
406+ const southId = first.serviceIds["rer-south"];
407+ assert.equal(Buffer.byteLength(northId, "utf8"), 47);
408+ assert.ok(Buffer.byteLength(northId, "utf8") <= LIMITS.idUtf8Bytes);
409+ assert.notEqual(northId, southId);
410+ assert.deepEqual(manager.resolveService(northId), {
411+ status: "RESOLVED",
412+ monitoringRef: "raw-shared-rer-station",
413+ lineRef: "raw-line-rer-b",
414+ directionId: "north",
415+ destinationRef: "terminal-rer-north",
416+ destinationLabel: "Aéroport Charles de Gaulle 2",
417+ });
418+ assert.deepEqual(manager.resolveService(southId), {
419+ status: "RESOLVED",
420+ monitoringRef: "raw-shared-rer-station",
421+ lineRef: "raw-line-rer-b",
422+ directionId: "south",
423+ destinationRef: "terminal-rer-south",
424+ destinationLabel: "Saint-Rémy-lès-Chevreuse",
425+ });
426+
427+ const invalid = buildCatalog({ catalogVersion: "2" });
428+ assert.throws(() => manager.reload(invalid.path), /Invalid catalog candidate/u);
429+ assert.equal(manager.resolveService(southId).status, "RESOLVED");
430+
431+ const renamedNorth: ServiceSeed = {
432+ ...BASE_SERVICES.find((service) => service.key === "rer-north")!,
433+ destinationLabel: "Aéroport CDG 2",
434+ };
435+ const replacement = buildCatalog({
436+ revision: "fixture-revision-b",
437+ omitServiceKeys: ["rer-north"],
438+ services: [renamedNorth],
439+ });
440+ manager.reload(replacement.path);
441+
442+ assert.equal(replacement.serviceIds["rer-north"], northId);
443+ assert.deepEqual(manager.resolveService(northId), {
444+ status: "RESOLVED",
445+ monitoringRef: "raw-shared-rer-station",
446+ lineRef: "raw-line-rer-b",
447+ directionId: "north",
448+ destinationRef: "terminal-rer-north",
449+ destinationLabel: "Aéroport CDG 2",
450+ });
451+ assert.deepEqual(
452+ manager.resolveService(southId),
453+ { status: "UNRESOLVED", code: "INVALID_SERVICE" },
454+ );
455+ assert.equal(manager.listServices("plc_missing"), undefined);
456+ assert.deepEqual(Object.keys(manager.listServices(replacement.placeIds.metro)![0]!).sort(), [
457+ "destinationLabel",
458+ "lineLabel",
459+ "serviceId",
460+ "stopLabel",
461+ ]);
462+});
463+
464+test("candidate validation rejects incomplete FTS coverage", () => {
465+ const invalid = buildCatalog({ omitSearchFor: "metro" });
466+ assert.throws(() => SqliteCatalogReader.open(invalid.path), /search consistency/u);
467+});
modified packages/companion/src/contracts.js +4 -1
@@ -8,7 +8,10 @@ var LIMITS = Object.freeze({
88 favorites: 8,
99 departures: 4,
1010 httpTimeoutMs: 8000,
11- httpResponseBytes: 262144
11+ httpResponseBytes: 262144,
12+ catalogQueryMinCharacters: 2,
13+ catalogQueryMaxCharacters: 100,
14+ catalogSearchResults: 20
1215 });
1316 var CACHE_FRESH_SECONDS = 60;
1417 var FAVORITE_SETTLE_MS = 500;
@@ -8,7 +8,10 @@ var LIMITS = Object.freeze({
8 favorites: 8,8 favorites: 8,
9 departures: 4,9 departures: 4,
10 httpTimeoutMs: 8000,10 httpTimeoutMs: 8000,
11- httpResponseBytes: 26214411+ httpResponseBytes: 262144,
12+ catalogQueryMinCharacters: 2,
13+ catalogQueryMaxCharacters: 100,
14+ catalogSearchResults: 20
12 });15 });
13 var CACHE_FRESH_SECONDS = 60;16 var CACHE_FRESH_SECONDS = 60;
14 var FAVORITE_SETTLE_MS = 500;17 var FAVORITE_SETTLE_MS = 500;
modified packages/contracts/src/index.ts +80 -0
@@ -8,6 +8,9 @@ export const LIMITS = {
88 departures: 4,
99 httpTimeoutMs: 8_000,
1010 httpResponseBytes: 262_144,
11+ catalogQueryMinCharacters: 2,
12+ catalogQueryMaxCharacters: 100,
13+ catalogSearchResults: 20,
1114 } as const;
1215
1316 export const PRIM_ORIGIN = "https://prim.iledefrance-mobilites.fr" as const;
@@ -24,6 +27,9 @@ export const ERROR_CODE = [
2427 "RATE_LIMITED",
2528 "INVALID_RESPONSE",
2629 ] as const;
30+export const TRANSPORT_MODE = ["BUS", "METRO", "TRAM", "RER", "TRANSILIEN"] as const;
31+export const CATALOG_ERROR_CODE = ["INVALID_QUERY", "PLACE_NOT_FOUND", "CATALOG_UNAVAILABLE", "METHOD_NOT_ALLOWED"] as const;
32+export const CATALOG_ROUTE = { places: "/api/catalog/places", placeServices: "/api/catalog/places/:placeId/services" } as const;
2733 export const API_KEY_ACTION = ["KEEP", "REPLACE", "REMOVE"] as const;
2834 export const KEY_STATUS = { MISSING: 0, CONFIGURED: 1, INVALID: 2 } as const;
2935 export const REQUEST_TRIGGER = { APP_OPEN: 0, FAVORITE_SELECTION: 1, MANUAL_SELECT: 2 } as const;
@@ -97,6 +103,8 @@ export const WIRE_LANGUAGE = { EN: "en", FR: "fr" } as const;
97103 export type Freshness = (typeof FRESHNESS)[number];
98104 export type DepartureStatus = (typeof DEPARTURE_STATUS)[number];
99105 export type ErrorCode = (typeof ERROR_CODE)[number];
106+export type TransportMode = (typeof TRANSPORT_MODE)[number];
107+export type CatalogErrorCode = (typeof CATALOG_ERROR_CODE)[number];
100108 export type ApiKeyAction = (typeof API_KEY_ACTION)[number];
101109 export type AppMessageKey = (typeof APP_MESSAGE_KEY_ORDER)[number];
102110 export type WireLanguage = (typeof WIRE_LANGUAGE)[keyof typeof WIRE_LANGUAGE];
@@ -138,6 +146,35 @@ export interface ErrorResult {
138146 occurredAt: number;
139147 retryAfterSeconds?: number;
140148 }
149+export interface PlaceSearchItem {
150+ placeId: string;
151+ stopLabel: string;
152+ localityLabel?: string;
153+ mode: TransportMode;
154+}
155+
156+export interface PlaceSearchResult {
157+ schemaVersion: typeof SCHEMA_VERSION;
158+ places: PlaceSearchItem[];
159+}
160+
161+export interface ServiceOption {
162+ serviceId: string;
163+ stopLabel: string;
164+ lineLabel: string;
165+ destinationLabel: string;
166+}
167+
168+export interface ServiceOptionsResult {
169+ schemaVersion: typeof SCHEMA_VERSION;
170+ placeId: string;
171+ services: ServiceOption[];
172+}
173+
174+export interface CatalogErrorResult {
175+ schemaVersion: typeof SCHEMA_VERSION;
176+ code: CatalogErrorCode;
177+}
141178
142179 export type ApiKeyUpdate =
143180 | { schemaVersion: typeof SCHEMA_VERSION; action: "KEEP" }
@@ -155,6 +192,11 @@ const domainKeys = {
155192 result: ["schemaVersion", "requestId", "favoriteId", "fetchedAt", "sourceUpdatedAt", "freshness", "departures"],
156193 error: ["schemaVersion", "requestId", "favoriteId", "code", "occurredAt", "retryAfterSeconds"],
157194 keyUpdate: ["schemaVersion", "action", "value"],
195+ placeSearchItem: ["placeId", "stopLabel", "localityLabel", "mode"],
196+ placeSearchResult: ["schemaVersion", "places"],
197+ serviceOption: ["serviceId", "stopLabel", "lineLabel", "destinationLabel"],
198+ serviceOptionsResult: ["schemaVersion", "placeId", "services"],
199+ catalogError: ["schemaVersion", "code"],
158200 } as const;
159201
160202 function object(value: unknown): value is Record<string, unknown> {
@@ -234,6 +276,44 @@ export function isErrorResult(value: unknown): value is ErrorResult {
234276 && optionalUint32(value.retryAfterSeconds);
235277 }
236278
279+export function isPlaceSearchItem(value: unknown): value is PlaceSearchItem {
280+ if (!object(value) || !exactKeys(value, domainKeys.placeSearchItem)) return false;
281+ return boundedString(value.placeId, LIMITS.idUtf8Bytes)
282+ && boundedString(value.stopLabel, LIMITS.labelUtf8Bytes)
283+ && optionalBoundedString(value.localityLabel, LIMITS.labelUtf8Bytes)
284+ && TRANSPORT_MODE.includes(value.mode as TransportMode);
285+}
286+
287+export function isPlaceSearchResult(value: unknown): value is PlaceSearchResult {
288+ if (!object(value) || !exactKeys(value, domainKeys.placeSearchResult)) return false;
289+ return value.schemaVersion === SCHEMA_VERSION
290+ && Array.isArray(value.places)
291+ && value.places.length <= LIMITS.catalogSearchResults
292+ && value.places.every(isPlaceSearchItem);
293+}
294+
295+export function isServiceOption(value: unknown): value is ServiceOption {
296+ if (!object(value) || !exactKeys(value, domainKeys.serviceOption)) return false;
297+ return boundedString(value.serviceId, LIMITS.idUtf8Bytes)
298+ && boundedString(value.stopLabel, LIMITS.labelUtf8Bytes)
299+ && boundedString(value.lineLabel, LIMITS.labelUtf8Bytes)
300+ && boundedString(value.destinationLabel, LIMITS.labelUtf8Bytes);
301+}
302+
303+export function isServiceOptionsResult(value: unknown): value is ServiceOptionsResult {
304+ if (!object(value) || !exactKeys(value, domainKeys.serviceOptionsResult)) return false;
305+ return value.schemaVersion === SCHEMA_VERSION
306+ && boundedString(value.placeId, LIMITS.idUtf8Bytes)
307+ && Array.isArray(value.services)
308+ && value.services.every(isServiceOption);
309+}
310+
311+export function isCatalogErrorResult(value: unknown): value is CatalogErrorResult {
312+ if (!object(value) || !exactKeys(value, domainKeys.catalogError)) return false;
313+ return value.schemaVersion === SCHEMA_VERSION
314+ && CATALOG_ERROR_CODE.includes(value.code as CatalogErrorCode);
315+}
316+
237317 export function isApiKeyUpdate(value: unknown): value is ApiKeyUpdate {
238318 if (!object(value) || !exactKeys(value, domainKeys.keyUpdate) || value.schemaVersion !== SCHEMA_VERSION) return false;
239319 if (value.action === "REPLACE") return boundedString(value.value, LIMITS.apiKeyUtf8Bytes);
@@ -8,6 +8,9 @@ export const LIMITS = {
8 departures: 4,8 departures: 4,
9 httpTimeoutMs: 8_000,9 httpTimeoutMs: 8_000,
10 httpResponseBytes: 262_144,10 httpResponseBytes: 262_144,
11+ catalogQueryMinCharacters: 2,
12+ catalogQueryMaxCharacters: 100,
13+ catalogSearchResults: 20,
11 } as const;14 } as const;
12 15
13 export const PRIM_ORIGIN = "https://prim.iledefrance-mobilites.fr" as const;16 export const PRIM_ORIGIN = "https://prim.iledefrance-mobilites.fr" as const;
@@ -24,6 +27,9 @@ export const ERROR_CODE = [
24 "RATE_LIMITED",27 "RATE_LIMITED",
25 "INVALID_RESPONSE",28 "INVALID_RESPONSE",
26 ] as const;29 ] as const;
30+export const TRANSPORT_MODE = ["BUS", "METRO", "TRAM", "RER", "TRANSILIEN"] as const;
31+export const CATALOG_ERROR_CODE = ["INVALID_QUERY", "PLACE_NOT_FOUND", "CATALOG_UNAVAILABLE", "METHOD_NOT_ALLOWED"] as const;
32+export const CATALOG_ROUTE = { places: "/api/catalog/places", placeServices: "/api/catalog/places/:placeId/services" } as const;
27 export const API_KEY_ACTION = ["KEEP", "REPLACE", "REMOVE"] as const;33 export const API_KEY_ACTION = ["KEEP", "REPLACE", "REMOVE"] as const;
28 export const KEY_STATUS = { MISSING: 0, CONFIGURED: 1, INVALID: 2 } as const;34 export const KEY_STATUS = { MISSING: 0, CONFIGURED: 1, INVALID: 2 } as const;
29 export const REQUEST_TRIGGER = { APP_OPEN: 0, FAVORITE_SELECTION: 1, MANUAL_SELECT: 2 } as const;35 export const REQUEST_TRIGGER = { APP_OPEN: 0, FAVORITE_SELECTION: 1, MANUAL_SELECT: 2 } as const;
@@ -97,6 +103,8 @@ export const WIRE_LANGUAGE = { EN: "en", FR: "fr" } as const;
97 export type Freshness = (typeof FRESHNESS)[number];103 export type Freshness = (typeof FRESHNESS)[number];
98 export type DepartureStatus = (typeof DEPARTURE_STATUS)[number];104 export type DepartureStatus = (typeof DEPARTURE_STATUS)[number];
99 export type ErrorCode = (typeof ERROR_CODE)[number];105 export type ErrorCode = (typeof ERROR_CODE)[number];
106+export type TransportMode = (typeof TRANSPORT_MODE)[number];
107+export type CatalogErrorCode = (typeof CATALOG_ERROR_CODE)[number];
100 export type ApiKeyAction = (typeof API_KEY_ACTION)[number];108 export type ApiKeyAction = (typeof API_KEY_ACTION)[number];
101 export type AppMessageKey = (typeof APP_MESSAGE_KEY_ORDER)[number];109 export type AppMessageKey = (typeof APP_MESSAGE_KEY_ORDER)[number];
102 export type WireLanguage = (typeof WIRE_LANGUAGE)[keyof typeof WIRE_LANGUAGE];110 export type WireLanguage = (typeof WIRE_LANGUAGE)[keyof typeof WIRE_LANGUAGE];
@@ -138,6 +146,35 @@ export interface ErrorResult {
138 occurredAt: number;146 occurredAt: number;
139 retryAfterSeconds?: number;147 retryAfterSeconds?: number;
140 }148 }
149+export interface PlaceSearchItem {
150+ placeId: string;
151+ stopLabel: string;
152+ localityLabel?: string;
153+ mode: TransportMode;
154+}
155+
156+export interface PlaceSearchResult {
157+ schemaVersion: typeof SCHEMA_VERSION;
158+ places: PlaceSearchItem[];
159+}
160+
161+export interface ServiceOption {
162+ serviceId: string;
163+ stopLabel: string;
164+ lineLabel: string;
165+ destinationLabel: string;
166+}
167+
168+export interface ServiceOptionsResult {
169+ schemaVersion: typeof SCHEMA_VERSION;
170+ placeId: string;
171+ services: ServiceOption[];
172+}
173+
174+export interface CatalogErrorResult {
175+ schemaVersion: typeof SCHEMA_VERSION;
176+ code: CatalogErrorCode;
177+}
141 178
142 export type ApiKeyUpdate =179 export type ApiKeyUpdate =
143 | { schemaVersion: typeof SCHEMA_VERSION; action: "KEEP" }180 | { schemaVersion: typeof SCHEMA_VERSION; action: "KEEP" }
@@ -155,6 +192,11 @@ const domainKeys = {
155 result: ["schemaVersion", "requestId", "favoriteId", "fetchedAt", "sourceUpdatedAt", "freshness", "departures"],192 result: ["schemaVersion", "requestId", "favoriteId", "fetchedAt", "sourceUpdatedAt", "freshness", "departures"],
156 error: ["schemaVersion", "requestId", "favoriteId", "code", "occurredAt", "retryAfterSeconds"],193 error: ["schemaVersion", "requestId", "favoriteId", "code", "occurredAt", "retryAfterSeconds"],
157 keyUpdate: ["schemaVersion", "action", "value"],194 keyUpdate: ["schemaVersion", "action", "value"],
195+ placeSearchItem: ["placeId", "stopLabel", "localityLabel", "mode"],
196+ placeSearchResult: ["schemaVersion", "places"],
197+ serviceOption: ["serviceId", "stopLabel", "lineLabel", "destinationLabel"],
198+ serviceOptionsResult: ["schemaVersion", "placeId", "services"],
199+ catalogError: ["schemaVersion", "code"],
158 } as const;200 } as const;
159 201
160 function object(value: unknown): value is Record<string, unknown> {202 function object(value: unknown): value is Record<string, unknown> {
@@ -234,6 +276,44 @@ export function isErrorResult(value: unknown): value is ErrorResult {
234 && optionalUint32(value.retryAfterSeconds);276 && optionalUint32(value.retryAfterSeconds);
235 }277 }
236 278
279+export function isPlaceSearchItem(value: unknown): value is PlaceSearchItem {
280+ if (!object(value) || !exactKeys(value, domainKeys.placeSearchItem)) return false;
281+ return boundedString(value.placeId, LIMITS.idUtf8Bytes)
282+ && boundedString(value.stopLabel, LIMITS.labelUtf8Bytes)
283+ && optionalBoundedString(value.localityLabel, LIMITS.labelUtf8Bytes)
284+ && TRANSPORT_MODE.includes(value.mode as TransportMode);
285+}
286+
287+export function isPlaceSearchResult(value: unknown): value is PlaceSearchResult {
288+ if (!object(value) || !exactKeys(value, domainKeys.placeSearchResult)) return false;
289+ return value.schemaVersion === SCHEMA_VERSION
290+ && Array.isArray(value.places)
291+ && value.places.length <= LIMITS.catalogSearchResults
292+ && value.places.every(isPlaceSearchItem);
293+}
294+
295+export function isServiceOption(value: unknown): value is ServiceOption {
296+ if (!object(value) || !exactKeys(value, domainKeys.serviceOption)) return false;
297+ return boundedString(value.serviceId, LIMITS.idUtf8Bytes)
298+ && boundedString(value.stopLabel, LIMITS.labelUtf8Bytes)
299+ && boundedString(value.lineLabel, LIMITS.labelUtf8Bytes)
300+ && boundedString(value.destinationLabel, LIMITS.labelUtf8Bytes);
301+}
302+
303+export function isServiceOptionsResult(value: unknown): value is ServiceOptionsResult {
304+ if (!object(value) || !exactKeys(value, domainKeys.serviceOptionsResult)) return false;
305+ return value.schemaVersion === SCHEMA_VERSION
306+ && boundedString(value.placeId, LIMITS.idUtf8Bytes)
307+ && Array.isArray(value.services)
308+ && value.services.every(isServiceOption);
309+}
310+
311+export function isCatalogErrorResult(value: unknown): value is CatalogErrorResult {
312+ if (!object(value) || !exactKeys(value, domainKeys.catalogError)) return false;
313+ return value.schemaVersion === SCHEMA_VERSION
314+ && CATALOG_ERROR_CODE.includes(value.code as CatalogErrorCode);
315+}
316+
237 export function isApiKeyUpdate(value: unknown): value is ApiKeyUpdate {317 export function isApiKeyUpdate(value: unknown): value is ApiKeyUpdate {
238 if (!object(value) || !exactKeys(value, domainKeys.keyUpdate) || value.schemaVersion !== SCHEMA_VERSION) return false;318 if (!object(value) || !exactKeys(value, domainKeys.keyUpdate) || value.schemaVersion !== SCHEMA_VERSION) return false;
239 if (value.action === "REPLACE") return boundedString(value.value, LIMITS.apiKeyUtf8Bytes);319 if (value.action === "REPLACE") return boundedString(value.value, LIMITS.apiKeyUtf8Bytes);
modified packages/contracts/test/contracts.test.ts +99 -0
@@ -4,6 +4,8 @@ import {
44 API_KEY_ACTION,
55 APP_MESSAGE_KEY,
66 APP_MESSAGE_KEY_ORDER,
7+ CATALOG_ERROR_CODE,
8+ CATALOG_ROUTE,
79 DEPARTURE_STATUS,
810 ERROR_CODE,
911 FRESHNESS,
@@ -13,14 +15,20 @@ import {
1315 ProtocolReceiver,
1416 REQUEST_TRIGGER,
1517 SCHEMA_VERSION,
18+ TRANSPORT_MODE,
1619 cstringBytes,
1720 dictionaryBytes,
1821 isApiKeyUpdate,
1922 isAppMessage,
23+ isCatalogErrorResult,
2024 isDepartureResult,
2125 isErrorResult,
2226 isFavorite,
2327 isPersonalApiKey,
28+ isPlaceSearchItem,
29+ isPlaceSearchResult,
30+ isServiceOption,
31+ isServiceOptionsResult,
2432 utf8Bytes,
2533 type AppMessage,
2634 } from "../src/index.ts";
@@ -219,3 +227,94 @@ test("result staging requires request and favorite binding and preserves committ
219227 assert.equal(receiver.receive({ ...begin, SCHEMA_VERSION: 99 }), false);
220228 assert.equal(receiver.committed.result, committed);
221229 });
230+
231+test("catalog contracts freeze routes, transport modes, error codes, and limits", () => {
232+ assert.deepEqual(CATALOG_ROUTE, {
233+ places: "/api/catalog/places",
234+ placeServices: "/api/catalog/places/:placeId/services",
235+ });
236+ assert.equal(Object.values(CATALOG_ROUTE).every((route) => route.startsWith("/api/catalog/")), true);
237+ assert.deepEqual(TRANSPORT_MODE, ["BUS", "METRO", "TRAM", "RER", "TRANSILIEN"]);
238+ assert.deepEqual(CATALOG_ERROR_CODE, [
239+ "INVALID_QUERY",
240+ "PLACE_NOT_FOUND",
241+ "CATALOG_UNAVAILABLE",
242+ "METHOD_NOT_ALLOWED",
243+ ]);
244+ assert.equal(LIMITS.catalogQueryMinCharacters, 2);
245+ assert.equal(LIMITS.catalogQueryMaxCharacters, 100);
246+ assert.equal(LIMITS.catalogSearchResults, 20);
247+});
248+
249+test("catalog validators enforce exact public fields, bounds, and no raw source identifiers", () => {
250+ const opaque64 = "x".repeat(LIMITS.idUtf8Bytes);
251+ const placeItem = {
252+ placeId: `plc_${"A".repeat(43)}`,
253+ stopLabel: "Châtelet",
254+ localityLabel: "Paris",
255+ mode: "METRO",
256+ };
257+ const serviceOption = {
258+ serviceId: `svc_${"B".repeat(43)}`,
259+ stopLabel: "Châtelet",
260+ lineLabel: "Métro 1",
261+ destinationLabel: "La Défense",
262+ };
263+ const requiredPlace = { placeId: placeItem.placeId, stopLabel: placeItem.stopLabel, mode: placeItem.mode };
264+
265+ assert.equal(isPlaceSearchItem(placeItem), true);
266+ assert.equal(isPlaceSearchItem(requiredPlace), true);
267+ assert.equal(isPlaceSearchItem({ ...placeItem, localityLabel: undefined }), true);
268+ assert.equal(isPlaceSearchItem({ ...placeItem, placeId: opaque64 }), true);
269+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [placeItem] }), true);
270+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [] }), true);
271+ assert.equal(isServiceOption(serviceOption), true);
272+ assert.equal(isServiceOptionsResult({
273+ schemaVersion: SCHEMA_VERSION,
274+ placeId: placeItem.placeId,
275+ services: [serviceOption],
276+ }), true);
277+ CATALOG_ERROR_CODE.forEach((code) => {
278+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code }), true);
279+ });
280+
281+ assert.equal(isPlaceSearchItem({ ...placeItem, placeId: "x".repeat(LIMITS.idUtf8Bytes + 1) }), false);
282+ assert.equal(isPlaceSearchItem({ ...placeItem, stopLabel: "" }), false);
283+ assert.equal(isPlaceSearchItem({ ...placeItem, stopLabel: "é".repeat(LIMITS.labelUtf8Bytes / 2 + 1) }), false);
284+ assert.equal(isPlaceSearchItem({ ...placeItem, mode: "TER" }), false);
285+ assert.equal(isPlaceSearchItem({ ...placeItem, mode: "bus" }), false);
286+ assert.equal(isPlaceSearchItem({ ...placeItem, monitoringRef: "StopPoint:Q-1" }), false);
287+ assert.equal(isPlaceSearchItem({ ...requiredPlace, lineRef: "C-01234" }), false);
288+ assert.equal(isPlaceSearchItem({ ...requiredPlace, q: "chatelet" }), false);
289+
290+ const places20 = Array.from({ length: LIMITS.catalogSearchResults }, () => placeItem);
291+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: places20 }), true);
292+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [...places20, placeItem] }), false);
293+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [{ ...placeItem, unknown: true }] }), false);
294+ assert.equal(isPlaceSearchResult({ schemaVersion: 2, places: [] }), false);
295+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION }), false);
296+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [], extra: 1 }), false);
297+
298+ assert.equal(isServiceOption({ ...serviceOption, serviceId: opaque64 }), true);
299+ assert.equal(isServiceOption({ ...serviceOption, destinationLabel: "" }), false);
300+ assert.equal(isServiceOption({ ...serviceOption, directionId: 1 }), false);
301+ assert.equal(isServiceOption({ ...serviceOption, destinationRef: "Q-1" }), false);
302+ assert.equal(isServiceOption({ serviceId: opaque64, stopLabel: "Châtelet", lineLabel: "1" }), false);
303+ const servicesBeyondLimit = Array.from({ length: LIMITS.catalogSearchResults + 1 }, () => serviceOption);
304+ assert.equal(isServiceOptionsResult({
305+ schemaVersion: SCHEMA_VERSION,
306+ placeId: opaque64,
307+ services: servicesBeyondLimit,
308+ }), true);
309+ assert.equal(isServiceOptionsResult({
310+ schemaVersion: SCHEMA_VERSION,
311+ placeId: placeItem.placeId,
312+ services: servicesBeyondLimit,
313+ apiKey: "secret",
314+ }), false);
315+ assert.equal(isServiceOptionsResult({ schemaVersion: SCHEMA_VERSION, placeId: placeItem.placeId, services: "all" }), false);
316+
317+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code: "INVALID_SERVICE" }), false);
318+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY", requestId: "r" }), false);
319+ assert.equal(isCatalogErrorResult({ code: "INVALID_QUERY" }), false);
320+});
@@ -4,6 +4,8 @@ import {
4 API_KEY_ACTION,4 API_KEY_ACTION,
5 APP_MESSAGE_KEY,5 APP_MESSAGE_KEY,
6 APP_MESSAGE_KEY_ORDER,6 APP_MESSAGE_KEY_ORDER,
7+ CATALOG_ERROR_CODE,
8+ CATALOG_ROUTE,
7 DEPARTURE_STATUS,9 DEPARTURE_STATUS,
8 ERROR_CODE,10 ERROR_CODE,
9 FRESHNESS,11 FRESHNESS,
@@ -13,14 +15,20 @@ import {
13 ProtocolReceiver,15 ProtocolReceiver,
14 REQUEST_TRIGGER,16 REQUEST_TRIGGER,
15 SCHEMA_VERSION,17 SCHEMA_VERSION,
18+ TRANSPORT_MODE,
16 cstringBytes,19 cstringBytes,
17 dictionaryBytes,20 dictionaryBytes,
18 isApiKeyUpdate,21 isApiKeyUpdate,
19 isAppMessage,22 isAppMessage,
23+ isCatalogErrorResult,
20 isDepartureResult,24 isDepartureResult,
21 isErrorResult,25 isErrorResult,
22 isFavorite,26 isFavorite,
23 isPersonalApiKey,27 isPersonalApiKey,
28+ isPlaceSearchItem,
29+ isPlaceSearchResult,
30+ isServiceOption,
31+ isServiceOptionsResult,
24 utf8Bytes,32 utf8Bytes,
25 type AppMessage,33 type AppMessage,
26 } from "../src/index.ts";34 } from "../src/index.ts";
@@ -219,3 +227,94 @@ test("result staging requires request and favorite binding and preserves committ
219 assert.equal(receiver.receive({ ...begin, SCHEMA_VERSION: 99 }), false);227 assert.equal(receiver.receive({ ...begin, SCHEMA_VERSION: 99 }), false);
220 assert.equal(receiver.committed.result, committed);228 assert.equal(receiver.committed.result, committed);
221 });229 });
230+
231+test("catalog contracts freeze routes, transport modes, error codes, and limits", () => {
232+ assert.deepEqual(CATALOG_ROUTE, {
233+ places: "/api/catalog/places",
234+ placeServices: "/api/catalog/places/:placeId/services",
235+ });
236+ assert.equal(Object.values(CATALOG_ROUTE).every((route) => route.startsWith("/api/catalog/")), true);
237+ assert.deepEqual(TRANSPORT_MODE, ["BUS", "METRO", "TRAM", "RER", "TRANSILIEN"]);
238+ assert.deepEqual(CATALOG_ERROR_CODE, [
239+ "INVALID_QUERY",
240+ "PLACE_NOT_FOUND",
241+ "CATALOG_UNAVAILABLE",
242+ "METHOD_NOT_ALLOWED",
243+ ]);
244+ assert.equal(LIMITS.catalogQueryMinCharacters, 2);
245+ assert.equal(LIMITS.catalogQueryMaxCharacters, 100);
246+ assert.equal(LIMITS.catalogSearchResults, 20);
247+});
248+
249+test("catalog validators enforce exact public fields, bounds, and no raw source identifiers", () => {
250+ const opaque64 = "x".repeat(LIMITS.idUtf8Bytes);
251+ const placeItem = {
252+ placeId: `plc_${"A".repeat(43)}`,
253+ stopLabel: "Châtelet",
254+ localityLabel: "Paris",
255+ mode: "METRO",
256+ };
257+ const serviceOption = {
258+ serviceId: `svc_${"B".repeat(43)}`,
259+ stopLabel: "Châtelet",
260+ lineLabel: "Métro 1",
261+ destinationLabel: "La Défense",
262+ };
263+ const requiredPlace = { placeId: placeItem.placeId, stopLabel: placeItem.stopLabel, mode: placeItem.mode };
264+
265+ assert.equal(isPlaceSearchItem(placeItem), true);
266+ assert.equal(isPlaceSearchItem(requiredPlace), true);
267+ assert.equal(isPlaceSearchItem({ ...placeItem, localityLabel: undefined }), true);
268+ assert.equal(isPlaceSearchItem({ ...placeItem, placeId: opaque64 }), true);
269+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [placeItem] }), true);
270+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [] }), true);
271+ assert.equal(isServiceOption(serviceOption), true);
272+ assert.equal(isServiceOptionsResult({
273+ schemaVersion: SCHEMA_VERSION,
274+ placeId: placeItem.placeId,
275+ services: [serviceOption],
276+ }), true);
277+ CATALOG_ERROR_CODE.forEach((code) => {
278+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code }), true);
279+ });
280+
281+ assert.equal(isPlaceSearchItem({ ...placeItem, placeId: "x".repeat(LIMITS.idUtf8Bytes + 1) }), false);
282+ assert.equal(isPlaceSearchItem({ ...placeItem, stopLabel: "" }), false);
283+ assert.equal(isPlaceSearchItem({ ...placeItem, stopLabel: "é".repeat(LIMITS.labelUtf8Bytes / 2 + 1) }), false);
284+ assert.equal(isPlaceSearchItem({ ...placeItem, mode: "TER" }), false);
285+ assert.equal(isPlaceSearchItem({ ...placeItem, mode: "bus" }), false);
286+ assert.equal(isPlaceSearchItem({ ...placeItem, monitoringRef: "StopPoint:Q-1" }), false);
287+ assert.equal(isPlaceSearchItem({ ...requiredPlace, lineRef: "C-01234" }), false);
288+ assert.equal(isPlaceSearchItem({ ...requiredPlace, q: "chatelet" }), false);
289+
290+ const places20 = Array.from({ length: LIMITS.catalogSearchResults }, () => placeItem);
291+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: places20 }), true);
292+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [...places20, placeItem] }), false);
293+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [{ ...placeItem, unknown: true }] }), false);
294+ assert.equal(isPlaceSearchResult({ schemaVersion: 2, places: [] }), false);
295+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION }), false);
296+ assert.equal(isPlaceSearchResult({ schemaVersion: SCHEMA_VERSION, places: [], extra: 1 }), false);
297+
298+ assert.equal(isServiceOption({ ...serviceOption, serviceId: opaque64 }), true);
299+ assert.equal(isServiceOption({ ...serviceOption, destinationLabel: "" }), false);
300+ assert.equal(isServiceOption({ ...serviceOption, directionId: 1 }), false);
301+ assert.equal(isServiceOption({ ...serviceOption, destinationRef: "Q-1" }), false);
302+ assert.equal(isServiceOption({ serviceId: opaque64, stopLabel: "Châtelet", lineLabel: "1" }), false);
303+ const servicesBeyondLimit = Array.from({ length: LIMITS.catalogSearchResults + 1 }, () => serviceOption);
304+ assert.equal(isServiceOptionsResult({
305+ schemaVersion: SCHEMA_VERSION,
306+ placeId: opaque64,
307+ services: servicesBeyondLimit,
308+ }), true);
309+ assert.equal(isServiceOptionsResult({
310+ schemaVersion: SCHEMA_VERSION,
311+ placeId: placeItem.placeId,
312+ services: servicesBeyondLimit,
313+ apiKey: "secret",
314+ }), false);
315+ assert.equal(isServiceOptionsResult({ schemaVersion: SCHEMA_VERSION, placeId: placeItem.placeId, services: "all" }), false);
316+
317+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code: "INVALID_SERVICE" }), false);
318+ assert.equal(isCatalogErrorResult({ schemaVersion: SCHEMA_VERSION, code: "INVALID_QUERY", requestId: "r" }), false);
319+ assert.equal(isCatalogErrorResult({ code: "INVALID_QUERY" }), false);
320+});
added scripts/measure-catalog.mjs +262 -0
new file mode 100644
@@ -0,0 +1,262 @@
1+import {
2+ existsSync,
3+ mkdirSync,
4+ renameSync,
5+ rmSync,
6+ writeFileSync,
7+} from "node:fs";
8+import { dirname, join, resolve } from "node:path";
9+import { performance } from "node:perf_hooks";
10+import { DatabaseSync } from "node:sqlite";
11+import { fileURLToPath } from "node:url";
12+import { redactSecrets, SqliteCatalogReader } from "../packages/backend/src/index.ts";
13+import { normalizeCatalogSearchText } from "../packages/backend/src/catalog-import.ts";
14+import {
15+ LIMITS,
16+ TRANSPORT_MODE,
17+} from "../packages/contracts/src/index.ts";
18+
19+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
20+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
21+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
22+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/measure-catalog.json");
23+const QUERY_COUNT = 1_000;
24+const MIN_QUERY_COUNT = 1_000;
25+let executedQueries = 0;
26+
27+function record(value) {
28+ return typeof value === "object" && value !== null && !Array.isArray(value)
29+ ? value
30+ : undefined;
31+}
32+
33+function boundedPublicQuery(stopLabel, localityLabel) {
34+ const stop = typeof stopLabel === "string" ? stopLabel.trim() : "";
35+ let points = [...stop];
36+ if (points.length < LIMITS.catalogQueryMinCharacters) {
37+ const locality = typeof localityLabel === "string" ? localityLabel.trim() : "";
38+ points = [...`${stop} ${locality}`.trim()];
39+ }
40+ if (points.length < LIMITS.catalogQueryMinCharacters) return undefined;
41+ return points.slice(0, LIMITS.catalogQueryMaxCharacters).join("");
42+}
43+
44+function selectRepresentativeQueries(database) {
45+ const statement = database.prepare(`
46+ SELECT stop_label AS stopLabel, locality_label AS localityLabel
47+ FROM places
48+ WHERE mode = ?
49+ ORDER BY stop_label COLLATE NOCASE, place_id
50+ `);
51+ const representatives = [];
52+
53+ for (const mode of TRANSPORT_MODE) {
54+ let fallback;
55+ let accented;
56+ for (const rowValue of statement.iterate(mode)) {
57+ const row = record(rowValue);
58+ const query = boundedPublicQuery(row?.stopLabel, row?.localityLabel);
59+ if (query === undefined) continue;
60+ fallback ??= query;
61+ const withoutAccents = normalizeCatalogSearchText(query);
62+ if (withoutAccents !== "" && withoutAccents !== query) {
63+ accented = { mode, accented: query, unaccented: withoutAccents };
64+ break;
65+ }
66+ }
67+
68+ const query = accented ?? (fallback === undefined
69+ ? undefined
70+ : { mode, accented: fallback, unaccented: normalizeCatalogSearchText(fallback) });
71+ if (query === undefined) {
72+ throw new Error(`Active catalog has no usable public ${mode} label`);
73+ }
74+ representatives.push(query);
75+ }
76+
77+ if (!representatives.some((query) => query.accented !== query.unaccented)) {
78+ throw new Error("Active catalog has no accented public label for the measurement set");
79+ }
80+ return representatives;
81+}
82+
83+function catalogFacts() {
84+ const database = new DatabaseSync(ACTIVE_PATH, {
85+ readOnly: true,
86+ enableForeignKeyConstraints: true,
87+ enableDoubleQuotedStringLiterals: false,
88+ allowExtension: false,
89+ defensive: true,
90+ });
91+
92+ try {
93+ const metadataRows = database.prepare(`
94+ SELECT key, value
95+ FROM metadata
96+ WHERE key IN ('schema_version', 'catalog_version', 'source_revision', 'created_at')
97+ `).all();
98+ const metadata = Object.fromEntries(metadataRows.map((row) => [row.key, row.value]));
99+ for (const key of ["schema_version", "catalog_version", "source_revision", "created_at"]) {
100+ if (typeof metadata[key] !== "string" || metadata[key].trim() === "") {
101+ throw new Error(`Active catalog metadata is missing ${key}`);
102+ }
103+ }
104+
105+ const placeCount = Number(database.prepare("SELECT count(*) AS count FROM places").get().count);
106+ const serviceCount = Number(database.prepare("SELECT count(*) AS count FROM services").get().count);
107+ if (!Number.isSafeInteger(placeCount) || placeCount < 1
108+ || !Number.isSafeInteger(serviceCount) || serviceCount < 1) {
109+ throw new Error("Active catalog counts are invalid");
110+ }
111+
112+ const countsByMode = Object.fromEntries(TRANSPORT_MODE.map((mode) => [mode, 0]));
113+ for (const row of database.prepare(`
114+ SELECT places.mode AS mode, count(services.service_id) AS count
115+ FROM places
116+ JOIN services ON services.place_id = places.place_id
117+ GROUP BY places.mode
118+ `).all()) {
119+ if (Object.hasOwn(countsByMode, row.mode)) countsByMode[row.mode] = Number(row.count);
120+ }
121+ for (const mode of TRANSPORT_MODE) {
122+ if (!Number.isSafeInteger(countsByMode[mode]) || countsByMode[mode] < 1) {
123+ throw new Error(`Active catalog has no ${mode} services`);
124+ }
125+ }
126+
127+ return {
128+ metadata,
129+ placeCount,
130+ serviceCount,
131+ countsByMode,
132+ representatives: selectRepresentativeQueries(database),
133+ };
134+ } finally {
135+ database.close();
136+ }
137+}
138+
139+function percentile(sorted, percentage) {
140+ const index = Math.max(0, Math.ceil((percentage / 100) * sorted.length) - 1);
141+ return sorted[index];
142+}
143+
144+function milliseconds(value) {
145+ return Math.round(value * 1_000) / 1_000;
146+}
147+
148+function evidenceSecrets() {
149+ return [process.env.IDFM_DATASET_TOKEN, process.env.PRIM_API_KEY]
150+ .filter((value) => typeof value === "string" && value.length > 0);
151+}
152+
153+function writeEvidence(report) {
154+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
155+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
156+ const serialized = `${JSON.stringify(report, null, 2)}\n`;
157+ try {
158+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
159+ renameSync(temporaryPath, EVIDENCE_PATH);
160+ } finally {
161+ rmSync(temporaryPath, { force: true });
162+ }
163+ process.stdout.write(serialized);
164+}
165+
166+function measure() {
167+ if (!existsSync(ACTIVE_PATH)) {
168+ throw new Error("Active catalog is missing; run npm run catalog:refresh first");
169+ }
170+
171+ const facts = catalogFacts();
172+ const queries = facts.representatives.flatMap((entry) => [
173+ { mode: entry.mode, variant: "accented", value: entry.accented },
174+ { mode: entry.mode, variant: "unaccented", value: entry.unaccented },
175+ ]);
176+ const catalog = SqliteCatalogReader.open(ACTIVE_PATH);
177+
178+ let warmMaxRows = 0;
179+ for (const query of queries) {
180+ const results = catalog.searchPlaces(query.value);
181+ executedQueries += 1;
182+ if (results.length === 0) {
183+ throw new Error(`Representative ${query.mode} ${query.variant} query returned no place`);
184+ }
185+ warmMaxRows = Math.max(warmMaxRows, results.length);
186+ }
187+
188+ const latencies = new Array(QUERY_COUNT);
189+ let maxRows = warmMaxRows;
190+ for (let index = 0; index < QUERY_COUNT; index += 1) {
191+ const query = queries[index % queries.length];
192+ const started = performance.now();
193+ const results = catalog.searchPlaces(query.value);
194+ latencies[index] = performance.now() - started;
195+ executedQueries += 1;
196+ maxRows = Math.max(maxRows, results.length);
197+ }
198+
199+ const sorted = [...latencies].sort((left, right) => left - right);
200+ const p50 = percentile(sorted, 50);
201+ const p95 = percentile(sorted, 95);
202+ const maximum = sorted[sorted.length - 1];
203+ const memory = process.memoryUsage();
204+ const pass = executedQueries >= MIN_QUERY_COUNT
205+ && p95 <= 200
206+ && maxRows <= LIMITS.catalogSearchResults;
207+
208+ return {
209+ command: "measure:catalog",
210+ status: pass ? "ok" : "failed",
211+ measuredAt: new Date().toISOString(),
212+ catalog: {
213+ schemaVersion: facts.metadata.schema_version,
214+ catalogVersion: facts.metadata.catalog_version,
215+ sourceRevision: facts.metadata.source_revision,
216+ createdAt: facts.metadata.created_at,
217+ placeCount: facts.placeCount,
218+ serviceCount: facts.serviceCount,
219+ countsByMode: facts.countsByMode,
220+ },
221+ representativeQueries: facts.representatives,
222+ queryCount: executedQueries,
223+ p50Ms: milliseconds(p50),
224+ p95Ms: milliseconds(p95),
225+ maxMs: milliseconds(maximum),
226+ maxRows,
227+ maxRSSKiB: process.resourceUsage().maxRSS,
228+ heapUsedBytes: memory.heapUsed,
229+ heapTotalBytes: memory.heapTotal,
230+ gates: {
231+ queriesAtLeast: MIN_QUERY_COUNT,
232+ p95MsAtMost: 200,
233+ maxRowsAtMost: LIMITS.catalogSearchResults,
234+ },
235+ pass,
236+ };
237+}
238+
239+try {
240+ const report = measure();
241+ writeEvidence(report);
242+ if (!report.pass) process.exitCode = 1;
243+} catch (error) {
244+ const message = redactSecrets(
245+ error instanceof Error && error.message ? error.message : "Catalog measurement failed",
246+ evidenceSecrets(),
247+ );
248+ process.stderr.write(`measure:catalog failed: ${message}\n`);
249+ try {
250+ writeEvidence({
251+ command: "measure:catalog",
252+ status: "failed",
253+ failedAt: new Date().toISOString(),
254+ activePath: ACTIVE_PATH,
255+ queryCount: executedQueries,
256+ error: message,
257+ });
258+ } catch {
259+ // Reporting evidence must not mask the primary failure.
260+ }
261+ process.exitCode = 1;
262+}
new file mode 100644
@@ -0,0 +1,262 @@
1+import {
2+ existsSync,
3+ mkdirSync,
4+ renameSync,
5+ rmSync,
6+ writeFileSync,
7+} from "node:fs";
8+import { dirname, join, resolve } from "node:path";
9+import { performance } from "node:perf_hooks";
10+import { DatabaseSync } from "node:sqlite";
11+import { fileURLToPath } from "node:url";
12+import { redactSecrets, SqliteCatalogReader } from "../packages/backend/src/index.ts";
13+import { normalizeCatalogSearchText } from "../packages/backend/src/catalog-import.ts";
14+import {
15+ LIMITS,
16+ TRANSPORT_MODE,
17+} from "../packages/contracts/src/index.ts";
18+
19+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
20+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
21+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
22+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/measure-catalog.json");
23+const QUERY_COUNT = 1_000;
24+const MIN_QUERY_COUNT = 1_000;
25+let executedQueries = 0;
26+
27+function record(value) {
28+ return typeof value === "object" && value !== null && !Array.isArray(value)
29+ ? value
30+ : undefined;
31+}
32+
33+function boundedPublicQuery(stopLabel, localityLabel) {
34+ const stop = typeof stopLabel === "string" ? stopLabel.trim() : "";
35+ let points = [...stop];
36+ if (points.length < LIMITS.catalogQueryMinCharacters) {
37+ const locality = typeof localityLabel === "string" ? localityLabel.trim() : "";
38+ points = [...`${stop} ${locality}`.trim()];
39+ }
40+ if (points.length < LIMITS.catalogQueryMinCharacters) return undefined;
41+ return points.slice(0, LIMITS.catalogQueryMaxCharacters).join("");
42+}
43+
44+function selectRepresentativeQueries(database) {
45+ const statement = database.prepare(`
46+ SELECT stop_label AS stopLabel, locality_label AS localityLabel
47+ FROM places
48+ WHERE mode = ?
49+ ORDER BY stop_label COLLATE NOCASE, place_id
50+ `);
51+ const representatives = [];
52+
53+ for (const mode of TRANSPORT_MODE) {
54+ let fallback;
55+ let accented;
56+ for (const rowValue of statement.iterate(mode)) {
57+ const row = record(rowValue);
58+ const query = boundedPublicQuery(row?.stopLabel, row?.localityLabel);
59+ if (query === undefined) continue;
60+ fallback ??= query;
61+ const withoutAccents = normalizeCatalogSearchText(query);
62+ if (withoutAccents !== "" && withoutAccents !== query) {
63+ accented = { mode, accented: query, unaccented: withoutAccents };
64+ break;
65+ }
66+ }
67+
68+ const query = accented ?? (fallback === undefined
69+ ? undefined
70+ : { mode, accented: fallback, unaccented: normalizeCatalogSearchText(fallback) });
71+ if (query === undefined) {
72+ throw new Error(`Active catalog has no usable public ${mode} label`);
73+ }
74+ representatives.push(query);
75+ }
76+
77+ if (!representatives.some((query) => query.accented !== query.unaccented)) {
78+ throw new Error("Active catalog has no accented public label for the measurement set");
79+ }
80+ return representatives;
81+}
82+
83+function catalogFacts() {
84+ const database = new DatabaseSync(ACTIVE_PATH, {
85+ readOnly: true,
86+ enableForeignKeyConstraints: true,
87+ enableDoubleQuotedStringLiterals: false,
88+ allowExtension: false,
89+ defensive: true,
90+ });
91+
92+ try {
93+ const metadataRows = database.prepare(`
94+ SELECT key, value
95+ FROM metadata
96+ WHERE key IN ('schema_version', 'catalog_version', 'source_revision', 'created_at')
97+ `).all();
98+ const metadata = Object.fromEntries(metadataRows.map((row) => [row.key, row.value]));
99+ for (const key of ["schema_version", "catalog_version", "source_revision", "created_at"]) {
100+ if (typeof metadata[key] !== "string" || metadata[key].trim() === "") {
101+ throw new Error(`Active catalog metadata is missing ${key}`);
102+ }
103+ }
104+
105+ const placeCount = Number(database.prepare("SELECT count(*) AS count FROM places").get().count);
106+ const serviceCount = Number(database.prepare("SELECT count(*) AS count FROM services").get().count);
107+ if (!Number.isSafeInteger(placeCount) || placeCount < 1
108+ || !Number.isSafeInteger(serviceCount) || serviceCount < 1) {
109+ throw new Error("Active catalog counts are invalid");
110+ }
111+
112+ const countsByMode = Object.fromEntries(TRANSPORT_MODE.map((mode) => [mode, 0]));
113+ for (const row of database.prepare(`
114+ SELECT places.mode AS mode, count(services.service_id) AS count
115+ FROM places
116+ JOIN services ON services.place_id = places.place_id
117+ GROUP BY places.mode
118+ `).all()) {
119+ if (Object.hasOwn(countsByMode, row.mode)) countsByMode[row.mode] = Number(row.count);
120+ }
121+ for (const mode of TRANSPORT_MODE) {
122+ if (!Number.isSafeInteger(countsByMode[mode]) || countsByMode[mode] < 1) {
123+ throw new Error(`Active catalog has no ${mode} services`);
124+ }
125+ }
126+
127+ return {
128+ metadata,
129+ placeCount,
130+ serviceCount,
131+ countsByMode,
132+ representatives: selectRepresentativeQueries(database),
133+ };
134+ } finally {
135+ database.close();
136+ }
137+}
138+
139+function percentile(sorted, percentage) {
140+ const index = Math.max(0, Math.ceil((percentage / 100) * sorted.length) - 1);
141+ return sorted[index];
142+}
143+
144+function milliseconds(value) {
145+ return Math.round(value * 1_000) / 1_000;
146+}
147+
148+function evidenceSecrets() {
149+ return [process.env.IDFM_DATASET_TOKEN, process.env.PRIM_API_KEY]
150+ .filter((value) => typeof value === "string" && value.length > 0);
151+}
152+
153+function writeEvidence(report) {
154+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
155+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
156+ const serialized = `${JSON.stringify(report, null, 2)}\n`;
157+ try {
158+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
159+ renameSync(temporaryPath, EVIDENCE_PATH);
160+ } finally {
161+ rmSync(temporaryPath, { force: true });
162+ }
163+ process.stdout.write(serialized);
164+}
165+
166+function measure() {
167+ if (!existsSync(ACTIVE_PATH)) {
168+ throw new Error("Active catalog is missing; run npm run catalog:refresh first");
169+ }
170+
171+ const facts = catalogFacts();
172+ const queries = facts.representatives.flatMap((entry) => [
173+ { mode: entry.mode, variant: "accented", value: entry.accented },
174+ { mode: entry.mode, variant: "unaccented", value: entry.unaccented },
175+ ]);
176+ const catalog = SqliteCatalogReader.open(ACTIVE_PATH);
177+
178+ let warmMaxRows = 0;
179+ for (const query of queries) {
180+ const results = catalog.searchPlaces(query.value);
181+ executedQueries += 1;
182+ if (results.length === 0) {
183+ throw new Error(`Representative ${query.mode} ${query.variant} query returned no place`);
184+ }
185+ warmMaxRows = Math.max(warmMaxRows, results.length);
186+ }
187+
188+ const latencies = new Array(QUERY_COUNT);
189+ let maxRows = warmMaxRows;
190+ for (let index = 0; index < QUERY_COUNT; index += 1) {
191+ const query = queries[index % queries.length];
192+ const started = performance.now();
193+ const results = catalog.searchPlaces(query.value);
194+ latencies[index] = performance.now() - started;
195+ executedQueries += 1;
196+ maxRows = Math.max(maxRows, results.length);
197+ }
198+
199+ const sorted = [...latencies].sort((left, right) => left - right);
200+ const p50 = percentile(sorted, 50);
201+ const p95 = percentile(sorted, 95);
202+ const maximum = sorted[sorted.length - 1];
203+ const memory = process.memoryUsage();
204+ const pass = executedQueries >= MIN_QUERY_COUNT
205+ && p95 <= 200
206+ && maxRows <= LIMITS.catalogSearchResults;
207+
208+ return {
209+ command: "measure:catalog",
210+ status: pass ? "ok" : "failed",
211+ measuredAt: new Date().toISOString(),
212+ catalog: {
213+ schemaVersion: facts.metadata.schema_version,
214+ catalogVersion: facts.metadata.catalog_version,
215+ sourceRevision: facts.metadata.source_revision,
216+ createdAt: facts.metadata.created_at,
217+ placeCount: facts.placeCount,
218+ serviceCount: facts.serviceCount,
219+ countsByMode: facts.countsByMode,
220+ },
221+ representativeQueries: facts.representatives,
222+ queryCount: executedQueries,
223+ p50Ms: milliseconds(p50),
224+ p95Ms: milliseconds(p95),
225+ maxMs: milliseconds(maximum),
226+ maxRows,
227+ maxRSSKiB: process.resourceUsage().maxRSS,
228+ heapUsedBytes: memory.heapUsed,
229+ heapTotalBytes: memory.heapTotal,
230+ gates: {
231+ queriesAtLeast: MIN_QUERY_COUNT,
232+ p95MsAtMost: 200,
233+ maxRowsAtMost: LIMITS.catalogSearchResults,
234+ },
235+ pass,
236+ };
237+}
238+
239+try {
240+ const report = measure();
241+ writeEvidence(report);
242+ if (!report.pass) process.exitCode = 1;
243+} catch (error) {
244+ const message = redactSecrets(
245+ error instanceof Error && error.message ? error.message : "Catalog measurement failed",
246+ evidenceSecrets(),
247+ );
248+ process.stderr.write(`measure:catalog failed: ${message}\n`);
249+ try {
250+ writeEvidence({
251+ command: "measure:catalog",
252+ status: "failed",
253+ failedAt: new Date().toISOString(),
254+ activePath: ACTIVE_PATH,
255+ queryCount: executedQueries,
256+ error: message,
257+ });
258+ } catch {
259+ // Reporting evidence must not mask the primary failure.
260+ }
261+ process.exitCode = 1;
262+}
added scripts/probe-catalog.mjs +275 -0
new file mode 100644
@@ -0,0 +1,275 @@
1+import {
2+ existsSync,
3+ mkdirSync,
4+ renameSync,
5+ rmSync,
6+ writeFileSync,
7+} from "node:fs";
8+import { dirname, join, resolve } from "node:path";
9+import { DatabaseSync } from "node:sqlite";
10+import { fileURLToPath } from "node:url";
11+import {
12+ parseBearerAuthorization,
13+ redactSecrets,
14+ relayPrimRequest,
15+} from "../packages/backend/src/index.ts";
16+import {
17+ PRIM_ORIGIN,
18+ TRANSPORT_MODE,
19+} from "../packages/contracts/src/index.ts";
20+
21+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
22+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
23+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
24+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/probe-catalog.json");
25+const STOP_MONITORING_PATH = "/marketplace/stop-monitoring";
26+const RETURNED_FIELD_NAME = /^(?:Direction|Destination)[A-Za-z0-9_]{0,48}$/u;
27+let callCount = 0;
28+
29+function record(value) {
30+ return typeof value === "object" && value !== null && !Array.isArray(value)
31+ ? value
32+ : undefined;
33+}
34+
35+function requiredText(value, field) {
36+ if (typeof value !== "string" || value.trim() === "" || value.includes("\0")) {
37+ throw new Error(`Active catalog has an invalid ${field}`);
38+ }
39+ return value;
40+}
41+
42+function siriText(value) {
43+ if (typeof value === "string") return value;
44+ const object = record(value);
45+ return typeof object?.value === "string" ? object.value : undefined;
46+}
47+
48+function validateSiri(bytes, expectedLineRef) {
49+ let root;
50+ try {
51+ const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
52+ root = record(JSON.parse(text));
53+ } catch {
54+ return {
55+ siriValid: false,
56+ lineRefMatched: false,
57+ directionFields: [],
58+ destinationFields: [],
59+ };
60+ }
61+
62+ const siri = record(root?.Siri) ?? root;
63+ const serviceDelivery = record(siri?.ServiceDelivery);
64+ const stopMonitoringDelivery = serviceDelivery?.StopMonitoringDelivery;
65+ if (serviceDelivery === undefined || !Array.isArray(stopMonitoringDelivery)) {
66+ return {
67+ siriValid: false,
68+ lineRefMatched: false,
69+ directionFields: [],
70+ destinationFields: [],
71+ };
72+ }
73+
74+ let validDelivery = stopMonitoringDelivery.length > 0;
75+ let lineRefMatched = false;
76+ const directionFields = new Set();
77+ const destinationFields = new Set();
78+
79+ for (const deliveryValue of stopMonitoringDelivery) {
80+ const delivery = record(deliveryValue);
81+ if (delivery === undefined) {
82+ validDelivery = false;
83+ continue;
84+ }
85+ const visitsValue = delivery.MonitoredStopVisit;
86+ if (visitsValue !== undefined && !Array.isArray(visitsValue)) {
87+ validDelivery = false;
88+ continue;
89+ }
90+
91+ for (const visitValue of visitsValue ?? []) {
92+ const visit = record(visitValue);
93+ const journey = record(visit?.MonitoredVehicleJourney);
94+ if (journey === undefined) {
95+ validDelivery = false;
96+ continue;
97+ }
98+ if (siriText(journey.LineRef) !== expectedLineRef) continue;
99+ lineRefMatched = true;
100+ for (const field of Object.keys(journey)) {
101+ if (!RETURNED_FIELD_NAME.test(field)) continue;
102+ if (field.startsWith("Direction")) directionFields.add(field);
103+ if (field.startsWith("Destination")) destinationFields.add(field);
104+ }
105+ }
106+ }
107+
108+ return {
109+ siriValid: validDelivery,
110+ lineRefMatched,
111+ directionFields: [...directionFields].sort(),
112+ destinationFields: [...destinationFields].sort(),
113+ };
114+}
115+
116+function selectServices() {
117+ if (!existsSync(ACTIVE_PATH)) {
118+ throw new Error("Active catalog is missing; run npm run catalog:refresh first");
119+ }
120+
121+ const database = new DatabaseSync(ACTIVE_PATH, {
122+ readOnly: true,
123+ enableForeignKeyConstraints: true,
124+ enableDoubleQuotedStringLiterals: false,
125+ allowExtension: false,
126+ defensive: true,
127+ });
128+ try {
129+ const select = database.prepare(`
130+ SELECT
131+ min(services.service_id) AS serviceId,
132+ services.monitoring_ref AS monitoringRef,
133+ services.line_ref AS lineRef
134+ FROM services
135+ JOIN places ON places.place_id = services.place_id
136+ WHERE places.mode = ?
137+ GROUP BY services.monitoring_ref, services.line_ref
138+ ORDER BY count(*) DESC, min(services.service_id)
139+ LIMIT 1
140+ `);
141+
142+ return TRANSPORT_MODE.map((mode) => {
143+ const row = record(select.get(mode));
144+ if (row === undefined) throw new Error(`Active catalog has no resolved ${mode} service`);
145+ return {
146+ mode,
147+ serviceId: requiredText(row.serviceId, `${mode} serviceId`),
148+ monitoringRef: requiredText(row.monitoringRef, `${mode} monitoringRef`),
149+ lineRef: requiredText(row.lineRef, `${mode} lineRef`),
150+ };
151+ });
152+ } finally {
153+ database.close();
154+ }
155+}
156+
157+function targetFor(monitoringRef) {
158+ const target = new URL(STOP_MONITORING_PATH, PRIM_ORIGIN);
159+ target.searchParams.set("MonitoringRef", monitoringRef);
160+ return target;
161+}
162+
163+function writeEvidence(report, apiKey) {
164+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
165+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
166+ const serialized = `${redactSecrets(JSON.stringify(report, null, 2), [apiKey])}\n`;
167+ try {
168+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
169+ renameSync(temporaryPath, EVIDENCE_PATH);
170+ } finally {
171+ rmSync(temporaryPath, { force: true });
172+ }
173+ process.stdout.write(serialized);
174+}
175+
176+async function probe(apiKey) {
177+ const services = selectServices();
178+ if (services.length !== TRANSPORT_MODE.length) {
179+ throw new Error("Active catalog did not resolve exactly one service per mode");
180+ }
181+ const requests = services.map((service) => ({
182+ ...service,
183+ target: targetFor(service.monitoringRef),
184+ }));
185+
186+ const countedFetch = (target, init) => {
187+ callCount += 1;
188+ return globalThis.fetch(target, init);
189+ };
190+ const probes = [];
191+
192+ for (const request of requests) {
193+ let status = null;
194+ let validation = {
195+ siriValid: false,
196+ lineRefMatched: false,
197+ directionFields: [],
198+ destinationFields: [],
199+ };
200+ try {
201+ const response = await relayPrimRequest(
202+ {
203+ target: request.target,
204+ authorization: `Bearer ${apiKey}`,
205+ },
206+ { fetch: countedFetch },
207+ );
208+ status = response.status;
209+ if (status === 200) validation = validateSiri(response.bytes, request.lineRef);
210+ } catch {
211+ // A failed call is recorded without response content; the probe still spends exactly one call per mode.
212+ }
213+
214+ probes.push({
215+ mode: request.mode,
216+ status,
217+ siriValid: validation.siriValid,
218+ lineRefMatched: validation.lineRefMatched,
219+ directionFields: validation.directionFields,
220+ destinationFields: validation.destinationFields,
221+ acceptedStoredQuery: status === 200
222+ && validation.siriValid
223+ && validation.lineRefMatched,
224+ });
225+ }
226+
227+ const pass = callCount === TRANSPORT_MODE.length
228+ && probes.every((result) => result.acceptedStoredQuery);
229+ return {
230+ command: "catalog:probe",
231+ status: pass ? "ok" : "failed",
232+ probedAt: new Date().toISOString(),
233+ requestStructure: {
234+ origin: PRIM_ORIGIN,
235+ path: STOP_MONITORING_PATH,
236+ queryParameters: ["MonitoringRef"],
237+ headers: ["apikey"],
238+ },
239+ callCount,
240+ probes,
241+ pass,
242+ };
243+}
244+
245+const configuredKey = process.env.PRIM_API_KEY;
246+if (typeof configuredKey !== "string" || configuredKey.length === 0) {
247+ process.stderr.write("catalog:probe requires PRIM_API_KEY\n");
248+ process.exitCode = 1;
249+} else {
250+ let apiKey;
251+ try {
252+ apiKey = parseBearerAuthorization(`Bearer ${configuredKey}`);
253+ const report = await probe(apiKey);
254+ writeEvidence(report, apiKey);
255+ if (!report.pass) process.exitCode = 1;
256+ } catch (error) {
257+ const message = redactSecrets(
258+ error instanceof Error && error.message ? error.message : "Catalog probe failed",
259+ [configuredKey],
260+ );
261+ process.stderr.write(`catalog:probe failed: ${message}\n`);
262+ try {
263+ writeEvidence({
264+ command: "catalog:probe",
265+ status: "failed",
266+ failedAt: new Date().toISOString(),
267+ callCount,
268+ error: message,
269+ }, configuredKey);
270+ } catch {
271+ // Reporting evidence must not mask the primary failure.
272+ }
273+ process.exitCode = 1;
274+ }
275+}
new file mode 100644
@@ -0,0 +1,275 @@
1+import {
2+ existsSync,
3+ mkdirSync,
4+ renameSync,
5+ rmSync,
6+ writeFileSync,
7+} from "node:fs";
8+import { dirname, join, resolve } from "node:path";
9+import { DatabaseSync } from "node:sqlite";
10+import { fileURLToPath } from "node:url";
11+import {
12+ parseBearerAuthorization,
13+ redactSecrets,
14+ relayPrimRequest,
15+} from "../packages/backend/src/index.ts";
16+import {
17+ PRIM_ORIGIN,
18+ TRANSPORT_MODE,
19+} from "../packages/contracts/src/index.ts";
20+
21+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
22+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
23+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
24+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/probe-catalog.json");
25+const STOP_MONITORING_PATH = "/marketplace/stop-monitoring";
26+const RETURNED_FIELD_NAME = /^(?:Direction|Destination)[A-Za-z0-9_]{0,48}$/u;
27+let callCount = 0;
28+
29+function record(value) {
30+ return typeof value === "object" && value !== null && !Array.isArray(value)
31+ ? value
32+ : undefined;
33+}
34+
35+function requiredText(value, field) {
36+ if (typeof value !== "string" || value.trim() === "" || value.includes("\0")) {
37+ throw new Error(`Active catalog has an invalid ${field}`);
38+ }
39+ return value;
40+}
41+
42+function siriText(value) {
43+ if (typeof value === "string") return value;
44+ const object = record(value);
45+ return typeof object?.value === "string" ? object.value : undefined;
46+}
47+
48+function validateSiri(bytes, expectedLineRef) {
49+ let root;
50+ try {
51+ const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
52+ root = record(JSON.parse(text));
53+ } catch {
54+ return {
55+ siriValid: false,
56+ lineRefMatched: false,
57+ directionFields: [],
58+ destinationFields: [],
59+ };
60+ }
61+
62+ const siri = record(root?.Siri) ?? root;
63+ const serviceDelivery = record(siri?.ServiceDelivery);
64+ const stopMonitoringDelivery = serviceDelivery?.StopMonitoringDelivery;
65+ if (serviceDelivery === undefined || !Array.isArray(stopMonitoringDelivery)) {
66+ return {
67+ siriValid: false,
68+ lineRefMatched: false,
69+ directionFields: [],
70+ destinationFields: [],
71+ };
72+ }
73+
74+ let validDelivery = stopMonitoringDelivery.length > 0;
75+ let lineRefMatched = false;
76+ const directionFields = new Set();
77+ const destinationFields = new Set();
78+
79+ for (const deliveryValue of stopMonitoringDelivery) {
80+ const delivery = record(deliveryValue);
81+ if (delivery === undefined) {
82+ validDelivery = false;
83+ continue;
84+ }
85+ const visitsValue = delivery.MonitoredStopVisit;
86+ if (visitsValue !== undefined && !Array.isArray(visitsValue)) {
87+ validDelivery = false;
88+ continue;
89+ }
90+
91+ for (const visitValue of visitsValue ?? []) {
92+ const visit = record(visitValue);
93+ const journey = record(visit?.MonitoredVehicleJourney);
94+ if (journey === undefined) {
95+ validDelivery = false;
96+ continue;
97+ }
98+ if (siriText(journey.LineRef) !== expectedLineRef) continue;
99+ lineRefMatched = true;
100+ for (const field of Object.keys(journey)) {
101+ if (!RETURNED_FIELD_NAME.test(field)) continue;
102+ if (field.startsWith("Direction")) directionFields.add(field);
103+ if (field.startsWith("Destination")) destinationFields.add(field);
104+ }
105+ }
106+ }
107+
108+ return {
109+ siriValid: validDelivery,
110+ lineRefMatched,
111+ directionFields: [...directionFields].sort(),
112+ destinationFields: [...destinationFields].sort(),
113+ };
114+}
115+
116+function selectServices() {
117+ if (!existsSync(ACTIVE_PATH)) {
118+ throw new Error("Active catalog is missing; run npm run catalog:refresh first");
119+ }
120+
121+ const database = new DatabaseSync(ACTIVE_PATH, {
122+ readOnly: true,
123+ enableForeignKeyConstraints: true,
124+ enableDoubleQuotedStringLiterals: false,
125+ allowExtension: false,
126+ defensive: true,
127+ });
128+ try {
129+ const select = database.prepare(`
130+ SELECT
131+ min(services.service_id) AS serviceId,
132+ services.monitoring_ref AS monitoringRef,
133+ services.line_ref AS lineRef
134+ FROM services
135+ JOIN places ON places.place_id = services.place_id
136+ WHERE places.mode = ?
137+ GROUP BY services.monitoring_ref, services.line_ref
138+ ORDER BY count(*) DESC, min(services.service_id)
139+ LIMIT 1
140+ `);
141+
142+ return TRANSPORT_MODE.map((mode) => {
143+ const row = record(select.get(mode));
144+ if (row === undefined) throw new Error(`Active catalog has no resolved ${mode} service`);
145+ return {
146+ mode,
147+ serviceId: requiredText(row.serviceId, `${mode} serviceId`),
148+ monitoringRef: requiredText(row.monitoringRef, `${mode} monitoringRef`),
149+ lineRef: requiredText(row.lineRef, `${mode} lineRef`),
150+ };
151+ });
152+ } finally {
153+ database.close();
154+ }
155+}
156+
157+function targetFor(monitoringRef) {
158+ const target = new URL(STOP_MONITORING_PATH, PRIM_ORIGIN);
159+ target.searchParams.set("MonitoringRef", monitoringRef);
160+ return target;
161+}
162+
163+function writeEvidence(report, apiKey) {
164+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
165+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
166+ const serialized = `${redactSecrets(JSON.stringify(report, null, 2), [apiKey])}\n`;
167+ try {
168+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
169+ renameSync(temporaryPath, EVIDENCE_PATH);
170+ } finally {
171+ rmSync(temporaryPath, { force: true });
172+ }
173+ process.stdout.write(serialized);
174+}
175+
176+async function probe(apiKey) {
177+ const services = selectServices();
178+ if (services.length !== TRANSPORT_MODE.length) {
179+ throw new Error("Active catalog did not resolve exactly one service per mode");
180+ }
181+ const requests = services.map((service) => ({
182+ ...service,
183+ target: targetFor(service.monitoringRef),
184+ }));
185+
186+ const countedFetch = (target, init) => {
187+ callCount += 1;
188+ return globalThis.fetch(target, init);
189+ };
190+ const probes = [];
191+
192+ for (const request of requests) {
193+ let status = null;
194+ let validation = {
195+ siriValid: false,
196+ lineRefMatched: false,
197+ directionFields: [],
198+ destinationFields: [],
199+ };
200+ try {
201+ const response = await relayPrimRequest(
202+ {
203+ target: request.target,
204+ authorization: `Bearer ${apiKey}`,
205+ },
206+ { fetch: countedFetch },
207+ );
208+ status = response.status;
209+ if (status === 200) validation = validateSiri(response.bytes, request.lineRef);
210+ } catch {
211+ // A failed call is recorded without response content; the probe still spends exactly one call per mode.
212+ }
213+
214+ probes.push({
215+ mode: request.mode,
216+ status,
217+ siriValid: validation.siriValid,
218+ lineRefMatched: validation.lineRefMatched,
219+ directionFields: validation.directionFields,
220+ destinationFields: validation.destinationFields,
221+ acceptedStoredQuery: status === 200
222+ && validation.siriValid
223+ && validation.lineRefMatched,
224+ });
225+ }
226+
227+ const pass = callCount === TRANSPORT_MODE.length
228+ && probes.every((result) => result.acceptedStoredQuery);
229+ return {
230+ command: "catalog:probe",
231+ status: pass ? "ok" : "failed",
232+ probedAt: new Date().toISOString(),
233+ requestStructure: {
234+ origin: PRIM_ORIGIN,
235+ path: STOP_MONITORING_PATH,
236+ queryParameters: ["MonitoringRef"],
237+ headers: ["apikey"],
238+ },
239+ callCount,
240+ probes,
241+ pass,
242+ };
243+}
244+
245+const configuredKey = process.env.PRIM_API_KEY;
246+if (typeof configuredKey !== "string" || configuredKey.length === 0) {
247+ process.stderr.write("catalog:probe requires PRIM_API_KEY\n");
248+ process.exitCode = 1;
249+} else {
250+ let apiKey;
251+ try {
252+ apiKey = parseBearerAuthorization(`Bearer ${configuredKey}`);
253+ const report = await probe(apiKey);
254+ writeEvidence(report, apiKey);
255+ if (!report.pass) process.exitCode = 1;
256+ } catch (error) {
257+ const message = redactSecrets(
258+ error instanceof Error && error.message ? error.message : "Catalog probe failed",
259+ [configuredKey],
260+ );
261+ process.stderr.write(`catalog:probe failed: ${message}\n`);
262+ try {
263+ writeEvidence({
264+ command: "catalog:probe",
265+ status: "failed",
266+ failedAt: new Date().toISOString(),
267+ callCount,
268+ error: message,
269+ }, configuredKey);
270+ } catch {
271+ // Reporting evidence must not mask the primary failure.
272+ }
273+ process.exitCode = 1;
274+ }
275+}
added scripts/refresh-catalog.mjs +442 -0
new file mode 100644
@@ -0,0 +1,442 @@
1+import { createHash } from "node:crypto";
2+import {
3+ mkdirSync,
4+ mkdtempSync,
5+ renameSync,
6+ rmSync,
7+ writeFileSync,
8+} from "node:fs";
9+import { dirname, join, resolve } from "node:path";
10+import { fileURLToPath } from "node:url";
11+import {
12+ DOWNLOAD_TIMEOUT_MS,
13+ IDFM_CATALOG_SOURCE_URL,
14+ activateCatalogCandidate,
15+ buildCatalogCandidate,
16+ downloadCatalogSource,
17+ resolveIdfmGtfsDownload,
18+ validateCatalogCandidate,
19+} from "../packages/backend/src/catalog-import.ts";
20+import { redactSecrets } from "../packages/backend/src/index.ts";
21+
22+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
23+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
24+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
25+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/refresh-catalog.json");
26+const METADATA_BODY_LIMIT = 1024 * 1024;
27+const DATA_ORIGIN = new URL(IDFM_CATALOG_SOURCE_URL.gtfsRecord).origin;
28+
29+const GTFS_SOURCE = {
30+ key: "gtfs",
31+ dataset: "offre-horaires-tc-gtfs-idfm",
32+ restricted: true,
33+ requiredFields: ["filename", "url"],
34+};
35+
36+const CSV_SOURCES = [
37+ {
38+ key: "perimeter",
39+ dataset: "perimetre-des-donnees-tr-disponibles-plateforme-idfm",
40+ url: IDFM_CATALOG_SOURCE_URL.perimeter,
41+ restricted: true,
42+ requiredFields: ["line", "name_line", "ns2_stoppointref", "ns2_stopname", "operatorname"],
43+ },
44+ {
45+ key: "arrets",
46+ dataset: "arrets",
47+ url: IDFM_CATALOG_SOURCE_URL.arrets,
48+ restricted: false,
49+ requiredFields: ["arrid", "zdaid"],
50+ },
51+ {
52+ key: "zones",
53+ dataset: "zones-d-arrets",
54+ url: IDFM_CATALOG_SOURCE_URL.zones,
55+ restricted: false,
56+ requiredFields: ["zdaid", "zdaname", "zdatown"],
57+ },
58+ {
59+ key: "relations",
60+ dataset: "relations",
61+ url: IDFM_CATALOG_SOURCE_URL.relations,
62+ restricted: false,
63+ requiredFields: ["zdaid", "arrid", "artid"],
64+ },
65+ {
66+ key: "lines",
67+ dataset: "referentiel-des-lignes",
68+ url: IDFM_CATALOG_SOURCE_URL.lines,
69+ restricted: false,
70+ requiredFields: ["id_line", "name_line", "shortname_line", "transportmode", "transportsubmode", "status"],
71+ },
72+ {
73+ key: "arretsLignes",
74+ dataset: "arrets-lignes",
75+ url: IDFM_CATALOG_SOURCE_URL.arretsLignes,
76+ restricted: false,
77+ requiredFields: ["id", "stop_id"],
78+ },
79+];
80+
81+function record(value) {
82+ return typeof value === "object" && value !== null && !Array.isArray(value)
83+ ? value
84+ : undefined;
85+}
86+
87+function requiredText(value, field) {
88+ if (typeof value !== "string" || value.trim() === "" || value.includes("\0")) {
89+ throw new Error(`Invalid ${field} in IDFM dataset metadata`);
90+ }
91+ return value;
92+}
93+
94+async function boundedJson(response, dataset) {
95+ if (response.body === null) throw new Error(`IDFM metadata for ${dataset} returned no body`);
96+ const chunks = [];
97+ let byteLength = 0;
98+ for await (const chunkValue of response.body) {
99+ const chunk = chunkValue instanceof Uint8Array
100+ ? chunkValue
101+ : new Uint8Array(chunkValue);
102+ byteLength += chunk.byteLength;
103+ if (byteLength > METADATA_BODY_LIMIT) {
104+ throw new Error(`IDFM metadata for ${dataset} exceeds 1 MiB`);
105+ }
106+ chunks.push(chunk);
107+ }
108+
109+ try {
110+ const text = new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks));
111+ return JSON.parse(text);
112+ } catch {
113+ throw new Error(`IDFM metadata for ${dataset} is not valid UTF-8 JSON`);
114+ }
115+}
116+
117+async function fetchDatasetMetadata(source, datasetToken) {
118+ const target = new URL(
119+ `/api/explore/v2.1/catalog/datasets/${encodeURIComponent(source.dataset)}`,
120+ DATA_ORIGIN,
121+ );
122+ const controller = new AbortController();
123+ const timeout = setTimeout(
124+ () => controller.abort(new Error("IDFM metadata request timed out")),
125+ DOWNLOAD_TIMEOUT_MS,
126+ );
127+
128+ try {
129+ const response = await fetch(target, {
130+ method: "GET",
131+ redirect: "manual",
132+ credentials: "omit",
133+ cache: "no-store",
134+ headers: source.restricted
135+ ? { Authorization: `apikey ${datasetToken}` }
136+ : {},
137+ signal: controller.signal,
138+ });
139+ if (response.status !== 200) {
140+ throw new Error(`IDFM metadata for ${source.dataset} returned HTTP ${response.status}`);
141+ }
142+ const mediaType = (response.headers.get("content-type") ?? "")
143+ .split(";", 1)[0]
144+ .trim()
145+ .toLowerCase();
146+ if (mediaType !== "application/json") {
147+ throw new Error(`IDFM metadata for ${source.dataset} is not JSON`);
148+ }
149+
150+ const value = record(await boundedJson(response, source.dataset));
151+ const metas = record(value?.metas);
152+ const defaults = record(metas?.default);
153+ if (value?.dataset_id !== source.dataset || value?.has_records !== true) {
154+ throw new Error(`IDFM metadata identity for ${source.dataset} is invalid`);
155+ }
156+ if (value.data_visible !== !source.restricted) {
157+ throw new Error(`IDFM metadata visibility for ${source.dataset} changed`);
158+ }
159+
160+ const fields = Array.isArray(value.fields) ? value.fields : [];
161+ const fieldNames = new Set(fields.map((field) => record(field)?.name));
162+ for (const requiredField of source.requiredFields) {
163+ if (!fieldNames.has(requiredField)) {
164+ throw new Error(`IDFM metadata for ${source.dataset} lacks ${requiredField}`);
165+ }
166+ }
167+
168+ const recordsCount = defaults?.records_count;
169+ if (!Number.isSafeInteger(recordsCount) || recordsCount < 1) {
170+ throw new Error(`IDFM metadata record count for ${source.dataset} is invalid`);
171+ }
172+ const modified = requiredText(defaults?.modified, `${source.dataset}.modified`);
173+ if (!Number.isFinite(Date.parse(modified))) {
174+ throw new Error(`IDFM metadata timestamp for ${source.dataset} is invalid`);
175+ }
176+
177+ return {
178+ dataset: source.dataset,
179+ datasetUid: requiredText(value.dataset_uid, `${source.dataset}.dataset_uid`),
180+ title: requiredText(defaults?.title, `${source.dataset}.title`),
181+ license: requiredText(defaults?.license, `${source.dataset}.license`),
182+ modified,
183+ recordsCount,
184+ restricted: source.restricted,
185+ retrievedAt: new Date().toISOString(),
186+ };
187+ } finally {
188+ clearTimeout(timeout);
189+ }
190+}
191+
192+function sourceAttribution(metadata, url) {
193+ return {
194+ dataset: metadata.dataset,
195+ url,
196+ retrievedAt: metadata.retrievedAt,
197+ license: metadata.license,
198+ restricted: metadata.restricted,
199+ };
200+}
201+
202+function revisionFor(downloads) {
203+ const sourceState = downloads.map(({ metadata, download }) => ({
204+ dataset: metadata.dataset,
205+ modified: metadata.modified,
206+ recordsCount: metadata.recordsCount,
207+ sha256: download.sha256,
208+ }));
209+ const digest = createHash("sha256")
210+ .update(JSON.stringify(sourceState), "utf8")
211+ .digest("hex");
212+ return `idfm-v1-${digest}`;
213+}
214+
215+function assertSameResult(expected, actual, phase) {
216+ const sameCounts = expected.placeCount === actual.placeCount
217+ && expected.serviceCount === actual.serviceCount
218+ && Object.keys(expected.countsByMode).every(
219+ (mode) => expected.countsByMode[mode] === actual.countsByMode[mode],
220+ );
221+ if (expected.sourceRevision !== actual.sourceRevision || !sameCounts) {
222+ throw new Error(`Catalog ${phase} result changed after validation`);
223+ }
224+}
225+
226+function evidenceSource({ metadata, download, filename }) {
227+ return {
228+ dataset: metadata.dataset,
229+ datasetUid: metadata.datasetUid,
230+ title: metadata.title,
231+ license: metadata.license,
232+ restricted: metadata.restricted,
233+ modified: metadata.modified,
234+ retrievedAt: metadata.retrievedAt,
235+ recordsCount: metadata.recordsCount,
236+ bytes: download.bytes,
237+ sha256: download.sha256,
238+ ...(filename === undefined ? {} : { filename }),
239+ };
240+}
241+
242+function writeEvidence(report, secrets) {
243+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
244+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
245+ const serialized = `${redactSecrets(JSON.stringify(report, null, 2), secrets)}\n`;
246+ try {
247+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
248+ renameSync(temporaryPath, EVIDENCE_PATH);
249+ } finally {
250+ rmSync(temporaryPath, { force: true });
251+ }
252+ process.stdout.write(serialized);
253+}
254+
255+// Closes the snapshot/download window: re-fetches each dataset's metadata after
256+// all downloads complete and aborts the refresh when any revision moved, so the
257+// built candidate can never mix content from two source revisions.
258+async function verifySnapshotsUnchanged(downloads, datasetToken) {
259+ for (const entry of downloads) {
260+ if (entry.filename !== undefined) {
261+ const resolved = await resolveIdfmGtfsDownload(datasetToken);
262+ if (resolved.filename !== entry.filename) {
263+ throw new Error(`IDFM GTFS record changed during refresh (${entry.filename} -> ${resolved.filename})`);
264+ }
265+ }
266+ const source = entry.key === GTFS_SOURCE.key
267+ ? GTFS_SOURCE
268+ : CSV_SOURCES.find((candidate) => candidate.key === entry.key);
269+ if (source === undefined) {
270+ throw new Error(`IDFM source ${entry.key} is missing from the refresh snapshot`);
271+ }
272+ const fresh = await fetchDatasetMetadata(source, source.restricted ? datasetToken : undefined);
273+ if (fresh.modified !== entry.metadata.modified) {
274+ throw new Error(`IDFM dataset ${entry.metadata.dataset} changed during refresh (modified ${entry.metadata.modified} -> ${fresh.modified})`);
275+ }
276+ }
277+}
278+
279+async function refreshCatalog(datasetToken, state) {
280+ mkdirSync(dirname(ACTIVE_PATH), { recursive: true });
281+ const workspace = mkdtempSync(join(dirname(ACTIVE_PATH), ".refresh-"));
282+
283+ let downloads;
284+ let activated;
285+ try {
286+ const gtfsMetadata = await fetchDatasetMetadata(GTFS_SOURCE, datasetToken);
287+ const resolvedGtfs = await resolveIdfmGtfsDownload(datasetToken);
288+ if (new URL(resolvedGtfs.url).origin !== DATA_ORIGIN) {
289+ throw new Error("Resolved IDFM GTFS download left the trusted data origin");
290+ }
291+ const gtfsFilename = requiredText(resolvedGtfs.filename, "GTFS filename");
292+ if (Buffer.byteLength(gtfsFilename, "utf8") > 512) {
293+ throw new Error("Resolved IDFM GTFS filename exceeds 512 bytes");
294+ }
295+ const gtfsDownload = await downloadCatalogSource({
296+ source: sourceAttribution(gtfsMetadata, resolvedGtfs.url),
297+ destinationPath: join(workspace, "gtfs.zip"),
298+ expectedContentTypes: ["application/zip"],
299+ datasetToken,
300+ });
301+ downloads = [{
302+ key: GTFS_SOURCE.key,
303+ metadata: gtfsMetadata,
304+ download: gtfsDownload,
305+ filename: gtfsFilename,
306+ }];
307+
308+ for (const source of CSV_SOURCES) {
309+ const metadata = await fetchDatasetMetadata(
310+ source,
311+ source.restricted ? datasetToken : undefined,
312+ );
313+ const download = await downloadCatalogSource({
314+ source: sourceAttribution(metadata, source.url),
315+ destinationPath: join(workspace, `${source.key}.csv`),
316+ expectedContentTypes: ["text/csv"],
317+ ...(source.restricted ? { datasetToken } : {}),
318+ });
319+ downloads.push({ key: source.key, metadata, download });
320+ }
321+ await verifySnapshotsUnchanged(downloads, datasetToken);
322+ state.metadataRechecked = true;
323+
324+ const byKey = Object.fromEntries(downloads.map((entry) => [entry.key, entry]));
325+ const sourceRevision = revisionFor(downloads);
326+ const createdAt = new Date().toISOString();
327+ const candidatePath = join(workspace, "candidate.sqlite");
328+ const buildResult = await buildCatalogCandidate({
329+ candidatePath,
330+ sourceRevision,
331+ createdAt,
332+ sources: {
333+ gtfsZipPath: byKey.gtfs.download.path,
334+ perimeter: {
335+ path: byKey.perimeter.download.path,
336+ expectedRows: byKey.perimeter.metadata.recordsCount,
337+ },
338+ arrets: {
339+ path: byKey.arrets.download.path,
340+ expectedRows: byKey.arrets.metadata.recordsCount,
341+ },
342+ zones: {
343+ path: byKey.zones.download.path,
344+ expectedRows: byKey.zones.metadata.recordsCount,
345+ },
346+ relations: {
347+ path: byKey.relations.download.path,
348+ expectedRows: byKey.relations.metadata.recordsCount,
349+ },
350+ lines: {
351+ path: byKey.lines.download.path,
352+ expectedRows: byKey.lines.metadata.recordsCount,
353+ },
354+ arretsLignes: {
355+ path: byKey.arretsLignes.download.path,
356+ expectedRows: byKey.arretsLignes.metadata.recordsCount,
357+ },
358+ },
359+ });
360+ const validated = validateCatalogCandidate(candidatePath);
361+ assertSameResult(buildResult, validated, "candidate");
362+ activated = activateCatalogCandidate(candidatePath, ACTIVE_PATH);
363+ state.atomicActivation = true;
364+ assertSameResult(validated, activated, "activation");
365+ } finally {
366+ rmSync(workspace, { recursive: true, force: true });
367+ state.temporaryInputsRemoved = true;
368+ }
369+
370+ return {
371+ command: "catalog:refresh",
372+ status: "ok",
373+ completedAt: new Date().toISOString(),
374+ sourceRevision: activated.sourceRevision,
375+ sources: downloads.map(evidenceSource),
376+ catalog: {
377+ createdAt: activated.createdAt,
378+ placeCount: activated.placeCount,
379+ serviceCount: activated.serviceCount,
380+ countsByMode: activated.countsByMode,
381+ placeResolution: buildResult.placeResolution ?? null,
382+ },
383+ validation: {
384+ sqliteIntegrity: "ok",
385+ foreignKeys: "ok",
386+ schemaAndCatalogVersions: "ok",
387+ allModesCovered: true,
388+ everyServiceResolvable: true,
389+ metadataRechecked: state.metadataRechecked === true,
390+ },
391+ activation: {
392+ atomic: state.atomicActivation,
393+ temporaryInputsRemoved: state.temporaryInputsRemoved,
394+ ...(activated.revalidatedAfterRenameFailure === true
395+ ? { revalidatedAfterRenameFailure: true }
396+ : {}),
397+ },
398+ maxRSSKiB: process.resourceUsage().maxRSS,
399+ };
400+}
401+
402+const startedAt = new Date().toISOString();
403+const state = {
404+ atomicActivation: false,
405+ temporaryInputsRemoved: false,
406+ metadataRechecked: false,
407+};
408+const datasetToken = process.env.IDFM_DATASET_TOKEN;
409+
410+if (typeof datasetToken !== "string" || datasetToken.length === 0) {
411+ process.stderr.write("catalog:refresh requires IDFM_DATASET_TOKEN\n");
412+ process.exitCode = 1;
413+} else {
414+ try {
415+ const report = await refreshCatalog(datasetToken, state);
416+ writeEvidence({ ...report, startedAt }, [datasetToken]);
417+ } catch (error) {
418+ const message = redactSecrets(
419+ error instanceof Error && error.message ? error.message : "Catalog refresh failed",
420+ [datasetToken],
421+ );
422+ const report = {
423+ command: "catalog:refresh",
424+ status: "failed",
425+ startedAt,
426+ completedAt: new Date().toISOString(),
427+ activation: {
428+ atomic: state.atomicActivation,
429+ temporaryInputsRemoved: state.temporaryInputsRemoved,
430+ },
431+ maxRSSKiB: process.resourceUsage().maxRSS,
432+ error: message,
433+ };
434+ try {
435+ writeEvidence(report, [datasetToken]);
436+ } catch {
437+ // The primary refresh failure remains authoritative if evidence persistence also fails.
438+ }
439+ process.stderr.write(`catalog:refresh failed: ${message}\n`);
440+ process.exitCode = 1;
441+ }
442+}
new file mode 100644
@@ -0,0 +1,442 @@
1+import { createHash } from "node:crypto";
2+import {
3+ mkdirSync,
4+ mkdtempSync,
5+ renameSync,
6+ rmSync,
7+ writeFileSync,
8+} from "node:fs";
9+import { dirname, join, resolve } from "node:path";
10+import { fileURLToPath } from "node:url";
11+import {
12+ DOWNLOAD_TIMEOUT_MS,
13+ IDFM_CATALOG_SOURCE_URL,
14+ activateCatalogCandidate,
15+ buildCatalogCandidate,
16+ downloadCatalogSource,
17+ resolveIdfmGtfsDownload,
18+ validateCatalogCandidate,
19+} from "../packages/backend/src/catalog-import.ts";
20+import { redactSecrets } from "../packages/backend/src/index.ts";
21+
22+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
23+const DEFAULT_ACTIVE_PATH = join(PROJECT_ROOT, "var/catalog/catalog.sqlite");
24+const ACTIVE_PATH = resolve(process.env.CATALOG_PATH ?? DEFAULT_ACTIVE_PATH);
25+const EVIDENCE_PATH = join(PROJECT_ROOT, "var/catalog/evidence/refresh-catalog.json");
26+const METADATA_BODY_LIMIT = 1024 * 1024;
27+const DATA_ORIGIN = new URL(IDFM_CATALOG_SOURCE_URL.gtfsRecord).origin;
28+
29+const GTFS_SOURCE = {
30+ key: "gtfs",
31+ dataset: "offre-horaires-tc-gtfs-idfm",
32+ restricted: true,
33+ requiredFields: ["filename", "url"],
34+};
35+
36+const CSV_SOURCES = [
37+ {
38+ key: "perimeter",
39+ dataset: "perimetre-des-donnees-tr-disponibles-plateforme-idfm",
40+ url: IDFM_CATALOG_SOURCE_URL.perimeter,
41+ restricted: true,
42+ requiredFields: ["line", "name_line", "ns2_stoppointref", "ns2_stopname", "operatorname"],
43+ },
44+ {
45+ key: "arrets",
46+ dataset: "arrets",
47+ url: IDFM_CATALOG_SOURCE_URL.arrets,
48+ restricted: false,
49+ requiredFields: ["arrid", "zdaid"],
50+ },
51+ {
52+ key: "zones",
53+ dataset: "zones-d-arrets",
54+ url: IDFM_CATALOG_SOURCE_URL.zones,
55+ restricted: false,
56+ requiredFields: ["zdaid", "zdaname", "zdatown"],
57+ },
58+ {
59+ key: "relations",
60+ dataset: "relations",
61+ url: IDFM_CATALOG_SOURCE_URL.relations,
62+ restricted: false,
63+ requiredFields: ["zdaid", "arrid", "artid"],
64+ },
65+ {
66+ key: "lines",
67+ dataset: "referentiel-des-lignes",
68+ url: IDFM_CATALOG_SOURCE_URL.lines,
69+ restricted: false,
70+ requiredFields: ["id_line", "name_line", "shortname_line", "transportmode", "transportsubmode", "status"],
71+ },
72+ {
73+ key: "arretsLignes",
74+ dataset: "arrets-lignes",
75+ url: IDFM_CATALOG_SOURCE_URL.arretsLignes,
76+ restricted: false,
77+ requiredFields: ["id", "stop_id"],
78+ },
79+];
80+
81+function record(value) {
82+ return typeof value === "object" && value !== null && !Array.isArray(value)
83+ ? value
84+ : undefined;
85+}
86+
87+function requiredText(value, field) {
88+ if (typeof value !== "string" || value.trim() === "" || value.includes("\0")) {
89+ throw new Error(`Invalid ${field} in IDFM dataset metadata`);
90+ }
91+ return value;
92+}
93+
94+async function boundedJson(response, dataset) {
95+ if (response.body === null) throw new Error(`IDFM metadata for ${dataset} returned no body`);
96+ const chunks = [];
97+ let byteLength = 0;
98+ for await (const chunkValue of response.body) {
99+ const chunk = chunkValue instanceof Uint8Array
100+ ? chunkValue
101+ : new Uint8Array(chunkValue);
102+ byteLength += chunk.byteLength;
103+ if (byteLength > METADATA_BODY_LIMIT) {
104+ throw new Error(`IDFM metadata for ${dataset} exceeds 1 MiB`);
105+ }
106+ chunks.push(chunk);
107+ }
108+
109+ try {
110+ const text = new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks));
111+ return JSON.parse(text);
112+ } catch {
113+ throw new Error(`IDFM metadata for ${dataset} is not valid UTF-8 JSON`);
114+ }
115+}
116+
117+async function fetchDatasetMetadata(source, datasetToken) {
118+ const target = new URL(
119+ `/api/explore/v2.1/catalog/datasets/${encodeURIComponent(source.dataset)}`,
120+ DATA_ORIGIN,
121+ );
122+ const controller = new AbortController();
123+ const timeout = setTimeout(
124+ () => controller.abort(new Error("IDFM metadata request timed out")),
125+ DOWNLOAD_TIMEOUT_MS,
126+ );
127+
128+ try {
129+ const response = await fetch(target, {
130+ method: "GET",
131+ redirect: "manual",
132+ credentials: "omit",
133+ cache: "no-store",
134+ headers: source.restricted
135+ ? { Authorization: `apikey ${datasetToken}` }
136+ : {},
137+ signal: controller.signal,
138+ });
139+ if (response.status !== 200) {
140+ throw new Error(`IDFM metadata for ${source.dataset} returned HTTP ${response.status}`);
141+ }
142+ const mediaType = (response.headers.get("content-type") ?? "")
143+ .split(";", 1)[0]
144+ .trim()
145+ .toLowerCase();
146+ if (mediaType !== "application/json") {
147+ throw new Error(`IDFM metadata for ${source.dataset} is not JSON`);
148+ }
149+
150+ const value = record(await boundedJson(response, source.dataset));
151+ const metas = record(value?.metas);
152+ const defaults = record(metas?.default);
153+ if (value?.dataset_id !== source.dataset || value?.has_records !== true) {
154+ throw new Error(`IDFM metadata identity for ${source.dataset} is invalid`);
155+ }
156+ if (value.data_visible !== !source.restricted) {
157+ throw new Error(`IDFM metadata visibility for ${source.dataset} changed`);
158+ }
159+
160+ const fields = Array.isArray(value.fields) ? value.fields : [];
161+ const fieldNames = new Set(fields.map((field) => record(field)?.name));
162+ for (const requiredField of source.requiredFields) {
163+ if (!fieldNames.has(requiredField)) {
164+ throw new Error(`IDFM metadata for ${source.dataset} lacks ${requiredField}`);
165+ }
166+ }
167+
168+ const recordsCount = defaults?.records_count;
169+ if (!Number.isSafeInteger(recordsCount) || recordsCount < 1) {
170+ throw new Error(`IDFM metadata record count for ${source.dataset} is invalid`);
171+ }
172+ const modified = requiredText(defaults?.modified, `${source.dataset}.modified`);
173+ if (!Number.isFinite(Date.parse(modified))) {
174+ throw new Error(`IDFM metadata timestamp for ${source.dataset} is invalid`);
175+ }
176+
177+ return {
178+ dataset: source.dataset,
179+ datasetUid: requiredText(value.dataset_uid, `${source.dataset}.dataset_uid`),
180+ title: requiredText(defaults?.title, `${source.dataset}.title`),
181+ license: requiredText(defaults?.license, `${source.dataset}.license`),
182+ modified,
183+ recordsCount,
184+ restricted: source.restricted,
185+ retrievedAt: new Date().toISOString(),
186+ };
187+ } finally {
188+ clearTimeout(timeout);
189+ }
190+}
191+
192+function sourceAttribution(metadata, url) {
193+ return {
194+ dataset: metadata.dataset,
195+ url,
196+ retrievedAt: metadata.retrievedAt,
197+ license: metadata.license,
198+ restricted: metadata.restricted,
199+ };
200+}
201+
202+function revisionFor(downloads) {
203+ const sourceState = downloads.map(({ metadata, download }) => ({
204+ dataset: metadata.dataset,
205+ modified: metadata.modified,
206+ recordsCount: metadata.recordsCount,
207+ sha256: download.sha256,
208+ }));
209+ const digest = createHash("sha256")
210+ .update(JSON.stringify(sourceState), "utf8")
211+ .digest("hex");
212+ return `idfm-v1-${digest}`;
213+}
214+
215+function assertSameResult(expected, actual, phase) {
216+ const sameCounts = expected.placeCount === actual.placeCount
217+ && expected.serviceCount === actual.serviceCount
218+ && Object.keys(expected.countsByMode).every(
219+ (mode) => expected.countsByMode[mode] === actual.countsByMode[mode],
220+ );
221+ if (expected.sourceRevision !== actual.sourceRevision || !sameCounts) {
222+ throw new Error(`Catalog ${phase} result changed after validation`);
223+ }
224+}
225+
226+function evidenceSource({ metadata, download, filename }) {
227+ return {
228+ dataset: metadata.dataset,
229+ datasetUid: metadata.datasetUid,
230+ title: metadata.title,
231+ license: metadata.license,
232+ restricted: metadata.restricted,
233+ modified: metadata.modified,
234+ retrievedAt: metadata.retrievedAt,
235+ recordsCount: metadata.recordsCount,
236+ bytes: download.bytes,
237+ sha256: download.sha256,
238+ ...(filename === undefined ? {} : { filename }),
239+ };
240+}
241+
242+function writeEvidence(report, secrets) {
243+ mkdirSync(dirname(EVIDENCE_PATH), { recursive: true });
244+ const temporaryPath = `${EVIDENCE_PATH}.${process.pid}.tmp`;
245+ const serialized = `${redactSecrets(JSON.stringify(report, null, 2), secrets)}\n`;
246+ try {
247+ writeFileSync(temporaryPath, serialized, { encoding: "utf8", mode: 0o600 });
248+ renameSync(temporaryPath, EVIDENCE_PATH);
249+ } finally {
250+ rmSync(temporaryPath, { force: true });
251+ }
252+ process.stdout.write(serialized);
253+}
254+
255+// Closes the snapshot/download window: re-fetches each dataset's metadata after
256+// all downloads complete and aborts the refresh when any revision moved, so the
257+// built candidate can never mix content from two source revisions.
258+async function verifySnapshotsUnchanged(downloads, datasetToken) {
259+ for (const entry of downloads) {
260+ if (entry.filename !== undefined) {
261+ const resolved = await resolveIdfmGtfsDownload(datasetToken);
262+ if (resolved.filename !== entry.filename) {
263+ throw new Error(`IDFM GTFS record changed during refresh (${entry.filename} -> ${resolved.filename})`);
264+ }
265+ }
266+ const source = entry.key === GTFS_SOURCE.key
267+ ? GTFS_SOURCE
268+ : CSV_SOURCES.find((candidate) => candidate.key === entry.key);
269+ if (source === undefined) {
270+ throw new Error(`IDFM source ${entry.key} is missing from the refresh snapshot`);
271+ }
272+ const fresh = await fetchDatasetMetadata(source, source.restricted ? datasetToken : undefined);
273+ if (fresh.modified !== entry.metadata.modified) {
274+ throw new Error(`IDFM dataset ${entry.metadata.dataset} changed during refresh (modified ${entry.metadata.modified} -> ${fresh.modified})`);
275+ }
276+ }
277+}
278+
279+async function refreshCatalog(datasetToken, state) {
280+ mkdirSync(dirname(ACTIVE_PATH), { recursive: true });
281+ const workspace = mkdtempSync(join(dirname(ACTIVE_PATH), ".refresh-"));
282+
283+ let downloads;
284+ let activated;
285+ try {
286+ const gtfsMetadata = await fetchDatasetMetadata(GTFS_SOURCE, datasetToken);
287+ const resolvedGtfs = await resolveIdfmGtfsDownload(datasetToken);
288+ if (new URL(resolvedGtfs.url).origin !== DATA_ORIGIN) {
289+ throw new Error("Resolved IDFM GTFS download left the trusted data origin");
290+ }
291+ const gtfsFilename = requiredText(resolvedGtfs.filename, "GTFS filename");
292+ if (Buffer.byteLength(gtfsFilename, "utf8") > 512) {
293+ throw new Error("Resolved IDFM GTFS filename exceeds 512 bytes");
294+ }
295+ const gtfsDownload = await downloadCatalogSource({
296+ source: sourceAttribution(gtfsMetadata, resolvedGtfs.url),
297+ destinationPath: join(workspace, "gtfs.zip"),
298+ expectedContentTypes: ["application/zip"],
299+ datasetToken,
300+ });
301+ downloads = [{
302+ key: GTFS_SOURCE.key,
303+ metadata: gtfsMetadata,
304+ download: gtfsDownload,
305+ filename: gtfsFilename,
306+ }];
307+
308+ for (const source of CSV_SOURCES) {
309+ const metadata = await fetchDatasetMetadata(
310+ source,
311+ source.restricted ? datasetToken : undefined,
312+ );
313+ const download = await downloadCatalogSource({
314+ source: sourceAttribution(metadata, source.url),
315+ destinationPath: join(workspace, `${source.key}.csv`),
316+ expectedContentTypes: ["text/csv"],
317+ ...(source.restricted ? { datasetToken } : {}),
318+ });
319+ downloads.push({ key: source.key, metadata, download });
320+ }
321+ await verifySnapshotsUnchanged(downloads, datasetToken);
322+ state.metadataRechecked = true;
323+
324+ const byKey = Object.fromEntries(downloads.map((entry) => [entry.key, entry]));
325+ const sourceRevision = revisionFor(downloads);
326+ const createdAt = new Date().toISOString();
327+ const candidatePath = join(workspace, "candidate.sqlite");
328+ const buildResult = await buildCatalogCandidate({
329+ candidatePath,
330+ sourceRevision,
331+ createdAt,
332+ sources: {
333+ gtfsZipPath: byKey.gtfs.download.path,
334+ perimeter: {
335+ path: byKey.perimeter.download.path,
336+ expectedRows: byKey.perimeter.metadata.recordsCount,
337+ },
338+ arrets: {
339+ path: byKey.arrets.download.path,
340+ expectedRows: byKey.arrets.metadata.recordsCount,
341+ },
342+ zones: {
343+ path: byKey.zones.download.path,
344+ expectedRows: byKey.zones.metadata.recordsCount,
345+ },
346+ relations: {
347+ path: byKey.relations.download.path,
348+ expectedRows: byKey.relations.metadata.recordsCount,
349+ },
350+ lines: {
351+ path: byKey.lines.download.path,
352+ expectedRows: byKey.lines.metadata.recordsCount,
353+ },
354+ arretsLignes: {
355+ path: byKey.arretsLignes.download.path,
356+ expectedRows: byKey.arretsLignes.metadata.recordsCount,
357+ },
358+ },
359+ });
360+ const validated = validateCatalogCandidate(candidatePath);
361+ assertSameResult(buildResult, validated, "candidate");
362+ activated = activateCatalogCandidate(candidatePath, ACTIVE_PATH);
363+ state.atomicActivation = true;
364+ assertSameResult(validated, activated, "activation");
365+ } finally {
366+ rmSync(workspace, { recursive: true, force: true });
367+ state.temporaryInputsRemoved = true;
368+ }
369+
370+ return {
371+ command: "catalog:refresh",
372+ status: "ok",
373+ completedAt: new Date().toISOString(),
374+ sourceRevision: activated.sourceRevision,
375+ sources: downloads.map(evidenceSource),
376+ catalog: {
377+ createdAt: activated.createdAt,
378+ placeCount: activated.placeCount,
379+ serviceCount: activated.serviceCount,
380+ countsByMode: activated.countsByMode,
381+ placeResolution: buildResult.placeResolution ?? null,
382+ },
383+ validation: {
384+ sqliteIntegrity: "ok",
385+ foreignKeys: "ok",
386+ schemaAndCatalogVersions: "ok",
387+ allModesCovered: true,
388+ everyServiceResolvable: true,
389+ metadataRechecked: state.metadataRechecked === true,
390+ },
391+ activation: {
392+ atomic: state.atomicActivation,
393+ temporaryInputsRemoved: state.temporaryInputsRemoved,
394+ ...(activated.revalidatedAfterRenameFailure === true
395+ ? { revalidatedAfterRenameFailure: true }
396+ : {}),
397+ },
398+ maxRSSKiB: process.resourceUsage().maxRSS,
399+ };
400+}
401+
402+const startedAt = new Date().toISOString();
403+const state = {
404+ atomicActivation: false,
405+ temporaryInputsRemoved: false,
406+ metadataRechecked: false,
407+};
408+const datasetToken = process.env.IDFM_DATASET_TOKEN;
409+
410+if (typeof datasetToken !== "string" || datasetToken.length === 0) {
411+ process.stderr.write("catalog:refresh requires IDFM_DATASET_TOKEN\n");
412+ process.exitCode = 1;
413+} else {
414+ try {
415+ const report = await refreshCatalog(datasetToken, state);
416+ writeEvidence({ ...report, startedAt }, [datasetToken]);
417+ } catch (error) {
418+ const message = redactSecrets(
419+ error instanceof Error && error.message ? error.message : "Catalog refresh failed",
420+ [datasetToken],
421+ );
422+ const report = {
423+ command: "catalog:refresh",
424+ status: "failed",
425+ startedAt,
426+ completedAt: new Date().toISOString(),
427+ activation: {
428+ atomic: state.atomicActivation,
429+ temporaryInputsRemoved: state.temporaryInputsRemoved,
430+ },
431+ maxRSSKiB: process.resourceUsage().maxRSS,
432+ error: message,
433+ };
434+ try {
435+ writeEvidence(report, [datasetToken]);
436+ } catch {
437+ // The primary refresh failure remains authoritative if evidence persistence also fails.
438+ }
439+ process.stderr.write(`catalog:refresh failed: ${message}\n`);
440+ process.exitCode = 1;
441+ }
442+}
modified scripts/test.mjs +3 -0
@@ -10,6 +10,9 @@ run(process.execPath, [
1010 "--test",
1111 "packages/contracts/test/contracts.test.ts",
1212 "packages/backend/test/backend.test.ts",
13+ "packages/backend/test/catalog.test.ts",
14+ "packages/backend/test/catalog-import.test.ts",
15+ "packages/backend/test/catalog-http.test.ts",
1316 "packages/config-page/test/config-page.test.ts",
1417 "packages/companion/test/protocol.test.js",
1518 "packages/companion/test/companion.test.js",
@@ -10,6 +10,9 @@ run(process.execPath, [
10 "--test",10 "--test",
11 "packages/contracts/test/contracts.test.ts",11 "packages/contracts/test/contracts.test.ts",
12 "packages/backend/test/backend.test.ts",12 "packages/backend/test/backend.test.ts",
13+ "packages/backend/test/catalog.test.ts",
14+ "packages/backend/test/catalog-import.test.ts",
15+ "packages/backend/test/catalog-http.test.ts",
13 "packages/config-page/test/config-page.test.ts",16 "packages/config-page/test/config-page.test.ts",
14 "packages/companion/test/protocol.test.js",17 "packages/companion/test/protocol.test.js",
15 "packages/companion/test/companion.test.js",18 "packages/companion/test/companion.test.js",