arah2/search/scripts/json-to-sqlite.mjs

#!/usr/bin/env node
/*
 * json-to-sqlite.mjs — the D-05 backend-#2 format converter.
 * -----------------------------------------------------------
 * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
 * bot/src/ or web/src/ imports it, and the Worker's canonical dataset
 * format stays typed JSON (D-07). This script exists so backend #2 (the
 * Malaysia-only WASM geocoder, post-v1) is a FORMAT change, not a rewrite:
 * the SQLite dataset format is reachable from day one (D-05).
 *
 * Reads the COMMITTED curated dataset (search/src/data/kl-gazetteer.json)
 * and emits <out>/kl-gazetteer.sqlite with the SAME id set:
 *   places   (id TEXT PRIMARY KEY, name TEXT, aliases TEXT,
 *             lat REAL, lng REAL, category TEXT, address TEXT)
 *   metadata (key TEXT PRIMARY KEY, value TEXT)
 *
 * ODbL §4.4 share-alike: the emitted database is a Derivative Database of
 * OSM data, so the ODbL notice travels with the output BY CONSTRUCTION —
 * the _license / _attribution / _notice fields are written into the
 * metadata table. Never redistribute an output stripped of them (T-04-20).
 *
 * Writer: Node 24's built-in node:sqlite (DatabaseSync) — zero added
 * dependencies. Output is deterministic: rows are inserted in the
 * committed dataset's order and carry no timestamps.
 *
 * Usage: node scripts/json-to-sqlite.mjs --out <dir>   (or --out=<dir>)
 * Exit codes: 0 on success; 1 on a usage error (missing/unwritable --out).
 */
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { DatabaseSync } from "node:sqlite";

const DATASET_URL = new URL("../src/data/kl-gazetteer.json", import.meta.url);
const OUTPUT_NAME = "kl-gazetteer.sqlite";

function parseOutDir(argv) {
  for (let i = 2; i < argv.length; i++) {
    const arg = argv[i];
    if (arg.startsWith("--out=")) return arg.slice("--out=".length);
    if (arg === "--out") return argv[i + 1];
  }
  return undefined;
}

const outDir = parseOutDir(process.argv);
if (!outDir) {
  console.error("json-to-sqlite: missing --out <dir> — pass a directory for the emitted database.");
  process.exit(1);
}
const outPath = resolve(outDir);
if (!existsSync(outPath)) {
  mkdirSync(outPath, { recursive: true });
}

const dataset = JSON.parse(readFileSync(DATASET_URL, "utf8"));
const entries = dataset.entries ?? [];

const db = new DatabaseSync(resolve(outPath, OUTPUT_NAME));
try {
  db.exec("BEGIN");
  db.exec(`
    CREATE TABLE places (
      id TEXT PRIMARY KEY,
      name TEXT NOT NULL,
      aliases TEXT NOT NULL,
      lat REAL NOT NULL,
      lng REAL NOT NULL,
      category TEXT NOT NULL,
      address TEXT
    );
    CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
  `);
  const insert = db.prepare(
    "INSERT INTO places (id, name, aliases, lat, lng, category, address) VALUES (?, ?, ?, ?, ?, ?, ?)",
  );
  for (const entry of entries) {
    insert.run(
      entry.id,
      entry.name,
      JSON.stringify(entry.aliases),
      entry.lat,
      entry.lng,
      entry.category,
      entry.address ?? null,
    );
  }
  const insertMeta = db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)");
  insertMeta.run("_license", dataset._license ?? "ODbL-1.0");
  insertMeta.run("_attribution", dataset._attribution ?? "© OpenStreetMap contributors");
  insertMeta.run("_notice", dataset._notice ?? "");
  insertMeta.run("source_version", String(dataset.version ?? 0));
  insertMeta.run("entry_count", String(entries.length));
  db.exec("COMMIT");
} catch (err) {
  db.exec("ROLLBACK");
  db.close();
  console.error(`json-to-sqlite: conversion failed: ${err?.message ?? err}`);
  process.exit(1);
}
db.close();

// A machine-readable sidecar stating the licence terms alongside the database.
writeFileSync(
  resolve(outPath, "kl-gazetteer.sqlite.NOTICE.txt"),
  `${dataset._notice ?? ""}\nLicense: ${dataset._license ?? "ODbL-1.0"}\n` +
    `Attribution: ${dataset._attribution ?? "© OpenStreetMap contributors"}\n` +
    `The same terms are embedded in the metadata table of ${OUTPUT_NAME} (ODbL §4.4).\n`,
);

console.log(
  `wrote ${resolve(outPath, OUTPUT_NAME)} — ${entries.length} places, id set preserved, ` +
    `${dataset._license ?? "ODbL-1.0"} notice included (metadata table + NOTICE sidecar).`,
);
process.exit(0);