// Tail numbers for travellers' past flights worldwide, from the adsb.lol ADS-B archive.
//
// Data: adsb.lol globe_history (https://github.com/adsblol), Open Database Licence (ODbL) 1.0.
// "Flight data © adsb.lol contributors, ODbL." This script is the method TripStow uses to derive
// its table of matched flights (public.adsb_legs) from that database, published at /open-data as
// the ODbL asks.
//
// How it works, one UTC day at a time:
// 1. Collect travellers' flights that day with no tail number that have not been looked up yet.
// 2. Read the day's archive once (a tar of one trace per aircraft, 3 to 4 GB), opening only the
//    traces whose callsigns belong to an airline someone flew with that day.
// 3. Split each aircraft's day into legs at the archive's "new leg" marks and long gaps. A leg's
//    callsign is the one it broadcast in the air; it took off at the first airborne point after
//    the ground and landed at the next ground point, at the nearest airport within 6 km.
// 4. Match each flight: its callsign (BA117 is BAW117), else the airline's prefix on the same route
//    (easyJet flies EZY6PL), else, for codeshares, the only flight on the route at that time.
//
// Usage:
//   node --conditions=react-server --env-file=.env.local --import tsx scripts/import-adsb.mts --day 2026-09-30
//   node --conditions=react-server --env-file=.env.local --import tsx scripts/import-adsb.mts --needed --max-days 6
// Testing: --sample DIR reads trace files from a folder instead of the archive; --tar a,b reads the
// archive's parts from local files; --print-legs lists the legs found.
import fs from "node:fs";
import path from "node:path";
import { spawn } from "node:child_process";
import { gunzipSync } from "node:zlib";
import { createClient } from "@supabase/supabase-js";
import type { Database, Json } from "@/lib/supabase/types";
import { aircraftName, airlineIcao } from "@/lib/geo/reference";
import airportsJson from "@/data/airports.json" with { type: "json" };

const admin = createClient<Database>(process.env.NEXT_PUBLIC_SUPABASE_URL!, process.env.SUPABASE_SECRET_KEY!, { auth: { persistSession: false } });
const FIRST_DAY = "2023-02-16";
// US domestic flights get their tail numbers from the US government (BTS) instead.
const BTS_COUNTRIES = ["US", "PR", "VI", "GU", "AS", "MP"];

type Seg = { id: string; number: string; provider_code: string | null; origin_code: string | null; dest_code: string | null; start_at: string; scheduled_start_at: string | null; aircraft: string | null; times_checked: Json; origin_country: string | null; dest_country: string | null };
type Leg = { hex: string; registration: string | null; type: string | null; callsign: string | null; origin: string | null; dest: string | null; takeoff: number | null; landing: number | null; first: number };

// Airports -----------------------------------------------------------------------------------------

type RawAirport = { la: number; lo: number };
const GRID = new Map<string, { code: string; lat: number; lon: number }[]>();
for (const [code, a] of Object.entries(airportsJson as Record<string, RawAirport>)) {
  const k = `${Math.floor(a.la)}|${Math.floor(a.lo)}`;
  GRID.set(k, [...(GRID.get(k) ?? []), { code, lat: a.la, lon: a.lo }]);
}
const km = (a: number, b: number, c: number, d: number) => {
  const r = Math.PI / 180;
  const x = Math.sin(((c - a) * r) / 2) ** 2 + Math.cos(a * r) * Math.cos(c * r) * Math.sin(((d - b) * r) / 2) ** 2;
  return 12742 * Math.asin(Math.sqrt(x));
};
function airportAt(lat: number, lon: number): string | null {
  let best: { code: string; d: number } | null = null;
  for (let i = -1; i <= 1; i++)
    for (let j = -1; j <= 1; j++)
      for (const a of GRID.get(`${Math.floor(lat) + i}|${Math.floor(lon) + j}`) ?? []) {
        const d = km(lat, lon, a.lat, a.lon);
        if (d <= 6 && (!best || d < best.d)) best = { code: a.code, d };
      }
  return best?.code ?? null;
}

// The archive ------------------------------------------------------------------------------------

async function archiveParts(day: string): Promise<string[]> {
  const [y, m, d] = day.split("-");
  const tag = `v${y}.${m}.${d}-planes-readsb-prod-0`;
  const headers: Record<string, string> = { accept: "application/vnd.github+json" };
  if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
  const res = await fetch(`https://api.github.com/repos/adsblol/globe_history_${y}/releases/tags/${tag}`, { headers });
  if (!res.ok) return [];
  const release = (await res.json()) as { assets: { name: string; browser_download_url: string }[] };
  // One .tar, or a tar split into .tar.aa, .tar.ab, ... to be read in order.
  return release.assets
    .filter((a) => /\.tar(\.[a-z]{2})?$/.test(a.name))
    .sort((a, b) => a.name.localeCompare(b.name))
    .map((a) => a.browser_download_url);
}

/** The parts one after another, as one stream. curl copes better than fetch with long, slow downloads. */
async function* download(urls: string[]) {
  for (const url of urls) {
    if (!/^https?:/.test(url)) {
      for await (const chunk of fs.createReadStream(url, { highWaterMark: 1 << 20 })) yield chunk as Buffer;
      continue;
    }
    const curl = spawn("curl", ["-sSfL", "--retry", "5", "--retry-delay", "10", "--speed-time", "120", "--speed-limit", "1000", url], { stdio: ["ignore", "pipe", "inherit"] });
    const exited = new Promise<number>((resolve) => curl.on("close", (code) => resolve(code ?? 1)));
    for await (const chunk of curl.stdout) yield chunk as Buffer;
    const code = await exited;
    if (code !== 0) throw new Error(`Download failed (curl ${code}): ${url}`);
  }
}

/** A minimal tar reader: each file's name and contents, without keeping the archive in memory. */
async function* tarEntries(chunks: AsyncIterable<Buffer>) {
  const it = chunks[Symbol.asyncIterator]();
  let buf: Buffer = Buffer.alloc(0);
  let done = false;
  const read = async (n: number): Promise<Buffer | null> => {
    const parts: Buffer[] = [buf];
    let have = buf.length;
    while (have < n && !done) {
      const next = await it.next();
      if (next.done) done = true;
      else {
        parts.push(next.value);
        have += next.value.length;
      }
    }
    const all = parts.length > 1 ? Buffer.concat(parts, have) : buf;
    if (all.length < n) return null;
    buf = all.subarray(n);
    return all.subarray(0, n);
  };
  let longName: string | null = null;
  for (;;) {
    const header = await read(512);
    if (!header || header.every((b) => b === 0)) return;
    const field = (o: number, l: number) => header.subarray(o, o + l).toString("latin1").replace(/\0[\s\S]*$/, "").trim();
    const size = parseInt(field(124, 12) || "0", 8);
    const type = String.fromCharCode(header[156] || 48);
    const name = longName ?? [field(345, 155), field(0, 100)].filter(Boolean).join("/");
    longName = null;
    const data = size ? await read(Math.ceil(size / 512) * 512) : Buffer.alloc(0);
    if (!data) return;
    const body = Buffer.from(data.subarray(0, size));
    if (type === "L") longName = body.toString("utf8").replace(/\0[\s\S]*$/, "");
    else if (type === "0" || type === "\0") yield { name, body };
  }
}

// Legs -------------------------------------------------------------------------------------------

type Trace = { icao: string; r?: string; t?: string; timestamp: number; trace: (number | string | null | Record<string, unknown>)[][] };

type Point = { at: number; lat: number; lon: number; ground: boolean; low: boolean; cs: string | null };

function legsOf(t: Trace): Leg[] {
  // 1. The day's points, split where the archive marks a new leg (bit 2) or after half an hour unseen.
  const raw: Point[][] = [];
  let cs: string | null = null;
  let last = -Infinity;
  for (const pt of t.trace) {
    const [off, lat, lon, alt, gs, , flags, , details] = pt as [number, number, number, number | string, number | null, unknown, number, unknown, Record<string, unknown> | null];
    const at = t.timestamp + off;
    if (details && typeof details.flight === "string") cs = details.flight.trim() || cs;
    if (typeof lat !== "number" || typeof lon !== "number") continue;
    // On the ground: reported as such, or barely moving close to the ground (some transponders never say "ground").
    const ground = alt === "ground" || (typeof alt === "number" && alt < 1500 && typeof gs === "number" && gs < 45);
    const low = ground || (typeof alt === "number" && alt < 3000);
    // Out of range at cruising height (over an ocean) is the same flight; a gap with a landing either side is not.
    const prev = raw.at(-1)?.at(-1);
    const gap = at - last;
    if (!raw.length || flags & 2 || gap > 5 * 3600 || (gap > 1800 && (low || !!prev?.low))) raw.push([]);
    last = at;
    raw[raw.length - 1].push({ at, lat, lon, ground, low, cs });
  }
  const chunks = raw.filter((c) => c.length >= 2);

  // 2. Each leg: its airborne callsign, take-off (first airborne point after the ground) and landing
  //    (next ground point). The archive often starts the next leg on the landing itself, so a leg
  //    with no landing of its own ends where the next one starts, if that is on or near the ground.
  const legs = chunks.map((p, i) => {
    const counts = new Map<string, number>();
    for (const x of p) if (!x.ground && x.cs) counts.set(x.cs, (counts.get(x.cs) ?? 0) + 1);
    const callsign = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
    const up = p.findIndex((x, j) => !x.ground && j > 0 && p[j - 1].ground);
    const start = up > 0 ? p[up - 1] : p[0].low ? p[0] : null;
    const down = p.findIndex((x, j) => j > Math.max(up, 0) && x.ground && !p[j - 1].ground);
    let end: Point | null = down >= 0 ? p[down] : null;
    const next = chunks[i + 1]?.[0];
    if (!end && next && next.low && next.at - p[p.length - 1].at < 1200) end = next;
    if (!end && p[p.length - 1].low) end = p[p.length - 1];
    return {
      hex: t.icao,
      registration: t.r?.trim() || null,
      type: t.t?.trim() || null,
      callsign,
      origin: start ? airportAt(start.lat, start.lon) : null,
      dest: end ? airportAt(end.lat, end.lon) : null,
      // First seen low and climbing (no ground points before it): it took off about then.
      takeoff: up > 0 ? p[up].at : !p[0].ground && p[0].low ? p[0].at : null,
      landing: end ? end.at : null,
      first: p[0].at,
    };
  });
  // 3. The archive sometimes marks a new leg mid-ocean: join pieces of one flight (same callsign, the
  //    first never landed, the next never took off).
  const joined: Leg[] = [];
  for (const l of legs) {
    const prev = joined.at(-1);
    if (prev && prev.callsign && prev.callsign === l.callsign && prev.landing === null && l.takeoff === null) {
      joined[joined.length - 1] = { ...prev, dest: l.dest, landing: l.landing };
    } else joined.push(l);
  }
  return joined;
}

// Matching ---------------------------------------------------------------------------------------

const digits = (n: string) => n.replace(/^[A-Z0-9]{2}/, "").replace(/^0+/, "");
const callsignOf = (s: Seg) => {
  const icao = airlineIcao(s.provider_code ?? s.number.slice(0, 2));
  return icao ? `${icao}${digits(s.number)}` : null;
};
const normalise = (cs: string) => cs.replace(/^([A-Z]{3})0+/, "$1");

function bestMatch(s: Seg, legs: Leg[]): Leg | null {
  const due = new Date(s.scheduled_start_at ?? s.start_at).getTime() / 1000;
  const when = (l: Leg) => l.takeoff ?? l.first;
  const gap = (l: Leg) => Math.abs(when(l) - due);
  const closest = (list: Leg[], within: number) => list.filter((l) => gap(l) <= within).sort((a, b) => gap(a) - gap(b))[0] ?? null;
  const cs = callsignOf(s);
  const prefix = airlineIcao(s.provider_code ?? s.number.slice(0, 2));
  // A known end that is somewhere else rules a leg out.
  const fits = (l: Leg) => (!l.origin || !s.origin_code || l.origin === s.origin_code) && (!l.dest || !s.dest_code || l.dest === s.dest_code);
  // 1. Its own callsign (BAW117, or BAW117A: airlines add a letter), on a route that fits, within four hours.
  const own = cs ? new RegExp(`^${cs}[A-Z]?$`) : null;
  const exact = own ? closest(legs.filter((l) => l.callsign && own.test(normalise(l.callsign)) && fits(l)), 4 * 3600) : null;
  if (exact) return exact;
  // 2. The airline's alphanumeric callsign (EZY6PL) from the same airport: to the same destination
  //    within an hour, or, when the landing was not seen, the only such flight within 30 minutes.
  //    Busy hubs have many of an airline's departures close together, so a wrong guess is refused.
  if (prefix) {
    // Only alphanumeric callsigns: a plain number that differs (EIN152 for EI154) is another flight.
    const ours = legs.filter((l) => l.callsign?.startsWith(prefix) && /[A-Z]/.test(l.callsign.slice(prefix.length)) && l.origin === s.origin_code && fits(l));
    // Several of the airline's flights on the route within the hour: only one within 15 minutes will do.
    const sameDest = ours.filter((l) => l.dest === s.dest_code && gap(l) <= 60 * 60).sort((a, b) => gap(a) - gap(b));
    if (sameDest.length === 1 || (sameDest.length > 1 && gap(sameDest[0]) <= 15 * 60)) return sameDest[0];
    const near = ours.filter((l) => gap(l) <= 30 * 60);
    if (near.length === 1 && !near[0].dest) return near[0];
  }
  // 3. A codeshare (AA6143 flown by BA): the only flight seen on the whole route within 25 minutes.
  //    Aircraft take off after the timetabled departure (pushback and taxi), so not more than 10 minutes before it.
  const route = legs.filter((l) => l.origin === s.origin_code && l.dest === s.dest_code && l.takeoff && l.takeoff >= due - 600 && l.takeoff <= due + 25 * 60);
  return route.length === 1 ? route[0] : null;
}

// One day ----------------------------------------------------------------------------------------

async function wantedOn(day: string): Promise<Seg[]> {
  const next = new Date(new Date(`${day}T00:00:00Z`).getTime() + 864e5).toISOString();
  const { data, error } = await admin
    .from("segments")
    .select("id, number, provider_code, origin_code, dest_code, start_at, scheduled_start_at, aircraft, times_checked, origin_country, dest_country")
    .eq("type", "flight")
    .neq("status", "cancelled")
    .is("tail_number", null)
    .is("times_checked->>adsb", null)
    .not("number", "is", null)
    .gte("start_at", `${day}T00:00:00Z`)
    .lt("start_at", next)
    .limit(5000);
  if (error) throw error;
  return ((data ?? []) as Seg[]).filter((s) => !(BTS_COUNTRIES.includes(s.origin_country ?? "") && BTS_COUNTRIES.includes(s.dest_country ?? "")));
}

/** For testing: trace files from a folder, as if they came out of the archive. */
async function* sampleEntries(dir: string) {
  for (const entry of fs.readdirSync(dir, { recursive: true, withFileTypes: true }))
    if (entry.isFile()) {
      const file = path.join(entry.parentPath, entry.name);
      yield { name: file, body: fs.readFileSync(file) };
    }
}

async function processDay(day: string, sample?: string) {
  const wanted = await wantedOn(day);
  if (!wanted.length && !printLegs) return console.log(`${day}: nothing to look up`);
  const parts = sample ? [] : localTar ? localTar.split(",") : await archiveParts(day);
  if (!sample && !parts.length) {
    // Some early days were never published: after a week, stop waiting for them.
    if (day < new Date(Date.now() - 7 * 864e5).toISOString().slice(0, 10)) {
      for (const s of wanted) await admin.from("segments").update({ times_checked: { ...((s.times_checked as Record<string, string>) ?? {}), adsb: new Date().toISOString() } }).eq("id", s.id);
      return console.log(`${day}: no archive for this day`);
    }
    return console.log(`${day}: archive not published yet`);
  }

  // Only traces that broadcast a callsign of an airline someone flew that day are opened.
  const prefixes = [...new Set(wanted.map((s) => airlineIcao(s.provider_code ?? s.number.slice(0, 2))).filter((p): p is string => !!p))];
  const routes = new Set(wanted.flatMap((s) => [s.origin_code, s.dest_code]).filter(Boolean) as string[]);
  const wants = new RegExp(`"flight":"(${prefixes.join("|") || "ZZZZ"})`);
  const legs: Leg[] = [];
  let files = 0;
  let opened = 0;
  const started = Date.now();
  for await (const { name, body } of sample ? sampleEntries(sample) : tarEntries(download(parts))) {
    if (!/trace_full_[0-9a-f~]+\.json$/.test(name) || name.includes("~")) continue;
    files++;
    let text: string;
    try {
      text = (body[0] === 0x1f && body[1] === 0x8b ? gunzipSync(body) : body).toString("utf8");
    } catch {
      continue; // a damaged file
    }
    // Codeshares can be on any airline: keep traces near the day's airports too (checked after parsing).
    if (!wants.test(text) && !routes.size && !printLegs) continue;
    let trace: Trace;
    try {
      trace = JSON.parse(text) as Trace;
    } catch {
      continue;
    }
    const byCallsign = wants.test(text);
    for (const l of legsOf(trace)) {
      if (printLegs && l.callsign) console.log(`  ${l.registration ?? l.hex} ${l.type ?? ""} ${l.callsign} ${l.origin ?? "?"}-${l.dest ?? "?"} off ${l.takeoff ? new Date(l.takeoff * 1000).toISOString().slice(11, 16) : "?"} on ${l.landing ? new Date(l.landing * 1000).toISOString().slice(11, 16) : "?"}`);
      if (byCallsign || (l.origin && routes.has(l.origin)) || (l.dest && routes.has(l.dest))) legs.push(l);
    }
    opened++;
  }

  let found = 0;
  const stamp = new Date().toISOString();
  const matchedLegs: Leg[] = [];
  for (const s of wanted) {
    const leg = bestMatch(s, legs);
    const patch: Database["public"]["Tables"]["segments"]["Update"] = { times_checked: { ...((s.times_checked as Record<string, string>) ?? {}), adsb: stamp } };
    if (leg?.registration) {
      patch.tail_number = leg.registration.toUpperCase();
      patch.tail_source = "adsb";
      if (!s.aircraft && leg.type) patch.aircraft = aircraftName(leg.type) ?? leg.type;
      matchedLegs.push(leg);
      found++;
    }
    await admin.from("segments").update(patch).eq("id", s.id);
  }
  if (matchedLegs.length) {
    const rows = matchedLegs.map((l) => ({ day, hex: l.hex, registration: l.registration, type_code: l.type, callsign: l.callsign, origin: l.origin, dest: l.dest, takeoff_at: l.takeoff ? new Date(l.takeoff * 1000).toISOString() : null, landing_at: l.landing ? new Date(l.landing * 1000).toISOString() : null }));
    await admin.from("adsb_legs").upsert(rows, { onConflict: "day,hex,takeoff_at", ignoreDuplicates: true });
  }
  console.log(`${day}: ${files.toLocaleString("en-GB")} aircraft in the archive, ${opened.toLocaleString("en-GB")} read, ${legs.length.toLocaleString("en-GB")} legs kept; ${found} of ${wanted.length} flights given a tail number (${Math.round((Date.now() - started) / 1000)}s)`);
}

/** Days with flights waiting, oldest first; the archive for a day is published the day after. */
async function neededDays(max: number) {
  const days = new Set<string>();
  const yesterday = new Date(Date.now() - 864e5).toISOString().slice(0, 10);
  for (let at = 0; days.size < max * 4; at += 1000) {
    const { data } = await admin
      .from("segments")
      .select("start_at, origin_country, dest_country")
      .eq("type", "flight")
      .neq("status", "cancelled")
      .is("tail_number", null)
      .is("times_checked->>adsb", null)
      .not("number", "is", null)
      .gte("start_at", `${FIRST_DAY}T00:00:00Z`)
      .lt("start_at", `${yesterday}T00:00:00Z`)
      .order("start_at")
      .range(at, at + 999);
    for (const s of data ?? []) if (s.start_at && !(BTS_COUNTRIES.includes(s.origin_country ?? "") && BTS_COUNTRIES.includes(s.dest_country ?? ""))) days.add(s.start_at.slice(0, 10));
    if (!data || data.length < 1000) break;
  }
  return [...days].sort().slice(0, max);
}

const args = process.argv.slice(2);
const at = (flag: string) => (args.includes(flag) ? args[args.indexOf(flag) + 1] : undefined);
const printLegs = args.includes("--print-legs");
const localTar = at("--tar");
const days = at("--day") ? [at("--day")!] : args.includes("--needed") ? await neededDays(Number(at("--max-days") ?? 6)) : [];
if (!days.length && !args.includes("--needed")) throw new Error("Give --day YYYY-MM-DD or --needed [--max-days N]");
for (const day of days) await processDay(day, at("--sample"));
console.log(`adsb.lol: ${days.length} day${days.length === 1 ? "" : "s"} done`);