fix(pgvector): resolve the deb version from the pgdg pool instead of pinning 0.8.1-2

README.md

@@ -648,6 +648,21 @@ pgserve --ram --pgvector
 
 When `--pgvector` is enabled, every new database automatically has the vector extension installed. No SQL setup required.
 
+On Linux the `vector` extension files are fetched once from the pgdg apt pool
+(`postgresql-<major>-pgvector_<ver>.pgdg+1_<arch>.deb`). The version is
+resolved from the pool listing at install time (highest wins) because pgdg
+removes superseded packages; if the listing is unreachable a short list of
+known versions is tried in order. Overrides:
+
+| Env var | Effect |
+|---|---|
+| `AUTOPG_PGVECTOR_VERSION=<ver>` | Pin one pool version (e.g. `0.8.6-1`) instead of resolving. |
+| `AUTOPG_PGVECTOR_DEB=<file.deb>` | Install a local `.deb`; no network access. |
+
+A failed install never stops the postmaster, but it is logged with the
+versions tried and the override to use; `CREATE EXTENSION vector` fails until
+it is fixed.
+
 <details>
 <summary><b>Using pgvector</b></summary>
 

src/pgvector-version.js

@@ -0,0 +1,180 @@
+/**
+ * pgvector .deb version resolution for the auto-installer in `src/postgres.js`.
+ *
+ * pgdg garbage-collects superseded packages from its pool, so any hard-coded
+ * `postgresql-<major>-pgvector_<ver>.pgdg+1_<arch>.deb` URL eventually 404s
+ * (issue #145: the pinned `0.8.1-2` vanished once `0.8.5`/`0.8.6` shipped).
+ *
+ * This module is pure (no I/O, no `this`) so the precedence rules can be unit
+ * tested without touching the network. The installer injects the listing
+ * fetcher and the environment.
+ *
+ * Precedence, highest first:
+ *   1. `AUTOPG_PGVECTOR_VERSION=<ver>` — operator pin, no listing fetch.
+ *   2. The pgdg pool directory listing — every matching version, highest
+ *      first (so a just-removed package is skipped in favour of the next).
+ *   3. `FALLBACK_PGVECTOR_VERSIONS` — a short known-good list tried in order
+ *      when the listing is unreachable or contains no match.
+ *
+ * `AUTOPG_PGVECTOR_DEB=<file>` (a local .deb, skips the network entirely) is
+ * handled by the installer itself; see `parsePgvectorDebFilename` for the
+ * version it records in `vector.meta.json`.
+ */
+
+/* global fetch, AbortSignal */
+
+export const PGVECTOR_POOL_URL = 'https://apt.postgresql.org/pub/repos/apt/pool/main/p/pgvector/';
+
+/**
+ * Known-good versions tried in order when the pool listing is unreachable.
+ * Newest first. Update when pgdg moves on — this list is only a safety net.
+ */
+export const FALLBACK_PGVECTOR_VERSIONS = Object.freeze(['0.8.6-1', '0.8.5-1', '0.8.1-2']);
+
+/**
+ * Build the pool download URL for one candidate version. The `+` in
+ * `.pgdg+1` must be percent-encoded in the URL path.
+ */
+export function pgvectorDebUrl({ pgMajor, arch, version }) {
+  return `${PGVECTOR_POOL_URL}postgresql-${pgMajor}-pgvector_${version}.pgdg%2B1_${arch}.deb`;
+}
+
+/**
+ * Compare two Debian version strings (dpkg semantics, minus epochs).
+ * Returns <0, 0, >0 like a sort comparator.
+ *
+ * Algorithm (from deb-version(7)): compare the upstream part, then the
+ * Debian revision. Each part is compared by alternating non-digit and digit
+ * runs; non-digit runs compare lexically with `~` sorting before anything
+ * (even the empty string) and letters before non-letters; digit runs
+ * compare numerically.
+ */
+export function compareDebianVersions(a, b) {
+  const [aUp, aRev] = splitRevision(String(a));
+  const [bUp, bRev] = splitRevision(String(b));
+  return comparePart(aUp, bUp) || comparePart(aRev, bRev);
+}
+
+function splitRevision(v) {
+  const idx = v.lastIndexOf('-');
+  if (idx === -1) return [v, ''];
+  return [v.slice(0, idx), v.slice(idx + 1)];
+}
+
+function charOrder(c) {
+  if (c === '~') return -1;
+  if (c === '') return 0;
+  if (/[A-Za-z]/.test(c)) return c.charCodeAt(0);
+  return c.charCodeAt(0) + 256;
+}
+
+function compareNonDigit(x, y) {
+  const len = Math.max(x.length, y.length);
+  for (let i = 0; i < len; i++) {
+    const d = charOrder(x[i] ?? '') - charOrder(y[i] ?? '');
+    if (d !== 0) return d;
+  }
+  return 0;
+}
+
+function comparePart(x, y) {
+  let i = 0;
+  let j = 0;
+  while (i < x.length || j < y.length) {
+    // Non-digit run
+    let xs = '';
+    let ys = '';
+    while (i < x.length && !/\d/.test(x[i])) xs += x[i++];
+    while (j < y.length && !/\d/.test(y[j])) ys += y[j++];
+    const nd = compareNonDigit(xs, ys);
+    if (nd !== 0) return nd;
+    // Digit run
+    let xd = '';
+    let yd = '';
+    while (i < x.length && /\d/.test(x[i])) xd += x[i++];
+    while (j < y.length && /\d/.test(y[j])) yd += y[j++];
+    const xn = xd === '' ? 0 : Number(xd);
+    const yn = yd === '' ? 0 : Number(yd);
+    if (xn !== yn) return xn - yn;
+  }
+  return 0;
+}
+
+/**
+ * Extract every pgvector version present in a pgdg pool directory listing
+ * for the given PG major + Debian arch. Returns unique versions sorted
+ * highest first. Tolerates both `+` and `%2B` in the `.pgdg+1` suffix (the
+ * listing uses `%2B` in `href` and `+` in the link text).
+ *
+ * @param {string} html - Raw directory listing body.
+ * @param {{pgMajor: string|number, arch: string}} target
+ * @returns {string[]}
+ */
+export function parsePgvectorPoolListing(html, { pgMajor, arch }) {
+  if (typeof html !== 'string' || html.length === 0) return [];
+  const re = new RegExp(
+    `postgresql-${escapeRe(String(pgMajor))}-pgvector_(\\d+\\.\\d+\\.\\d+-\\d+)\\.pgdg(?:\\+|%2B)1_${escapeRe(arch)}\\.deb`,
+    'g',
+  );
+  const found = new Set();
+  for (const m of html.matchAll(re)) found.add(m[1]);
+  return [...found].sort((x, y) => compareDebianVersions(y, x));
+}
+
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Recover the `<ver>` from a pool-style .deb filename or path. Used to
+ * record an honest version in `vector.meta.json` when the operator installs
+ * a local .deb via `AUTOPG_PGVECTOR_DEB`. Returns null when the name does
+ * not follow the pgdg pattern.
+ */
+export function parsePgvectorDebFilename(file) {
+  const m = String(file).match(/pgvector_(\d+\.\d+\.\d+-\d+)\.pgdg(?:\+|%2B)?\d*_/);
+  return m ? m[1] : null;
+}
+
+/**
+ * Resolve the ordered list of pgvector .deb versions the installer should
+ * try. Never throws: an unreachable listing degrades to the fallback list.
+ *
+ * @param {object} args
+ * @param {string|number} args.pgMajor
+ * @param {string} args.arch - Debian arch (`amd64` | `arm64`).
+ * @param {Record<string,string|undefined>} [args.env] - defaults to `process.env`.
+ * @param {(url: string) => Promise<string>} [args.fetchListing] - returns the
+ *   listing body; any throw is treated as "unreachable". Defaults to
+ *   `fetchPoolListing`.
+ * @returns {Promise<{versions: string[], source: 'pin'|'pool'|'fallback', error?: string}>}
+ */
+export async function resolvePgvectorDebVersions({ pgMajor, arch, env = process.env, fetchListing = fetchPoolListing }) {
+  const pin = (env.AUTOPG_PGVECTOR_VERSION || '').trim();
+  if (pin) {
+    return { versions: [pin], source: 'pin' };
+  }
+
+  let error;
+  try {
+    const html = await fetchListing(PGVECTOR_POOL_URL);
+    const versions = parsePgvectorPoolListing(html, { pgMajor, arch });
+    if (versions.length > 0) {
+      return { versions, source: 'pool' };
+    }
+    error = `no postgresql-${pgMajor}-pgvector ${arch} package found in pool listing`;
+  } catch (err) {
+    error = err && err.message ? err.message : String(err);
+  }
+  return { versions: [...FALLBACK_PGVECTOR_VERSIONS], source: 'fallback', error };
+}
+
+/**
+ * Default listing fetcher: bounded so an unreachable apt mirror cannot stall
+ * postmaster startup indefinitely.
+ */
+async function fetchPoolListing(url, timeoutMs = 15_000) {
+  const res = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+  if (!res.ok) throw new Error(`pool listing fetch failed: ${res.status}`);
+  return res.text();
+}

src/postgres.js

@@ -13,14 +13,20 @@
  * - No locale dependency (works on any system)
  */
 
-/* global fetch, Bun */
+/* global fetch, Bun, AbortSignal */
 import { EventEmitter } from 'events';
 import os from 'os';
 import path from 'path';
 import fs from 'fs';
 import crypto from 'crypto';
 import { loadEffectiveConfig } from './settings-loader.cjs';
 import { buildPostgresArgs } from './settings-pg-args.cjs';
+import {
+  PGVECTOR_POOL_URL,
+  parsePgvectorDebFilename,
+  pgvectorDebUrl,
+  resolvePgvectorDebVersions,
+} from './pgvector-version.js';
 
 /**
  * Get platform key for binary lookup (e.g., 'windows-x64', 'linux-x64', 'darwin-arm64')
@@ -1313,7 +1319,13 @@ export class PostgresManager extends EventEmitter {
     try {
       await this._installPgvectorFromDeb({ pgMajor, ...paths });
     } catch (error) {
-      this.logger.warn({ err: error.message }, 'Failed to install pgvector extension files (non-fatal)');
+      // Non-fatal for the postmaster, but never quiet: the message names the
+      // versions tried and the AUTOPG_PGVECTOR_DEB / AUTOPG_PGVECTOR_VERSION
+      // escape hatches (issue #145).
+      this.logger.warn(
+        { err: error.message, pgMajor },
+        'Failed to install pgvector extension files (non-fatal) — CREATE EXTENSION vector will fail until fixed',
+      );
     }
   }
 
@@ -1421,16 +1433,12 @@ export class PostgresManager extends EventEmitter {
       return;
     }
 
-    // Download prebuilt pgvector .deb from apt.postgresql.org (HTTPS)
-    // Version 0.8.1-2 — update when new releases ship
-    const pgvectorVersion = '0.8.1-2';
-    const debUrl = `https://apt.postgresql.org/pub/repos/apt/pool/main/p/pgvector/postgresql-${pgMajor}-pgvector_${pgvectorVersion}.pgdg%2B1_${arch}.deb`;
-    this.logger.info({ url: debUrl, pgMajor }, 'Downloading pgvector...');
-
-    const res = await fetch(debUrl);
-    if (!res.ok) throw new Error(`Download failed: ${res.status}`);
-
-    const buffer = Buffer.from(await res.arrayBuffer());
+    // Obtain the .deb: a local file (AUTOPG_PGVECTOR_DEB, no network) or a
+    // download from the pgdg pool. pgdg garbage-collects superseded
+    // packages, so the version is resolved from the pool listing instead of
+    // being pinned (issue #145); `resolvePgvectorDebVersions` documents the
+    // pin / pool / fallback precedence.
+    const { buffer, pgvectorVersion, sourceUrl } = await this._obtainPgvectorDeb({ pgMajor, arch });
 
     // Extract .deb (it's an ar archive containing data.tar.xz)
     const tmpDir = path.join(os.tmpdir(), `pgserve-pgvector-${process.pid}-${Date.now()}`);
@@ -1484,18 +1492,73 @@ export class PostgresManager extends EventEmitter {
       this._writePgvectorMeta(vectorMeta, {
         pgMajor,
         pgvectorVersion,
-        sourceUrl: debUrl,
+        sourceUrl,
         postgresPath: this.binaries.postgres,
         installedAt: new Date().toISOString(),
       });
 
-      this.logger.info({ pgMajor, pgvectorVersion }, 'pgvector extension installed successfully');
+      this.logger.info({ pgMajor, pgvectorVersion, sourceUrl }, 'pgvector extension installed successfully');
     } finally {
       // Always clean up tmpdir, even on failure
       fs.rmSync(tmpDir, { recursive: true, force: true });
     }
   }
 
+  /**
+   * Return the pgvector .deb bytes plus the version/source to record in
+   * `vector.meta.json`.
+   *
+   * - `AUTOPG_PGVECTOR_DEB=<file>`: read the local file, skip the network.
+   * - Otherwise resolve candidate versions (pin → pool listing → fallback
+   *   list) and download the first one that exists. Every miss is logged;
+   *   when all candidates fail the error names each attempt and the
+   *   `AUTOPG_PGVECTOR_DEB` / `AUTOPG_PGVECTOR_VERSION` escape hatches so
+   *   the non-fatal warning upstream is actionable.
+   */
+  async _obtainPgvectorDeb({ pgMajor, arch }) {
+    const localDeb = (process.env.AUTOPG_PGVECTOR_DEB || '').trim();
+    if (localDeb) {
+      if (!fs.existsSync(localDeb)) {
+        throw new Error(`AUTOPG_PGVECTOR_DEB points at a missing file: ${localDeb}`);
+      }
+      const pgvectorVersion = parsePgvectorDebFilename(localDeb) || 'local';
+      this.logger.info({ file: localDeb, pgMajor, pgvectorVersion }, 'Installing pgvector from local .deb (AUTOPG_PGVECTOR_DEB)');
+      return { buffer: fs.readFileSync(localDeb), pgvectorVersion, sourceUrl: `file://${path.resolve(localDeb)}` };
+    }
+
+    const resolved = await resolvePgvectorDebVersions({ pgMajor, arch });
+    if (resolved.source === 'fallback') {
+      this.logger.warn(
+        { err: resolved.error, versions: resolved.versions },
+        'pgvector pool listing unavailable — trying known versions in order',
+      );
+    } else {
+      this.logger.debug({ source: resolved.source, versions: resolved.versions }, 'Resolved pgvector .deb candidates');
+    }
+
+    const attempts = [];
+    for (const version of resolved.versions) {
+      const url = pgvectorDebUrl({ pgMajor, arch, version });
+      this.logger.info({ url, pgMajor, pgvectorVersion: version }, 'Downloading pgvector...');
+      try {
+        const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
+        if (!res.ok) {
+          attempts.push(`${version} (HTTP ${res.status})`);
+          continue;
+        }
+        return { buffer: Buffer.from(await res.arrayBuffer()), pgvectorVersion: version, sourceUrl: url };
+      } catch (err) {
+        attempts.push(`${version} (${err && err.message ? err.message : String(err)})`);
+      }
+    }
+
+    throw new Error(
+      `pgvector .deb download failed for PostgreSQL ${pgMajor}/${arch} — tried ${attempts.join(', ')}. `
+      + `Workaround: download postgresql-${pgMajor}-pgvector_<ver>.pgdg+1_${arch}.deb from ${PGVECTOR_POOL_URL} `
+      + 'and restart with AUTOPG_PGVECTOR_DEB=<file>, or pin a known version with AUTOPG_PGVECTOR_VERSION=<ver>.',
+    );
+  }
+
   /**
    * Tear down an existing pgvector install and reinstall from scratch.
    * Called reactively when CREATE EXTENSION surfaces an ABI mismatch —