feat: refactor storage into storage interface, add sqlite storage provider

packages/opencode/src/cli/cmd/sqlite.ts

@@ -0,0 +1,195 @@
+import type { Argv } from "yargs"
+import { cmd } from "./cmd"
+import { UI } from "../ui"
+import { SqliteStorage } from "../../util/sqlite-storage"
+import { ENTITY_TYPES } from "../../storage/sqlite-provider"
+import * as prompts from "@clack/prompts"
+
+export const SqliteCommand = cmd({
+  command: "sqlite",
+  describe: "manage SQLite storage backend",
+  builder: (yargs: Argv) =>
+    yargs.command(SqliteInitCommand).command(SqliteImportCommand).command(SqliteExportCommand).demandCommand(),
+  async handler() {},
+})
+
+export const SqliteInitCommand = cmd({
+  command: "init",
+  describe: "initialize SQLite database",
+  builder: (yargs: Argv) =>
+    yargs
+      .option("force", {
+        describe: "overwrite existing database",
+        type: "boolean",
+        default: false,
+      })
+      .option("config", {
+        describe: "path to sqlite-storage.json config",
+        type: "string",
+      }),
+  handler: async (args) => {
+    UI.empty()
+    prompts.intro("Initialize SQLite database")
+
+    const storage = await SqliteStorage.create(args.config)
+    const dbPath = storage.dbPath()
+    const configPath = storage.configPath()
+
+    prompts.log.info(`Using schema: ${configPath}`)
+    prompts.log.info(`Database: ${dbPath}`)
+
+    if (await Bun.file(dbPath).exists()) {
+      if (!args.force) {
+        const confirm = await prompts.confirm({
+          message: `Database already exists at ${dbPath}. Overwrite?`,
+          initialValue: false,
+        })
+
+        if (prompts.isCancel(confirm) || !confirm) {
+          throw new UI.CancelledError()
+        }
+      }
+
+      await Bun.file(dbPath).writer().end()
+      await storage.init()
+      prompts.log.success(`Database re-initialized at ${dbPath}`)
+    } else {
+      await storage.init()
+      prompts.log.success(`Database created at ${dbPath}`)
+    }
+
+    prompts.outro("Done")
+  },
+})
+
+export const SqliteImportCommand = cmd({
+  command: "import",
+  describe: "import JSON storage to SQLite",
+  builder: (yargs: Argv) =>
+    yargs
+      .option("config", {
+        describe: "path to sqlite-storage.json config",
+        type: "string",
+      })
+      .option("entity", {
+        describe: "entity type to import (all if not specified)",
+        type: "string",
+        choices: ENTITY_TYPES as unknown as string[],
+      })
+      .option("verbose", {
+        describe: "show detailed progress",
+        type: "boolean",
+        default: false,
+      }),
+  handler: async (args) => {
+    UI.empty()
+    prompts.intro("Import JSON storage to SQLite")
+
+    try {
+      const storage = await SqliteStorage.create(args.config)
+      const dbPath = storage.dbPath()
+      const configPath = storage.configPath()
+
+      prompts.log.info(`Using schema: ${configPath}`)
+      prompts.log.info(`Database: ${dbPath}`)
+      prompts.log.info(`Source: JSON storage`)
+
+      if (!(await Bun.file(dbPath).exists())) {
+        prompts.log.error(`Database not found. Run 'opencode sqlite init' first`)
+        throw new UI.CancelledError()
+      }
+
+      const spinner = prompts.spinner()
+      spinner.start("Importing data...")
+
+      const result = await storage.importFromJSON({
+        entity: args.entity as any,
+        verbose: args.verbose,
+        onProgress: (entity, count) => {
+          if (args.verbose) {
+            spinner.message(`Importing ${entity}: ${count} records`)
+          }
+        },
+      })
+
+      spinner.stop("Import complete")
+
+      prompts.log.success(
+        `Imported ${result.message} messages, ${result.part} parts, ${result.session} sessions, ${result.project} projects, ${result.todo} todos`,
+      )
+
+      prompts.outro("Done")
+    } catch (error) {
+      prompts.log.error(`Import failed: ${error instanceof Error ? error.message : String(error)}`)
+      if (error instanceof Error && error.stack) {
+        console.error(error.stack)
+      }
+      throw error
+    }
+  },
+})
+
+export const SqliteExportCommand = cmd({
+  command: "export",
+  describe: "export SQLite storage to JSON",
+  builder: (yargs: Argv) =>
+    yargs
+      .option("config", {
+        describe: "path to sqlite-storage.json config",
+        type: "string",
+      })
+      .option("entity", {
+        describe: "entity type to export (all if not specified)",
+        type: "string",
+        choices: ENTITY_TYPES as unknown as string[],
+      })
+      .option("verbose", {
+        describe: "show detailed progress",
+        type: "boolean",
+        default: false,
+      })
+      .option("force", {
+        describe: "overwrite existing JSON files",
+        type: "boolean",
+        default: false,
+      }),
+  handler: async (args) => {
+    UI.empty()
+    prompts.intro("Export SQLite storage to JSON")
+
+    const storage = await SqliteStorage.create(args.config)
+    const dbPath = storage.dbPath()
+    const configPath = storage.configPath()
+
+    prompts.log.info(`Using schema: ${configPath}`)
+    prompts.log.info(`Database: ${dbPath}`)
+    prompts.log.info(`Target: JSON storage`)
+
+    if (!(await Bun.file(dbPath).exists())) {
+      prompts.log.error(`Database not found at ${dbPath}`)
+      throw new UI.CancelledError()
+    }
+
+    const spinner = prompts.spinner()
+    spinner.start("Exporting data...")
+
+    const result = await storage.exportToJSON({
+      entity: args.entity as any,
+      verbose: args.verbose,
+      force: args.force,
+      onProgress: (entity, count) => {
+        if (args.verbose) {
+          spinner.message(`Exporting ${entity}: ${count} records`)
+        }
+      },
+    })
+
+    spinner.stop("Export complete")
+
+    prompts.log.success(
+      `Exported ${result.message} messages, ${result.part} parts, ${result.session} sessions, ${result.project} projects, ${result.todo} todos`,
+    )
+
+    prompts.outro("Done")
+  },
+})

packages/opencode/src/config/config.ts

@@ -1024,6 +1024,19 @@ export namespace Config {
           prune: z.boolean().optional().describe("Enable pruning of old tool outputs (default: true)"),
         })
         .optional(),
+      storage: z
+        .object({
+          backend: z.enum(["json", "sqlite"]).optional().describe("Storage backend to use (default: json)"),
+          sqlite: z
+            .object({
+              database: z.string().optional().describe("Path to SQLite database file"),
+              config: z.string().optional().describe("Path to sqlite-storage.json config file"),
+            })
+            .optional()
+            .describe("SQLite storage configuration"),
+        })
+        .optional()
+        .describe("Storage backend configuration"),
       experimental: z
         .object({
           hook: z

packages/opencode/src/index.ts

@@ -26,6 +26,7 @@ import { EOL } from "os"
 import { WebCommand } from "./cli/cmd/web"
 import { PrCommand } from "./cli/cmd/pr"
 import { SessionCommand } from "./cli/cmd/session"
+import { SqliteCommand } from "./cli/cmd/sqlite"
 
 process.on("unhandledRejection", (e) => {
   Log.Default.error("rejection", {
@@ -97,6 +98,7 @@ const cli = yargs(hideBin(process.argv))
   .command(GithubCommand)
   .command(PrCommand)
   .command(SessionCommand)
+  .command(SqliteCommand)
   .fail((msg, err) => {
     if (
       msg?.startsWith("Unknown argument") ||

packages/opencode/src/project/instance.ts

@@ -5,17 +5,24 @@ import { State } from "./state"
 import { iife } from "@/util/iife"
 import { GlobalBus } from "@/bus/global"
 import { Filesystem } from "@/util/filesystem"
+import { Storage } from "../storage/storage"
 
-interface Context {
+interface InstanceContext {
   directory: string
   worktree: string
   project: Project.Info
 }
-const context = Context.create<Context>("instance")
-const cache = new Map<string, Promise<Context>>()
+const context = Context.create<InstanceContext>("instance")
+const cache = new Map<string, Promise<InstanceContext>>()
+let storageInitialized = false
 
 export const Instance = {
   async provide<R>(input: { directory: string; init?: () => Promise<any>; fn: () => R }): Promise<R> {
+    if (!storageInitialized) {
+      await Storage.init()
+      storageInitialized = true
+    }
+
     let existing = cache.get(input.directory)
     if (!existing) {
       Log.Default.info("creating instance", { directory: input.directory })

packages/opencode/src/session/index.ts

@@ -307,7 +307,7 @@ export namespace Session {
 
   export async function* list() {
     const project = Instance.project
-    for (const item of await Storage.list(["session", project.id])) {
+    for (const item of await Storage.list(["session", project.id], { orderBy: "-time.updated" })) {
       yield Storage.read<Info>(item)
     }
   }

packages/opencode/src/session/message-v2.ts

@@ -580,11 +580,10 @@ export namespace MessageV2 {
 
   export const parts = fn(Identifier.schema("message"), async (messageID) => {
     const result = [] as MessageV2.Part[]
-    for (const item of await Storage.list(["part", messageID])) {
+    for (const item of await Storage.list(["part", messageID], { orderBy: "id" })) {
       const read = await Storage.read<MessageV2.Part>(item)
       result.push(read)
     }
-    result.sort((a, b) => (a.id > b.id ? 1 : -1))
     return result
   })
 

packages/opencode/src/storage/json-provider.ts

@@ -0,0 +1,112 @@
+import { Log } from "../util/log"
+import path from "path"
+import fs from "fs/promises"
+import { Global } from "../global"
+import { Filesystem } from "../util/filesystem"
+import { Lock } from "../util/lock"
+import { StorageProvider } from "./provider"
+
+const log = Log.create({ service: "storage:json" })
+
+export class JsonStorageProvider implements StorageProvider.Interface {
+  constructor(private dir: string) {}
+
+  async read<T>(key: string[]): Promise<T> {
+    const target = path.join(this.dir, ...key) + ".json"
+    return this.withErrorHandling(async () => {
+      using _ = await Lock.read(target)
+      const result = await Bun.file(target).json()
+      return result as T
+    })
+  }
+
+  async write<T>(key: string[], content: T): Promise<void> {
+    const target = path.join(this.dir, ...key) + ".json"
+    return this.withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      await Bun.write(target, JSON.stringify(content, null, 2))
+    })
+  }
+
+  async update<T>(key: string[], fn: (draft: T) => void): Promise<T> {
+    const target = path.join(this.dir, ...key) + ".json"
+    return this.withErrorHandling(async () => {
+      using _ = await Lock.write(target)
+      const content = await Bun.file(target).json()
+      fn(content)
+      await Bun.write(target, JSON.stringify(content, null, 2))
+      return content as T
+    })
+  }
+
+  async remove(key: string[]): Promise<void> {
+    const target = path.join(this.dir, ...key) + ".json"
+    return this.withErrorHandling(async () => {
+      await fs.unlink(target).catch(() => {})
+    })
+  }
+
+  async list(prefix: string[], options?: StorageProvider.ListOptions): Promise<string[][]> {
+    const glob = new Bun.Glob("**/*")
+    try {
+      const result = await Array.fromAsync(
+        glob.scan({
+          cwd: path.join(this.dir, ...prefix),
+          onlyFiles: true,
+        }),
+      ).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)]))
+      result.sort()
+
+      // Apply orderBy if specified (requires reading data)
+      if (options?.orderBy) {
+        const { field, desc } = this.parseOrderBy(options.orderBy)
+        const withData = await Promise.all(
+          result.map(async (key) => ({
+            key,
+            data: await this.read(key),
+          })),
+        )
+        withData.sort((a, b) => {
+          const aVal = this.getNestedValue(a.data, field)
+          const bVal = this.getNestedValue(b.data, field)
+          const cmp = aVal > bVal ? 1 : aVal < bVal ? -1 : 0
+          return desc ? -cmp : cmp
+        })
+        const sorted = withData.map((x) => x.key)
+        return options.limit ? sorted.slice(0, options.limit) : sorted
+      }
+
+      // Apply limit without orderBy
+      return options?.limit ? result.slice(0, options.limit) : result
+    } catch {
+      return []
+    }
+  }
+
+  private parseOrderBy(orderBy: string): { field: string; desc: boolean } {
+    const desc = orderBy.startsWith("-")
+    const field = desc ? orderBy.slice(1) : orderBy
+    return { field, desc }
+  }
+
+  private getNestedValue(obj: any, path: string): any {
+    const parts = path.split(".")
+    let current = obj
+    for (const part of parts) {
+      if (current === undefined || current === null) return undefined
+      current = current[part]
+    }
+    return current
+  }
+
+  private async withErrorHandling<T>(body: () => Promise<T>): Promise<T> {
+    return body().catch((e) => {
+      if (!(e instanceof Error)) throw e
+      const errnoException = e as NodeJS.ErrnoException
+      if (errnoException.code === "ENOENT") {
+        throw new StorageProvider.NotFoundError({ message: `Resource not found: ${errnoException.path}` })
+      }
+      throw e
+    })
+  }
+}