From e0ee7d1e446b18c05ed3166b0aa3b199ec021f1a Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Tue, 28 Jan 2025 11:59:53 -0800 Subject: [PATCH 1/8] fix: only perform lancedb cpu check for linux --- core/config/load.ts | 15 +++++++-------- core/config/util.ts | 38 ++++++++++++-------------------------- core/util/GlobalContext.ts | 2 +- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/core/config/load.ts b/core/config/load.ts index af71ca1725..e856c04c46 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -3,6 +3,7 @@ import * as fs from "fs"; import os from "os"; import path from "path"; +import { ConfigResult, ConfigValidationError } from "@continuedev/config-yaml"; import { fetchwithRequestOptions } from "@continuedev/fetch"; import * as JSONC from "comment-json"; import * as tar from "tar"; @@ -38,6 +39,7 @@ import CustomContextProviderClass from "../context/providers/CustomContextProvid import FileContextProvider from "../context/providers/FileContextProvider"; import { contextProviderClassFromName } from "../context/providers/index"; import PromptFilesContextProvider from "../context/providers/PromptFilesContextProvider"; +import { useHub } from "../control-plane/env"; import { allEmbeddingsProviders } from "../indexing/allEmbeddingsProviders"; import { BaseLLM } from "../llm"; import { llmFromDescription } from "../llm/llms"; @@ -61,8 +63,8 @@ import { getContinueDotEnv, getEsbuildBinaryPath, } from "../util/paths"; +import { localPathToUri } from "../util/pathToUri"; -import { ConfigResult, ConfigValidationError } from "@continuedev/config-yaml"; import { defaultContextProvidersJetBrains, defaultContextProvidersVsCode, @@ -70,9 +72,7 @@ import { defaultSlashCommandsVscode, } from "./default"; import { getSystemPromptDotFile } from "./getSystemPromptDotFile"; -// import { isSupportedLanceDbCpuTarget } from "./util"; -import { useHub } from "../control-plane/env"; -import { localPathToUri } from "../util/pathToUri"; +import { isSupportedLanceDbCpuTargetForLinux } from "./util"; import { validateConfig } from "./validation.js"; function resolveSerializedConfig(filepath: string): SerializedContinueConfig { @@ -175,10 +175,9 @@ function loadSerializedConfig( ? [...defaultSlashCommandsVscode] : [...defaultSlashCommandsJetBrains]; - // Temporarily disabling this check until we can verify the commands are accuarate - // if (!isSupportedLanceDbCpuTarget(ide)) { - // config.disableIndexing = true; - // } + if (os.platform() === "linux" && !isSupportedLanceDbCpuTargetForLinux(ide)) { + config.disableIndexing = true; + } return { config, errors, configLoadInterrupted: false }; } diff --git a/core/config/util.ts b/core/config/util.ts index 6d28dcbcbd..0dfd97667a 100644 --- a/core/config/util.ts +++ b/core/config/util.ts @@ -108,11 +108,13 @@ export function getModelByRole( * * See here for details: https://github.com/continuedev/continue/issues/940 */ -export function isSupportedLanceDbCpuTarget(ide: IDE) { +export function isSupportedLanceDbCpuTargetForLinux(ide: IDE) { const CPU_FEATURES_TO_CHECK = ["avx2", "fma"] as const; const globalContext = new GlobalContext(); - const globalContextVal = globalContext.get("isSupportedLanceDbCpuTarget"); + const globalContextVal = globalContext.get( + "isSupportedLanceDbCpuTargetForLinux", + ); // If we've already checked the CPU target, return the cached value if (globalContextVal !== undefined) { @@ -120,51 +122,35 @@ export function isSupportedLanceDbCpuTarget(ide: IDE) { } const arch = os.arch(); - const platform = os.platform(); // This check only applies to x64 //https://github.com/lancedb/lance/issues/2195#issuecomment-2057841311 if (arch !== "x64") { - globalContext.update("isSupportedLanceDbCpuTarget", true); + globalContext.update("isSupportedLanceDbCpuTargetForLinux", true); return true; } try { - const cpuFlags = (() => { - switch (platform) { - case "darwin": - return execSync("sysctl -n machdep.cpu.features") - .toString() - .toLowerCase(); - case "linux": - return execSync("cat /proc/cpuinfo").toString().toLowerCase(); - case "win32": - return execSync("wmic cpu get caption /format:list") - .toString() - .toLowerCase(); - default: - return ""; - } - })(); + const cpuFlags = execSync("cat /proc/cpuinfo").toString().toLowerCase(); - const isSupportedLanceDbCpuTarget = cpuFlags + const isSupportedLanceDbCpuTargetForLinux = cpuFlags ? CPU_FEATURES_TO_CHECK.every((feature) => cpuFlags.includes(feature)) : true; // If it's not a supported CPU target, and it's the first time we are checking, // show a toast to inform the user that we are going to disable indexing. - if (!isSupportedLanceDbCpuTarget) { + if (!isSupportedLanceDbCpuTargetForLinux) { // We offload our async toast to `showUnsupportedCpuToast` to prevent making - // our config loading async upstream of `isSupportedLanceDbCpuTarget` + // our config loading async upstream of `isSupportedLanceDbCpuTargetForLinux` void showUnsupportedCpuToast(ide); } globalContext.update( - "isSupportedLanceDbCpuTarget", - isSupportedLanceDbCpuTarget, + "isSupportedLanceDbCpuTargetForLinux", + isSupportedLanceDbCpuTargetForLinux, ); - return isSupportedLanceDbCpuTarget; + return isSupportedLanceDbCpuTargetForLinux; } catch (error) { // If we can't determine CPU features, default to true return true; diff --git a/core/util/GlobalContext.ts b/core/util/GlobalContext.ts index 336dc8293b..9f452a90ff 100644 --- a/core/util/GlobalContext.ts +++ b/core/util/GlobalContext.ts @@ -15,7 +15,7 @@ export type GlobalContextType = { hasDismissedConfigTsNoticeJetBrains: boolean; hasAlreadyCreatedAPromptFile: boolean; showConfigUpdateToast: boolean; - isSupportedLanceDbCpuTarget: boolean; + isSupportedLanceDbCpuTargetForLinux: boolean; }; /** From 8c6d5eaeec0e83dfe4a97fe15e664d66b574f398 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Wed, 29 Jan 2025 11:12:01 -0800 Subject: [PATCH 2/8] Update More.tsx --- gui/src/pages/More/More.tsx | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/gui/src/pages/More/More.tsx b/gui/src/pages/More/More.tsx index b543353b7f..9e3f507209 100644 --- a/gui/src/pages/More/More.tsx +++ b/gui/src/pages/More/More.tsx @@ -15,15 +15,20 @@ import DocsIndexingStatuses from "../../components/indexing/DocsIndexingStatuses import PageHeader from "../../components/PageHeader"; import { useAppDispatch, useAppSelector } from "../../redux/hooks"; import { saveCurrentSession } from "../../redux/thunks/session"; +import { updateConfig } from "../../redux/slices/configSlice"; function MorePage() { useNavigationListener(); const dispatch = useAppDispatch(); const navigate = useNavigate(); const ideMessenger = useContext(IdeMessengerContext); - const disableIndexing = useAppSelector( - (state) => state.config.config.disableIndexing, - ); + const config = useAppSelector((store) => store.config.config); + const { disableIndexing } = config; + function openConfig() { + // `undefined` will open the config for the current profile + const profileId = undefined; + ideMessenger.post("config/openProfile", { profileId }); + } return (
@@ -37,9 +42,23 @@ function MorePage() { Local embeddings of your codebase
- {disableIndexing ? ( -
- Indexing is disabled + {!disableIndexing ? ( +
+

+ Indexing is disabled +

+

+ + Open your config{" "} + + + and set disableIndexing to true to + enable indexing + +

) : ( From 90c663146edc965bbe0b87b737755742315a8fa1 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 6 Feb 2025 12:44:48 -0800 Subject: [PATCH 3/8] feat: more dynamic imports for LanceDB --- core/config/load.ts | 14 +--- core/config/util.ts | 4 +- .../pipelines/BaseRetrievalPipeline.ts | 19 ++++- core/indexing/CodebaseIndexer.ts | 25 +++--- core/indexing/LanceDbIndex.test.ts | 10 +-- core/indexing/LanceDbIndex.ts | 81 ++++++++++++------- core/indexing/docs/DocsService.ts | 43 ++++++++-- core/indexing/docs/migrations.ts | 2 +- gui/src/pages/More/More.tsx | 12 ++- 9 files changed, 138 insertions(+), 72 deletions(-) diff --git a/core/config/load.ts b/core/config/load.ts index e856c04c46..f899978750 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -31,9 +31,10 @@ import { import { slashCommandFromDescription, slashFromCustomCommand, -} from "../commands/index.js"; +} from "../commands/index"; import { AllRerankers } from "../context/allRerankers"; import { MCPManagerSingleton } from "../context/mcp"; +import CodebaseContextProvider from "../context/providers/CodebaseContextProvider"; import ContinueProxyContextProvider from "../context/providers/ContinueProxyContextProvider"; import CustomContextProviderClass from "../context/providers/CustomContextProvider"; import FileContextProvider from "../context/providers/FileContextProvider"; @@ -73,7 +74,7 @@ import { } from "./default"; import { getSystemPromptDotFile } from "./getSystemPromptDotFile"; import { isSupportedLanceDbCpuTargetForLinux } from "./util"; -import { validateConfig } from "./validation.js"; +import { validateConfig } from "./validation"; function resolveSerializedConfig(filepath: string): SerializedContinueConfig { let content = fs.readFileSync(filepath, "utf8"); @@ -234,13 +235,6 @@ export function isContextProviderWithParams( return (contextProvider as ContextProviderWithParams).name !== undefined; } -const getCodebaseProvider = async (params: any) => { - const { default: CodebaseContextProvider } = await import( - "../context/providers/CodebaseContextProvider" - ); - return new CodebaseContextProvider(params); -}; - /** Only difference between intermediate and final configs is the `models` array */ async function intermediateToFinalConfig( config: Config, @@ -405,7 +399,7 @@ async function intermediateToFinalConfig( new FileContextProvider({}), // Add codebase provider if indexing is enabled ...(!config.disableIndexing - ? [await getCodebaseProvider(codebaseContextParams)] + ? [new CodebaseContextProvider(codebaseContextParams)] : []), // Add prompt files provider if enabled ...(loadPromptFiles ? [new PromptFilesContextProvider({})] : []), diff --git a/core/config/util.ts b/core/config/util.ts index 0dfd97667a..d9737529c9 100644 --- a/core/config/util.ts +++ b/core/config/util.ts @@ -108,7 +108,7 @@ export function getModelByRole( * * See here for details: https://github.com/continuedev/continue/issues/940 */ -export function isSupportedLanceDbCpuTargetForLinux(ide: IDE) { +export function isSupportedLanceDbCpuTargetForLinux(ide?: IDE) { const CPU_FEATURES_TO_CHECK = ["avx2", "fma"] as const; const globalContext = new GlobalContext(); @@ -139,7 +139,7 @@ export function isSupportedLanceDbCpuTargetForLinux(ide: IDE) { // If it's not a supported CPU target, and it's the first time we are checking, // show a toast to inform the user that we are going to disable indexing. - if (!isSupportedLanceDbCpuTargetForLinux) { + if (!isSupportedLanceDbCpuTargetForLinux && ide) { // We offload our async toast to `showUnsupportedCpuToast` to prevent making // our config loading async upstream of `isSupportedLanceDbCpuTargetForLinux` void showUnsupportedCpuToast(ide); diff --git a/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts b/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts index 0ba94964c4..50b87cc828 100644 --- a/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts +++ b/core/context/retrieval/pipelines/BaseRetrievalPipeline.ts @@ -31,12 +31,16 @@ export interface IRetrievalPipeline { export default class BaseRetrievalPipeline implements IRetrievalPipeline { private ftsIndex = new FullTextSearchCodebaseIndex(); - private lanceDbIndex: LanceDbIndex; + private lanceDbIndex: LanceDbIndex | null = null; constructor(protected readonly options: RetrievalPipelineOptions) { - this.lanceDbIndex = new LanceDbIndex( - options.config.embeddingsProvider, - (uri) => options.ide.readFile(uri), + void this.initLanceDb(); + } + + private async initLanceDb() { + this.lanceDbIndex = await LanceDbIndex.create( + this.options.config.embeddingsProvider, + (uri) => this.options.ide.readFile(uri), ); } @@ -125,6 +129,13 @@ export default class BaseRetrievalPipeline implements IRetrievalPipeline { input: string, n: number, ): Promise { + if (!this.lanceDbIndex) { + console.warn( + "LanceDB index not available, skipping embeddings retrieval", + ); + return []; + } + return this.lanceDbIndex.retrieve( input, n, diff --git a/core/indexing/CodebaseIndexer.ts b/core/indexing/CodebaseIndexer.ts index 677cc43057..367e5e462a 100644 --- a/core/indexing/CodebaseIndexer.ts +++ b/core/indexing/CodebaseIndexer.ts @@ -5,8 +5,8 @@ import { IContinueServerClient } from "../continueServer/interface.js"; import { IDE, IndexingProgressUpdate, IndexTag } from "../index.js"; import { extractMinimalStackTraceInfo } from "../util/extractMinimalStackTraceInfo.js"; import { getIndexSqlitePath, getLanceDbPath } from "../util/paths.js"; - import { findUriInDirs, getUriPathBasename } from "../util/uri.js"; + import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex.js"; import { CodeSnippetsCodebaseIndex } from "./CodeSnippetsIndex.js"; import { FullTextSearchCodebaseIndex } from "./FullTextSearchCodebaseIndex.js"; @@ -80,24 +80,31 @@ export class CodebaseIndexer { return []; } - const indexes = [ + const indexes: CodebaseIndex[] = [ new ChunkCodebaseIndex( this.ide.readFile.bind(this.ide), this.continueServerClient, config.embeddingsProvider.maxEmbeddingChunkSize, ), // Chunking must come first - new LanceDbIndex( - config.embeddingsProvider, - this.ide.readFile.bind(this.ide), - this.continueServerClient, - ), + ]; + + const lanceDbIndex = await LanceDbIndex.create( + config.embeddingsProvider, + this.ide.readFile.bind(this.ide), + this.continueServerClient, + ); + + if (lanceDbIndex) { + indexes.push(lanceDbIndex); + } + + indexes.push( new FullTextSearchCodebaseIndex(), new CodeSnippetsCodebaseIndex(this.ide), - ]; + ); return indexes; } - public async refreshFile( file: string, workspaceDirs: string[], diff --git a/core/indexing/LanceDbIndex.test.ts b/core/indexing/LanceDbIndex.test.ts index 08c8f99825..b33dc8450f 100644 --- a/core/indexing/LanceDbIndex.test.ts +++ b/core/indexing/LanceDbIndex.test.ts @@ -3,15 +3,15 @@ import lance from "vectordb"; import { testConfigHandler, testIde } from "../test/fixtures"; import { getLanceDbPath } from "../util/paths"; + +import { LanceDbIndex } from "./LanceDbIndex"; +import { DatabaseConnection, SqliteDb } from "./refreshIndex"; import { mockPathAndCacheKey, mockTag, testContinueServerClient, updateIndexAndAwaitGenerator, } from "./test/indexing"; - -import { LanceDbIndex } from "./LanceDbIndex"; -import { DatabaseConnection, SqliteDb } from "./refreshIndex"; import { IndexResultType } from "./types"; jest.useFakeTimers(); @@ -31,11 +31,11 @@ describe.skip("ChunkCodebaseIndex", () => { throw new Error("Failed to load config"); } - index = new LanceDbIndex( + index = (await LanceDbIndex.create( mockConfig.embeddingsProvider, testIde.readFile.bind(testIde), testContinueServerClient, - ); + ))!; sqliteDb = await SqliteDb.get(); lanceDb = await lance.connect(getLanceDbPath()); diff --git a/core/indexing/LanceDbIndex.ts b/core/indexing/LanceDbIndex.ts index d11feb1d1d..702a3f9926 100644 --- a/core/indexing/LanceDbIndex.ts +++ b/core/indexing/LanceDbIndex.ts @@ -1,30 +1,30 @@ -// NOTE: vectordb requirement must be listed in extensions/vscode to avoid error import { RunResult } from "sqlite3"; import { v4 as uuidv4 } from "uuid"; -import lance, { Table } from "vectordb"; -import { IContinueServerClient } from "../continueServer/interface.js"; +import { isSupportedLanceDbCpuTargetForLinux } from "../config/util"; +import { IContinueServerClient } from "../continueServer/interface"; import { BranchAndDir, Chunk, ILLM, IndexTag, IndexingProgressUpdate, -} from "../index.js"; -import { getLanceDbPath, migrate } from "../util/paths.js"; +} from "../index"; +import { getLanceDbPath, migrate } from "../util/paths"; +import { getUriPathBasename } from "../util/uri"; -import { chunkDocument, shouldChunk } from "./chunk/chunk.js"; -import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex.js"; +import { chunkDocument, shouldChunk } from "./chunk/chunk"; +import { DatabaseConnection, SqliteDb, tagToString } from "./refreshIndex"; import { CodebaseIndex, IndexResultType, MarkCompleteCallback, PathAndCacheKey, RefreshIndexResults, -} from "./types.js"; -import { getUriPathBasename } from "../util/uri.js"; +} from "./types"; + +import type * as LanceType from "vectordb"; -// LanceDB converts to lowercase, so names must all be lowercase interface LanceDbRow { uuid: string; path: string; @@ -38,16 +38,53 @@ type ItemWithChunks = { item: PathAndCacheKey; chunks: Chunk[] }; type ChunkMap = Map; export class LanceDbIndex implements CodebaseIndex { + private static lance: typeof LanceType | null = null; + relativeExpectedTime: number = 13; get artifactId(): string { return `vectordb::${this.embeddingsProvider.embeddingId}`; } - constructor( + /** + * Factory method for creating LanceDbIndex instances. + * + * We dynamically import LanceDB only when supported to avoid native module loading errors + * on incompatible platforms. LanceDB has CPU-specific native dependencies that can crash + * the application if loaded on unsupported architectures. + * + * See isSupportedLanceDbCpuTargetForLinux() for platform compatibility details. + */ + static async create( + embeddingsProvider: ILLM, + readFile: (filepath: string) => Promise, + continueServerClient?: IContinueServerClient, + ): Promise { + if (!isSupportedLanceDbCpuTargetForLinux()) { + return null; + } + + try { + this.lance = await import("vectordb"); + return new LanceDbIndex( + embeddingsProvider, + readFile, + continueServerClient, + ); + } catch (err) { + console.error("Failed to load LanceDB:", err); + return null; + } + } + + private constructor( private readonly embeddingsProvider: ILLM, private readonly readFile: (filepath: string) => Promise, private readonly continueServerClient?: IContinueServerClient, - ) {} + ) { + if (!LanceDbIndex.lance) { + throw new Error("LanceDB not initialized"); + } + } tableNameForTag(tag: IndexTag) { return tagToString(tag).replace(/[^\w-_.]/g, ""); @@ -97,12 +134,10 @@ export class LanceDbIndex implements CodebaseIndex { ); const embeddings = await this.getEmbeddings(allChunks); - // Remove undefined embeddings and their corresponding chunks for (let i = embeddings.length - 1; i >= 0; i--) { if (embeddings[i] === undefined) { const chunk = allChunks[i]; const chunks = chunkMap.get(chunk.filepath)?.chunks; - if (chunks) { const index = chunks.findIndex((c) => c === chunk); if (index !== -1) { @@ -204,6 +239,7 @@ export class LanceDbIndex implements CodebaseIndex { markComplete: MarkCompleteCallback, repoName: string | undefined, ): AsyncGenerator { + const lance = LanceDbIndex.lance!; const sqliteDb = await SqliteDb.get(); await this.createSqliteCacheTable(sqliteDb); @@ -211,15 +247,13 @@ export class LanceDbIndex implements CodebaseIndex { const lanceDb = await lance.connect(getLanceDbPath()); const existingLanceTables = await lanceDb.tableNames(); - let lanceTable: Table | undefined = undefined; + let lanceTable: LanceType.Table | undefined = undefined; let needToCreateLanceTable = !existingLanceTables.includes(lanceTableName); - // Compute const addComputedLanceDbRows = async ( pathAndCacheKeys: PathAndCacheKey[], computedRows: LanceDbRow[], ) => { - // Create table if needed, add computed rows if (lanceTable) { if (computedRows.length > 0) { await lanceTable.add(computedRows); @@ -235,11 +269,9 @@ export class LanceDbIndex implements CodebaseIndex { needToCreateLanceTable = false; } - // Mark item complete await markComplete(pathAndCacheKeys, IndexResultType.Compute); }; - // Check remote cache if (this.continueServerClient?.connected) { try { const keys = results.compute.map(({ cacheKey }) => cacheKey); @@ -249,7 +281,6 @@ export class LanceDbIndex implements CodebaseIndex { repoName, ); for (const [cacheKey, chunks] of Object.entries(resp.files)) { - // Get path for cacheKey const path = results.compute.find( (item) => item.cacheKey === cacheKey, )?.path; @@ -261,7 +292,6 @@ export class LanceDbIndex implements CodebaseIndex { continue; } - // Build LanceDbRow objects const rows: LanceDbRow[] = []; for (const chunk of chunks) { const row = { @@ -288,7 +318,6 @@ export class LanceDbIndex implements CodebaseIndex { await addComputedLanceDbRows([{ cacheKey, path }], rows); } - // Remove items that don't need to be recomputed results.compute = results.compute.filter( (item) => !resp.files[item.cacheKey], ); @@ -310,7 +339,6 @@ export class LanceDbIndex implements CodebaseIndex { await addComputedLanceDbRows(results.compute, dbRows); let accumulatedProgress = 0; - // Add tag - retrieve the computed info from lance sqlite cache for (const { path, cacheKey } of results.addTag) { const stmt = await sqliteDb.prepare( "SELECT * FROM lance_db_cache WHERE cacheKey = ? AND path = ? AND artifact_id = ?", @@ -354,7 +382,6 @@ export class LanceDbIndex implements CodebaseIndex { }; } - // Delete or remove tag - remove from lance table) if (!needToCreateLanceTable) { const toDel = [...results.removeTag, ...results.del]; @@ -363,7 +390,6 @@ export class LanceDbIndex implements CodebaseIndex { } for (const { path, cacheKey } of toDel) { - // This is where the aforementioned lowercase conversion problem shows await lanceTable.delete( `cachekey = '${cacheKey}' AND path = '${path}'`, ); @@ -379,7 +405,6 @@ export class LanceDbIndex implements CodebaseIndex { await markComplete(results.removeTag, IndexResultType.RemoveTag); - // Delete - also remove from sqlite cache for (const { path, cacheKey } of results.del) { await sqliteDb.run( "DELETE FROM lance_db_cache WHERE cacheKey = ? AND path = ? AND artifact_id = ?", @@ -409,7 +434,7 @@ export class LanceDbIndex implements CodebaseIndex { n: number, directory: string | undefined, vector: number[], - db: any, /// lancedb.Connection + db: any, ): Promise { const tableName = this.tableNameForTag(tag); const tableNames = await db.tableNames(); @@ -421,7 +446,6 @@ export class LanceDbIndex implements CodebaseIndex { const table = await db.openTable(tableName); let query = table.search(vector); if (directory) { - // seems like lancedb is only post-filtering, so have to return a bunch of results and slice after query = query.where(`path LIKE '${directory}%'`).limit(300); } else { query = query.limit(n); @@ -436,6 +460,7 @@ export class LanceDbIndex implements CodebaseIndex { tags: BranchAndDir[], filterDirectory: string | undefined, ): Promise { + const lance = LanceDbIndex.lance!; const [vector] = await this.embeddingsProvider.embed([query]); const db = await lance.connect(getLanceDbPath()); diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index c8507f256b..861dacb524 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -1,6 +1,6 @@ +import { ConfigResult } from "@continuedev/config-yaml"; import { open, type Database } from "sqlite"; import sqlite3 from "sqlite3"; -import lancedb, { Connection } from "vectordb"; import { Chunk, @@ -12,7 +12,10 @@ import { SiteIndexingConfig, } from "../.."; import { ConfigHandler } from "../../config/ConfigHandler"; -import { addContextProvider } from "../../config/util"; +import { + addContextProvider, + isSupportedLanceDbCpuTargetForLinux, +} from "../../config/util"; import DocsContextProvider from "../../context/providers/DocsContextProvider"; import TransformersJsEmbeddingsProvider from "../../llm/llms/TransformersJsEmbeddingsProvider"; import { FromCoreProtocol, ToCoreProtocol } from "../../protocol"; @@ -31,7 +34,6 @@ import { markdownPageToArticleWithChunks, } from "./article"; import DocsCrawler, { DocsCrawlerType, PageData } from "./crawlers/DocsCrawler"; -import { ConfigResult } from "@continuedev/config-yaml"; import { runLanceMigrations, runSqliteMigrations } from "./migrations"; import { downloadFromS3, @@ -41,6 +43,8 @@ import { } from "./preIndexed"; import preIndexedDocs from "./preIndexedDocs"; +import type * as LanceType from "vectordb"; + // Purposefully lowercase because lancedb converts export interface LanceDbDocsRow { title: string; @@ -79,6 +83,7 @@ export type AddParams = { - add/index one */ export default class DocsService { + private static lance: typeof LanceType | null = null; static lanceTableName = "docs"; static sqlitebTableName = "docs"; @@ -110,6 +115,22 @@ export default class DocsService { this.githubToken = token; } + private async initLanceDb() { + if (!isSupportedLanceDbCpuTargetForLinux()) { + return null; + } + + try { + if (!DocsService.lance) { + DocsService.lance = await import("vectordb"); + } + return DocsService.lance; + } catch (err) { + console.error("Failed to load LanceDB:", err); + return null; + } + } + // Singleton pattern: only one service globally private static instance?: DocsService; static createSingleton( @@ -889,7 +910,7 @@ export default class DocsService { // Lance DB Initialization private async createLanceDocsTable( - connection: Connection, + connection: LanceType.Connection, initializationVector: number[], tableName: string, ) { @@ -936,7 +957,12 @@ export default class DocsService { initializationVector: number[]; startUrl: string; }) { - const conn = await lancedb.connect(getLanceDbPath()); + const lance = await this.initLanceDb(); + if (!lance) { + throw new Error("LanceDB not available on this platform"); + } + + const conn = await lance.connect(getLanceDbPath()); const tableNames = await conn.tableNames(); const { provider } = await this.getEmbeddingsProvider(startUrl); const tableNameFromEmbeddingsProvider = @@ -1045,8 +1071,13 @@ export default class DocsService { // Delete methods private async deleteEmbeddingsFromLance(startUrl: string) { + const lance = await this.initLanceDb(); + if (!lance) { + return; + } + for (const tableName of this.lanceTableNamesSet) { - const conn = await lancedb.connect(getLanceDbPath()); + const conn = await lance.connect(getLanceDbPath()); const table = await conn.openTable(tableName); await table.delete(`starturl = '${startUrl}'`); } diff --git a/core/indexing/docs/migrations.ts b/core/indexing/docs/migrations.ts index 38aa987b35..64e3131758 100644 --- a/core/indexing/docs/migrations.ts +++ b/core/indexing/docs/migrations.ts @@ -1,5 +1,5 @@ import { type Database } from "sqlite"; -import { Table } from "vectordb"; +import { type Table } from "vectordb"; import { editConfigJson, migrate } from "../../util/paths.js"; diff --git a/gui/src/pages/More/More.tsx b/gui/src/pages/More/More.tsx index 9e3f507209..d391f5ef29 100644 --- a/gui/src/pages/More/More.tsx +++ b/gui/src/pages/More/More.tsx @@ -15,7 +15,6 @@ import DocsIndexingStatuses from "../../components/indexing/DocsIndexingStatuses import PageHeader from "../../components/PageHeader"; import { useAppDispatch, useAppSelector } from "../../redux/hooks"; import { saveCurrentSession } from "../../redux/thunks/session"; -import { updateConfig } from "../../redux/slices/configSlice"; function MorePage() { useNavigationListener(); @@ -24,11 +23,6 @@ function MorePage() { const ideMessenger = useContext(IdeMessengerContext); const config = useAppSelector((store) => store.config.config); const { disableIndexing } = config; - function openConfig() { - // `undefined` will open the config for the current profile - const profileId = undefined; - ideMessenger.post("config/openProfile", { profileId }); - } return (
@@ -50,7 +44,11 @@ function MorePage() {

+ ideMessenger.post("config/openProfile", { + profileId: undefined, + }) + } > Open your config{" "} From a1104552d87734545661f98c449e1f7a4edc393c Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 6 Feb 2025 12:46:30 -0800 Subject: [PATCH 4/8] Update load.ts --- core/config/load.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/config/load.ts b/core/config/load.ts index 3f725cff1f..89baf6ba7b 100644 --- a/core/config/load.ts +++ b/core/config/load.ts @@ -41,7 +41,6 @@ import FileContextProvider from "../context/providers/FileContextProvider"; import { contextProviderClassFromName } from "../context/providers/index"; import PromptFilesContextProvider from "../context/providers/PromptFilesContextProvider"; import { useHub } from "../control-plane/env"; -import { useHub } from "../control-plane/env"; import { allEmbeddingsProviders } from "../indexing/allEmbeddingsProviders"; import { BaseLLM } from "../llm"; import { llmFromDescription } from "../llm/llms"; @@ -66,7 +65,6 @@ import { getEsbuildBinaryPath, } from "../util/paths"; import { localPathToUri } from "../util/pathToUri"; -import { localPathToUri } from "../util/pathToUri"; import { defaultContextProvidersJetBrains, @@ -76,6 +74,7 @@ import { } from "./default"; import { getSystemPromptDotFile } from "./getSystemPromptDotFile"; import { modifyContinueConfigWithSharedConfig } from "./sharedConfig"; +import { isSupportedLanceDbCpuTargetForLinux } from "./util"; import { validateConfig } from "./validation.js"; export function resolveSerializedConfig( From 19b1f9a2f2f591ac70542c3fdbfa3146d3a95e29 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 6 Feb 2025 12:49:23 -0800 Subject: [PATCH 5/8] Update GlobalContext.ts --- core/util/GlobalContext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/util/GlobalContext.ts b/core/util/GlobalContext.ts index aa1b66fcf3..b656385e05 100644 --- a/core/util/GlobalContext.ts +++ b/core/util/GlobalContext.ts @@ -25,7 +25,7 @@ export type GlobalContextType = { hasDismissedConfigTsNoticeJetBrains: boolean; hasAlreadyCreatedAPromptFile: boolean; showConfigUpdateToast: boolean; - isSupportedLanceDbCpuTarget: boolean; + isSupportedLanceDbCpuTargetForLinux: boolean; sharedConfig: SharedConfigSchema; }; From 13a24f441f58d65b2617c5998823e9c3d7906fe8 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 6 Feb 2025 13:16:17 -0800 Subject: [PATCH 6/8] Update DocsService.ts --- core/indexing/docs/DocsService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index 6429cc400e..a0e8bbbcae 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -43,7 +43,6 @@ import { SiteIndexingResults, } from "./preIndexed"; import preIndexedDocs from "./preIndexedDocs"; -import { ConfigResult } from "@continuedev/config-yaml"; import type * as LanceType from "vectordb"; From 6178d0d75010987850d90d531e91680cc446a525 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Thu, 6 Feb 2025 13:18:37 -0800 Subject: [PATCH 7/8] Update util.ts --- core/config/util.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/config/util.ts b/core/config/util.ts index d9737529c9..fec026c097 100644 --- a/core/config/util.ts +++ b/core/config/util.ts @@ -1,4 +1,4 @@ -import { execSync } from "child_process"; +import fs from "fs"; import os from "os"; import { @@ -131,7 +131,7 @@ export function isSupportedLanceDbCpuTargetForLinux(ide?: IDE) { } try { - const cpuFlags = execSync("cat /proc/cpuinfo").toString().toLowerCase(); + const cpuFlags = fs.readFileSync("/proc/cpuinfo", "utf-8").toLowerCase(); const isSupportedLanceDbCpuTargetForLinux = cpuFlags ? CPU_FEATURES_TO_CHECK.every((feature) => cpuFlags.includes(feature)) From c9ee5dcc93dff252ac174bf79032f0083fe28795 Mon Sep 17 00:00:00 2001 From: Patrick Erichsen Date: Fri, 7 Feb 2025 12:49:04 -0800 Subject: [PATCH 8/8] docs: update info on disabled indexing for linux-x64 --- core/config/util.ts | 6 ++++-- docs/docs/troubleshooting.md | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/config/util.ts b/core/config/util.ts index fec026c097..3fc1ab4654 100644 --- a/core/config/util.ts +++ b/core/config/util.ts @@ -160,11 +160,13 @@ export function isSupportedLanceDbCpuTargetForLinux(ide?: IDE) { async function showUnsupportedCpuToast(ide: IDE) { const shouldOpenLink = await ide.showToast( "warning", - "Codebase indexing is disabled due to CPU incompatibility", + "Codebase indexing disabled - Your Linux system lacks required CPU features (AVX2, FMA)", "Learn more", ); if (shouldOpenLink) { - void ide.openUrl("https://github.com/continuedev/continue/pull/3551"); + void ide.openUrl( + "https://docs.continue.dev/troubleshooting#i-received-a-codebase-indexing-disabled---your-linux-system-lacks-required-cpu-features-avx2-fma-notification", + ); } } diff --git a/docs/docs/troubleshooting.md b/docs/docs/troubleshooting.md index 52dc785b29..6a345b7dd2 100644 --- a/docs/docs/troubleshooting.md +++ b/docs/docs/troubleshooting.md @@ -134,6 +134,14 @@ This can be accomplished using the following command: `Continue: Rebuild codebas This can be fixed by selecting `Actions > Choose Boot runtime for the IDE` then selecting the latest version, and then restarting Android Studio. [See this thread](https://github.com/continuedev/continue/issues/596#issuecomment-1789327178) for details. +### I received a "Codebase indexing disabled - Your Linux system lacks required CPU features (AVX2, FMA)" notification + +We use LanceDB as our vector database for codebase search features. On x64 Linux systems, LanceDB requires specific CPU features (FMA and AVX2) which may not be available on older processors. + +Most Continue features will work normally, including autocomplete and chat. However, commands that rely on codebase indexing, such as `@codebase`, `@files`, and `@folder`, will be disabled. + +For more details about this requirement, see the [LanceDB issue #2195](https://github.com/lancedb/lance/issues/2195). + ## Still having trouble? You can also join our Discord community [here](https://discord.gg/vapESyrFmJ) for additional support and discussions. Alternatively, you can create a GitHub issue [here](https://github.com/continuedev/continue/issues/new?assignees=&labels=bug&projects=&template=bug-report-%F0%9F%90%9B.md&title=), providing details of your problem, and we'll be able to help you out more quickly.