Use Readability (JSDOM) and Query de-structuring

.env.example

@@ -8,11 +8,16 @@ SERPER_API=APIKEYGOESHERE
 BRAVE_SEARCH_API_KEY=APIKEYGOESHERE
 
 
+
 # OPTIONAL - Set LAN GPU server, examples:
 # PC | http://localhost:11434/v1
 # LAN GPU server | http://192.168.1.100:11434/v1
 OLLAMA_BASE_URL=http://localhost:11434/v1
 
+# Model selection (make sure to use a supported model)
+INFERENCE_MODEL=gemma3:4b
+# Remove any legacy model references like llama-3.1-70b-versatile
+
 # OPTIONAL - Rate Limiting: https://console.upstash.com/redis
 UPSTASH_REDIS_REST_URL=https://EXAMPLE.upstash.io
 UPSTASH_REDIS_REST_TOKEN=APIKEYGOESHERE

app/action.tsx

@@ -3,6 +3,7 @@
 import { createAI, createStreamableValue } from 'ai/rsc';
 import { config } from './config';
 import { functionCalling } from './function-calling';
+import type { SearchResult } from '@/components/answer/SearchResultsComponent';
 import { getSearchResults, getImages, getVideos } from './tools/searchProviders';
 import { get10BlueLinksContents, processAndVectorizeContent } from './tools/contentProcessing';
 import { setInSemanticCache, clearSemanticCache, initializeSemanticCache, getFromSemanticCache } from './tools/semanticCache';
@@ -30,28 +31,31 @@ async function myAction(userMessage: string, mentionTool: string | null, logo: s
       await lookupTool(mentionTool, userMessage, streamable, file);
     }
 
-    const [images, sources, videos, conditionalFunctionCallUI] = await Promise.all([
+    const [images, sourcesRaw, videos, conditionalFunctionCallUI] = await Promise.all([
       getImages(userMessage),
       getSearchResults(userMessage),
       getVideos(userMessage),
       functionCalling(userMessage),
     ]);
 
-    streamable.update({ searchResults: sources, images, videos });
+    // For UI: stream decomposed queries
+    streamable.update({ searchResults: sourcesRaw, images, videos });
 
     if (config.useFunctionCalling) {
       streamable.update({ conditionalFunctionCallUI });
     }
 
-    const html = await get10BlueLinksContents(sources);
+    // For vectorization and LLM: flatten sources
+    const sources = Array.isArray(sourcesRaw[0]) ? sourcesRaw.flat() : sourcesRaw;
+    const html = await get10BlueLinksContents(sources as SearchResult[]);
     const vectorResults = await processAndVectorizeContent(html, userMessage);
     const accumulatedLLMResponse = await streamingChatCompletion(userMessage, vectorResults, streamable);
-    const followUp = await relevantQuestions(sources, userMessage);
+    const followUp = await relevantQuestions(sources as SearchResult[], userMessage);
 
     streamable.update({ followUp });
 
     setInSemanticCache(userMessage, {
-      searchResults: sources,
+      searchResults: sourcesRaw,
       images,
       videos,
       conditionalFunctionCallUI: config.useFunctionCalling ? conditionalFunctionCallUI : undefined,

app/config.tsx

@@ -5,19 +5,34 @@
 // - IMPORTANT: Follow-up questions are not yet implrmented with Ollama models, only OpenAI compatible models that use  {type: "json_object"}
 
 export const config = {
-    useOllamaInference: false,
-    useOllamaEmbeddings: false,
+    // Enable Ollama as primary inference and embeddings
+    useOllamaInference: true,
+    useOllamaEmbeddings: true,
     searchProvider: 'serper', // 'serper', 'google' // 'serper' is the default
-    inferenceModel: 'llama-3.1-70b-versatile', // Groq: 'mixtral-8x7b-32768', 'gemma-7b-it' // OpenAI: 'gpt-3.5-turbo', 'gpt-4' // Ollama 'mistral', 'llama3' etc
-    inferenceAPIKey: process.env.GROQ_API_KEY, // Groq: process.env.GROQ_API_KEY // OpenAI: process.env.OPENAI_API_KEY // Ollama: 'ollama' is the default
-    embeddingsModel: 'text-embedding-3-small', // Ollama: 'llama2', 'nomic-embed-text' // OpenAI 'text-embedding-3-small', 'text-embedding-3-large'
-    textChunkSize: 800, // Recommended to decrease for Ollama
-    textChunkOverlap: 200, // Recommended to decrease for Ollama
-    numberOfSimilarityResults: 4, // Number of similarity results to return per page
-    numberOfPagesToScan: 10, // Recommended to decrease for Ollama
-    nonOllamaBaseURL: 'https://api.groq.com/openai/v1', //Groq: https://api.groq.com/openai/v1 // OpenAI: https://api.openai.com/v1 
-    useFunctionCalling: true, // Set to true to enable function calling and conditional streaming UI (currently in beta)
-    useRateLimiting: false, // Uses Upstash rate limiting to limit the number of requests per user
-    useSemanticCache: false, // Uses Upstash semantic cache to store and retrieve data for faster response times
-    usePortkey: false, // Uses Portkey for AI Gateway in @mentions (currently in beta) see config-tools.tsx to configure + mentionTools.tsx for source code
+    // Set your local Ollama model name here (e.g., 'gemma3:4b')
+    inferenceModel: 'gemma3:4b',
+    // API key for OpenAI (fallback)
+    openaiAPIKey: process.env.OPENAI_API_KEY,
+    // API key for Groq
+    groqAPIKey: process.env.GROQ_API_KEY,
+    // API key for Ollama (not required, but set to 'ollama' for compatibility)
+    ollamaAPIKey: 'ollama',
+    // Embeddings model (Ollama or OpenAI)
+    embeddingsModel: 'gemma3:4b',
+    textChunkSize: 400, // Lower for local models for faster response
+    textChunkOverlap: 100, // Lower for local models for faster response
+    numberOfSimilarityResults: 4,
+    numberOfPagesToScan: 5, // Lower for local models for faster response
+    // Fallback base URL for OpenAI
+    openaiBaseURL: 'https://api.openai.com/v1',
+    // Groq base URL
+    groqBaseURL: 'https://api.groq.com/openai/v1',
+    // Ollama base URL
+    ollamaBaseURL: 'http://host.docker.internal:11434/v1', // Docker can reach Ollama via host.docker.internal
+    useFunctionCalling: true,
+    useRateLimiting: false,
+    useSemanticCache: false,
+    usePortkey: false,
+    // Fallback logic: if Ollama fails, use OpenAI
+    fallbackProvider: 'openai',
 }

app/function-calling.tsx

@@ -3,10 +3,26 @@ import { OpenAI } from 'openai';
 import { config } from './config';
 import { SpotifyApi } from "@spotify/web-api-ts-sdk";
 
-const client = new OpenAI({
-    baseURL: config.nonOllamaBaseURL,
-    apiKey: config.inferenceAPIKey
-});
+
+function getOpenAIClient(provider: 'ollama' | 'openai' | 'groq' = 'ollama') {
+    if (provider === 'ollama') {
+        return new OpenAI({
+            baseURL: config.ollamaBaseURL,
+            apiKey: config.ollamaAPIKey
+        });
+    } else if (provider === 'openai') {
+        return new OpenAI({
+            baseURL: config.openaiBaseURL,
+            apiKey: config.openaiAPIKey
+        });
+    } else {
+        return new OpenAI({
+            baseURL: config.groqBaseURL,
+            apiKey: config.groqAPIKey
+        });
+    }
+}
+
 const MODEL = config.inferenceModel;
 
 const api = SpotifyApi.withClientCredentials(
@@ -165,13 +181,27 @@ export async function functionCalling(query: string) {
                 },
             },
         ];
-        const response = await client.chat.completions.create({
-            model: MODEL,
-            messages: messages,
-            tools: tools,
-            tool_choice: "auto",
-            max_tokens: 4096,
-        });
+        let client = getOpenAIClient('ollama');
+        let response;
+        try {
+            response = await client.chat.completions.create({
+                model: MODEL,
+                messages: [
+                    { role: 'user', content: query }
+                ],
+                stream: false
+            });
+        } catch (err) {
+            // Fallback to OpenAI if Ollama fails
+            client = getOpenAIClient('openai');
+            response = await client.chat.completions.create({
+                model: 'gpt-3.5-turbo',
+                messages: [
+                    { role: 'user', content: query }
+                ],
+                stream: false
+            });
+        }
         const responseMessage = response.choices[0].message;
         const toolCalls = responseMessage.tool_calls;
         if (toolCalls) {

app/page.tsx

@@ -6,7 +6,7 @@ import { type AI } from './action';
 import { ChatScrollAnchor } from '@/lib/hooks/chat-scroll-anchor';
 import Textarea from 'react-textarea-autosize';
 import { useEnterSubmit } from '@/lib/hooks/use-enter-submit';
-import { Tooltip, TooltipContent, TooltipTrigger, } from '@/components/ui/tooltip';
+import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
 import { Button } from '@/components/ui/button';
 import dynamic from 'next/dynamic';
 // Main components 
@@ -34,6 +34,7 @@ interface SearchResult {
   favicon: string;
   link: string;
   title: string;
+  readabilityContent?: string;
 }
 interface Message {
   falBase64Image: any;
@@ -336,7 +337,38 @@ export default function Page() {
                     {message.spotify && message.spotify.length > 0 && (
                       <Spotify key={`financialChart-${index}`} spotify={message.spotify} />
                     )}
-                    {message.searchResults && (<SearchResultsComponent key={`searchResults-${index}`} searchResults={message.searchResults} />)}
+                    {/* Enhanced: Show decomposed queries and Readability content */}
+                    {message.searchResults && Array.isArray(message.searchResults) && message.searchResults.length > 0 && (
+                      Array.isArray(message.searchResults[0]) ? (
+                        <div className="mb-4">
+                          <h3 className="font-bold text-lg mb-2">Decomposed Queries</h3>
+                          {(message.searchResults as any[]).map((subResults, subIdx) => Array.isArray(subResults) ? (
+                            <div key={`decomposed-query-${index}-${subIdx}`} className="mb-2 p-2 border rounded bg-gray-50 dark:bg-slate-900">
+                              <div className="font-semibold text-md mb-1">Sub-query {subIdx + 1}</div>
+                              <SearchResultsComponent searchResults={subResults} />
+                              {/* Show Readability content if present in any result */}
+                              {subResults.map((result, rIdx) => result.readabilityContent && (
+                                <ExpandableReadabilityContent
+                                  key={`readability-${index}-${subIdx}-${rIdx}`}
+                                  content={result.readabilityContent}
+                                />
+                              ))}
+                            </div>
+                          ) : null)}
+                        </div>
+                      ) : (
+                        <>
+                          <SearchResultsComponent key={`searchResults-${index}`} searchResults={message.searchResults as SearchResult[]} />
+                          {/* Show Readability content if present in any result */}
+                          {(message.searchResults as SearchResult[]).map((result, rIdx) => result.readabilityContent && (
+                            <ExpandableReadabilityContent
+                              key={`readability-${index}-${rIdx}`}
+                              content={result.readabilityContent}
+                            />
+                          ))}
+                        </>
+                      )
+                    )}
                     {message.places && message.places.length > 0 && (
                       <MapComponent key={`map-${index}`} places={message.places} />
                     )}
@@ -512,4 +544,34 @@ export default function Page() {
       <div className="pb-[80px] pt-4 md:pt-10"></div>
     </div>
   );
-};
+}
+// End of Page component
+
+// --- ExpandableReadabilityContent: UI component for expand/collapse and tooltip ---
+function ExpandableReadabilityContent({ content }: { content: string }) {
+  const [expanded, setExpanded] = useState(false);
+  return (
+    <div className="mt-2 p-2 border-l-4 border-blue-400 bg-blue-50 dark:bg-blue-900">
+      <div className="flex items-center font-semibold text-blue-700 dark:text-blue-200">
+        <span>Main Content Extracted</span>
+        <Tooltip>
+          <TooltipTrigger asChild>
+            <span className="ml-2 cursor-pointer text-blue-400" title="What is this?">ⓘ</span>
+          </TooltipTrigger>
+          <TooltipContent>
+            This is the main readable content extracted from the page using Mozilla Readability.
+          </TooltipContent>
+        </Tooltip>
+        <button
+          className="ml-4 px-2 py-1 text-xs bg-blue-200 dark:bg-blue-800 rounded"
+          onClick={() => setExpanded((prev) => !prev)}
+        >
+          {expanded ? 'Collapse' : 'Expand'}
+        </button>
+      </div>
+      {expanded && (
+        <div className="text-sm whitespace-pre-line mt-2">{content}</div>
+      )}
+    </div>
+  );
+}

app/tools/contentProcessing.tsx

@@ -1,21 +1,58 @@
 
 import { config } from '../config';
 import cheerio from 'cheerio';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+// Use Mozilla Readability to extract main content from HTML
+function extractReadableContent(html: string): string {
+    try {
+        const dom = new JSDOM(html);
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        return article?.textContent || '';
+    } catch (e) {
+        return '';
+    }
+}
+
+// Query decomposition: generate multiple search queries for complex questions
+export async function decomposeQuery(userQuery: string): Promise<string[]> {
+    // Use your LLM or a prompt to generate decomposed queries
+    // Example prompt:
+    // "Your role is to generate a few short and specific search queries to answer the QUERY below. List 3 options, 1 per line. QUERY: ..."
+    // For now, return a simple split for demonstration
+    // Replace with LLM call for production
+    return userQuery.split(/[.,;\n]/).map(q => q.trim()).filter(q => q.length > 0);
+}
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
 import { MemoryVectorStore } from 'langchain/vectorstores/memory';
 import { Document as DocumentInterface } from 'langchain/document';
 import { OpenAIEmbeddings } from '@langchain/openai';
 import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama";
+
+function getEmbeddingsProvider(provider: 'ollama' | 'openai' = 'ollama') {
+    if (provider === 'ollama') {
+        return new OllamaEmbeddings({
+            model: config.embeddingsModel,
+            baseUrl: config.ollamaBaseURL.replace('/v1','')
+        });
+    } else {
+        return new OpenAIEmbeddings({
+            modelName: config.embeddingsModel
+        });
+    }
+}
+
 let embeddings: OllamaEmbeddings | OpenAIEmbeddings;
-if (config.useOllamaEmbeddings) {
-    embeddings = new OllamaEmbeddings({
-        model: config.embeddingsModel,
-        baseUrl: "http://localhost:11434"
-    });
-} else {
-    embeddings = new OpenAIEmbeddings({
-        modelName: config.embeddingsModel
-    });
+try {
+    if (config.useOllamaEmbeddings) {
+        embeddings = getEmbeddingsProvider('ollama');
+    } else {
+        embeddings = getEmbeddingsProvider('openai');
+    }
+} catch (err) {
+    // fallback to OpenAI if Ollama fails
+    embeddings = getEmbeddingsProvider('openai');
 }
 
 interface SearchResult {
@@ -45,6 +82,10 @@ export async function get10BlueLinksContents(sources: SearchResult[]): Promise<C
         }
     }
     function extractMainContent(html: string): string {
+        // Use Readability for better extraction
+        const readable = extractReadableContent(html);
+        if (readable && readable.length > 100) return readable;
+        // fallback to cheerio
         try {
             const $ = cheerio.load(html);
             $("script, style, head, nav, footer, iframe, img").remove();