diff --git a/.changeset/three-sheep-return.md b/.changeset/three-sheep-return.md new file mode 100644 index 000000000..b7b47c234 --- /dev/null +++ b/.changeset/three-sheep-return.md @@ -0,0 +1,6 @@ +--- +'@microfox/tracker': patch +'microfox': patch +--- + +minor fixes in tracker workflow creation diff --git a/.github/workflows/trackers.yml b/.github/workflows/trackers.yml new file mode 100644 index 000000000..c087ab3cb --- /dev/null +++ b/.github/workflows/trackers.yml @@ -0,0 +1,33 @@ +name: Automated Trackers +'on': + push: + branches: + - main +jobs: + sync-micro-sdks-documentation: + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + token: ${{ secrets.ORG_PAT }} + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install Dependencies + run: npm install + - name: Run Tracker + run: npx microfox track --file ./__track__/sync-sdk-docs.tracker.ts + - name: Commit and Push Changes + run: |- + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git add . + if git diff --staged --quiet; then + echo "No changes to commit." + else + git commit -m "docs: Sync Micro SDKs documentation" + git push + fi diff --git a/docs/custom.js b/docs/custom.js new file mode 100644 index 000000000..4fa67711f --- /dev/null +++ b/docs/custom.js @@ -0,0 +1,78 @@ +// console.log("--- Custom script loaded ---"); +// document.body.style.border = "2px solid red"; + +// function setPackageNav() { +// console.log("setPackageNav called"); +// const sidebarContent = document.getElementById('sidebar-content'); +// if (!sidebarContent) { +// requestAnimationFrame(setPackageNav); +// return; +// } + +// const currentPath = window.location.pathname; +// console.log("Current path:", currentPath); +// const microSdksPathRegex = /^\/micro-sdks\/([^\/]+)/; +// const match = currentPath.match(microSdksPathRegex); + +// // Restore original sidebar if it exists +// if (window.originalSidebarHTML) { +// sidebarContent.innerHTML = window.originalSidebarHTML; +// } else { +// window.originalSidebarHTML = sidebarContent.innerHTML; +// } + +// if (match && match[1] && match[1] !== 'index') { +// const packageNameSlug = match[1]; +// console.log("Package name slug:", packageNameSlug); + +// const listItems = sidebarContent.querySelectorAll('li[data-title]'); +// let targetListItem = null; + +// listItems.forEach(li => { +// const dataTitle = li.getAttribute('data-title').toLowerCase().replace(/\s/g, '-'); +// if (dataTitle === packageNameSlug) { +// targetListItem = li; +// } +// }); + +// if (targetListItem) { +// console.log("Found target list item:", targetListItem); +// const packageNavContent = targetListItem.querySelector('ul'); +// if (packageNavContent) { +// sidebarContent.innerHTML = ''; +// sidebarContent.appendChild(packageNavContent.cloneNode(true)); +// } +// } else { +// console.log("No target list item found for slug:", packageNameSlug); +// } +// } +// } + +// // --- Observer logic --- +// let currentPath = window.location.pathname; +// function observeDOM() { +// const observer = new MutationObserver(() => { +// if (window.location.pathname !== currentPath) { +// currentPath = window.location.pathname; +// setPackageNav(); +// } +// }); + +// const bodyNode = document.body; +// if (bodyNode) { +// observer.observe(document.body, { childList: true, subtree: true }); +// } +// } + +// if (document.readyState === 'loading') { +// document.addEventListener('DOMContentLoaded', () => { +// setPackageNav(); +// observeDOM(); +// }); +// } else { +// setPackageNav(); +// observeDOM(); +// } + +// // To remove the debug border later: +// // document.body.style.border = "none"; diff --git a/docs/docs.json b/docs/docs.json index 3f2a2fa10..65b262d88 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -15,11 +15,17 @@ "groups": [ { "group": "Get Started", - "pages": ["index", "intro/quickstart"] + "pages": [ + "index", + "intro/quickstart" + ] }, { "group": "About", - "pages": ["intro/roadmap", "intro/reach-out"] + "pages": [ + "intro/roadmap", + "intro/reach-out" + ] } ] }, @@ -29,7 +35,257 @@ "groups": [ { "group": "Overview", - "pages": ["micro-sdks/index"] + "pages": [ + "micro-sdks/index" + ] + }, + { + "group": "AI", + "pages": [ + { + "group": "RAG Upstash", + "pages": [ + "micro-sdks/rag-upstash/index", + "micro-sdks/rag-upstash/getting-started", + "micro-sdks/rag-upstash/foundation", + { + "group": "API Reference", + "pages": [ + { + "group": "Constructors", + "pages": [ + "micro-sdks/rag-upstash/constructors/RagUpstashSdk" + ] + }, + { + "group": "Functions", + "pages": [ + "micro-sdks/rag-upstash/functions/delete", + "micro-sdks/rag-upstash/functions/feedDocsToRAG", + "micro-sdks/rag-upstash/functions/getConfig", + "micro-sdks/rag-upstash/functions/info", + "micro-sdks/rag-upstash/functions/listNamespaces", + "micro-sdks/rag-upstash/functions/query", + "micro-sdks/rag-upstash/functions/queryDocsFromRAG", + "micro-sdks/rag-upstash/functions/upsert" + ] + } + ] + } + ] + } + ] + }, + { + "group": "Communication", + "pages": [ + { + "group": "Slack", + "pages": [ + "micro-sdks/slack/index", + "micro-sdks/slack/getting-started", + "micro-sdks/slack/foundation", + { + "group": "API Reference", + "pages": [ + "micro-sdks/slack/constructors/MicrofoxSlackClient", + "micro-sdks/slack/functions/addUsersToChannel", + "micro-sdks/slack/functions/createChannel", + "micro-sdks/slack/functions/getActiveUsers", + "micro-sdks/slack/functions/getActiveUsersIds", + "micro-sdks/slack/functions/getChannelConversationInfo", + "micro-sdks/slack/functions/getChannelMembers", + "micro-sdks/slack/functions/getChannels", + "micro-sdks/slack/functions/getChannelsIds", + "micro-sdks/slack/functions/getConversationHistory", + "micro-sdks/slack/functions/getFileInfo", + "micro-sdks/slack/functions/getUserByEmail", + "micro-sdks/slack/functions/getUserChannels", + "micro-sdks/slack/functions/getUserInfo", + "micro-sdks/slack/functions/getUsersByEmails", + "micro-sdks/slack/functions/joinChannel", + "micro-sdks/slack/functions/messageChannel", + "micro-sdks/slack/functions/messageChannels", + "micro-sdks/slack/functions/messageUser", + "micro-sdks/slack/functions/messageUsers", + "micro-sdks/slack/functions/reactMessage", + "micro-sdks/slack/functions/removeUserFromChannel", + "micro-sdks/slack/functions/replyMessage", + "micro-sdks/slack/functions/setReminder", + "micro-sdks/slack/functions/uploadFile" + ] + } + ] + }, + { + "group": "AWS SES", + "pages": [ + "micro-sdks/aws-ses/index", + "micro-sdks/aws-ses/getting-started", + "micro-sdks/aws-ses/foundation", + { + "group": "API Reference", + "pages": [ + "micro-sdks/aws-ses/constructors/createSESSdk", + "micro-sdks/aws-ses/functions/sendEmail", + "micro-sdks/aws-ses/functions/sendBulkEmails" + ] + } + ] + } + ] + }, + { + "group": "Database", + "pages": [ + { + "group": "Rag Upstash", + "pages": [ + "micro-sdks/rag-upstash/index", + "micro-sdks/rag-upstash/getting-started", + "micro-sdks/rag-upstash/foundation", + { + "group": "API Reference", + "pages": [ + { + "group": "Constructors", + "pages": [ + "micro-sdks/rag-upstash/constructors/RagUpstashSdk" + ] + }, + { + "group": "Functions", + "pages": [ + "micro-sdks/rag-upstash/functions/delete", + "micro-sdks/rag-upstash/functions/feedDocsToRAG", + "micro-sdks/rag-upstash/functions/getConfig", + "micro-sdks/rag-upstash/functions/info", + "micro-sdks/rag-upstash/functions/listNamespaces", + "micro-sdks/rag-upstash/functions/query", + "micro-sdks/rag-upstash/functions/queryDocsFromRAG", + "micro-sdks/rag-upstash/functions/upsert" + ] + } + ] + } + ] + }, + { + "group": "Db Upstash", + "pages": [ + "micro-sdks/db-upstash/index", + "micro-sdks/db-upstash/getting-started", + "micro-sdks/db-upstash/foundation", + { + "group": "API Reference", + "pages": [ + { + "group": "Constructors", + "pages": [ + "micro-sdks/db-upstash/constructors/CrudHash", + "micro-sdks/db-upstash/constructors/CrudStore", + "micro-sdks/db-upstash/constructors/Paginator" + ] + }, + { + "group": "Functions", + "pages": [ + "micro-sdks/db-upstash/functions/CrudHash.del", + "micro-sdks/db-upstash/functions/CrudHash.get", + "micro-sdks/db-upstash/functions/CrudHash.getFields", + "micro-sdks/db-upstash/functions/CrudHash.list", + "micro-sdks/db-upstash/functions/CrudHash.query", + "micro-sdks/db-upstash/functions/CrudHash.set", + "micro-sdks/db-upstash/functions/CrudHash.update", + "micro-sdks/db-upstash/functions/CrudStore.del", + "micro-sdks/db-upstash/functions/CrudStore.experimental_reIndex", + "micro-sdks/db-upstash/functions/CrudStore.get", + "micro-sdks/db-upstash/functions/CrudStore.list", + "micro-sdks/db-upstash/functions/CrudStore.queryByComposite", + "micro-sdks/db-upstash/functions/CrudStore.queryByField", + "micro-sdks/db-upstash/functions/CrudStore.queryByFieldIn", + "micro-sdks/db-upstash/functions/CrudStore.queryByLex", + "micro-sdks/db-upstash/functions/CrudStore.queryByScore", + "micro-sdks/db-upstash/functions/CrudStore.set", + "micro-sdks/db-upstash/functions/completeIndexing", + "micro-sdks/db-upstash/functions/failIndexing", + "micro-sdks/db-upstash/functions/getCurrentStatus", + "micro-sdks/db-upstash/functions/isStale", + "micro-sdks/db-upstash/functions/pauseIndexing", + "micro-sdks/db-upstash/functions/resetIndexing", + "micro-sdks/db-upstash/functions/resumeIndexing", + "micro-sdks/db-upstash/functions/startNewIndexing", + "micro-sdks/db-upstash/functions/updateIndexingStatus" + ] + } + ] + } + ] + } + ] + }, + { + "group": "Research", + "pages": [ + { + "group": "Brave", + "pages": [ + "micro-sdks/brave/index", + "micro-sdks/brave/getting-started", + "micro-sdks/brave/foundation", + { + "group": "API Reference", + "pages": [ + { + "group": "Constructors", + "pages": [ + "micro-sdks/brave/constructors/createBraveSDK" + ] + }, + { + "group": "Functions", + "pages": [ + "micro-sdks/brave/functions/batchProcess", + "micro-sdks/brave/functions/headers", + "micro-sdks/brave/functions/imageSearch", + "micro-sdks/brave/functions/newsSearch", + "micro-sdks/brave/functions/videoSearch", + "micro-sdks/brave/functions/webSearch" + ] + } + ] + } + ] + }, + { + "group": "Puppeteer SLS", + "pages": [ + "micro-sdks/puppeteer-sls/index", + "micro-sdks/puppeteer-sls/getting-started", + "micro-sdks/puppeteer-sls/foundation", + { + "group": "API Reference", + "pages": [ + { + "group": "Functions", + "pages": [ + "micro-sdks/puppeteer-sls/functions/openPage", + "micro-sdks/puppeteer-sls/functions/takeSnapShot", + "micro-sdks/puppeteer-sls/functions/extractWebpage", + "micro-sdks/puppeteer-sls/functions/extractLinks", + "micro-sdks/puppeteer-sls/functions/extractImages", + "micro-sdks/puppeteer-sls/functions/extractVideos", + "micro-sdks/puppeteer-sls/functions/extractPureText", + "micro-sdks/puppeteer-sls/functions/colorExtract", + "micro-sdks/puppeteer-sls/functions/extractToMarkdown", + "micro-sdks/puppeteer-sls/functions/extractHTML" + ] + } + ] + } + ] + } + ] } ] }, @@ -39,7 +295,9 @@ "groups": [ { "group": "Overview", - "pages": ["code-loop/index"] + "pages": [ + "code-loop/index" + ] } ] }, @@ -49,7 +307,9 @@ "groups": [ { "group": "Overview", - "pages": ["x-framework/index"] + "pages": [ + "x-framework/index" + ] } ] }, @@ -89,7 +349,9 @@ }, { "group": "Examples", - "pages": ["ai-router/examples"] + "pages": [ + "ai-router/examples" + ] }, { "group": "API Reference", @@ -125,9 +387,6 @@ "href": "https://microfox.app" } }, - "styling": { - "eyebrows": "breadcrumbs" - }, "contextual": { "options": [ "copy", diff --git a/docs/micro-sdks/aws-ses/constructors/createSESSdk.mdx b/docs/micro-sdks/aws-ses/constructors/createSESSdk.mdx new file mode 100644 index 000000000..bb6c1c675 --- /dev/null +++ b/docs/micro-sdks/aws-ses/constructors/createSESSdk.mdx @@ -0,0 +1,41 @@ +--- +title: 'createSESSdk' +description: 'Creates an instance of the AWS SES SDK.' +--- + +The `createSESSdk` constructor creates an instance of the AWS SES SDK. This function configures and returns an object with methods for sending emails via the AWS SES service. + +## Usage + +```typescript +import { createSESSdk } from '@microfox/aws-ses'; + +const ses = createSESSdk({ + accessKeyId: process.env.AWS_SES_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SES_SECRET_ACCESS_KEY, + region: 'us-east-1', +}); +``` + +## Arguments + + + + AWS SES configuration object. + + + AWS access key ID. + + + AWS secret access key. + + + AWS region where SES is configured (e.g., "us-east-1"). + + + + + +## Response + +This constructor returns an `SESSDK` object containing the `sendEmail` and `sendBulkEmails` functions. diff --git a/docs/micro-sdks/aws-ses/foundation.mdx b/docs/micro-sdks/aws-ses/foundation.mdx new file mode 100644 index 000000000..06f589dee --- /dev/null +++ b/docs/micro-sdks/aws-ses/foundation.mdx @@ -0,0 +1,89 @@ +--- +title: 'Foundation' +description: 'A deep dive into the core concepts, features, and best practices for using Amazon SES with the @microfox/aws-ses SDK.' +--- + +## Core Concepts + +The `@microfox/aws-ses` SDK simplifies the use of the official AWS SDK v3 by providing a higher-level client. It focuses on the most common email-sending tasks, making them accessible through a clean and intuitive API. + +### How It Works +When you call a method like `ses.sendEmail()`, the SDK performs the following steps under the hood: +1. Constructs the appropriate `Command` object required by the `@aws-sdk/client-sesv2` (e.g., `SendEmailCommand`). +2. Populates the command's input with the parameters you provided, translating the simplified format into the structure AWS expects. For emails with attachments, it correctly builds the raw MIME message. +3. Initializes the official `SESv2Client` using the credentials from your environment. +4. Sends the command to the Amazon SES API endpoint. +5. Returns a simplified `Promise` that resolves on success or rejects with a descriptive error on failure. + +## Sending Templated Emails + +For sending personalized emails to a large number of recipients, using SES Templates is highly efficient. It allows you to define a template in SES and then just send the dynamic data for each recipient, reducing the amount of data sent in each API call. + +**Step 1: Create a Template in AWS** +First, you must create an email template in the Amazon SES console. A template consists of a subject line and an HTML/text body with placeholder variables. + +Example Template: +- **Template Name:** `WelcomeUser` +- **Subject:** `Welcome to Our Service, {{name}}!` +- **HTML Body:** `

Hi {{name}},

Thanks for signing up. We're excited to have you!

` + +**Step 2: Send with the SDK** +Use the `sendBulkTemplatedEmail` method to send an email using your template. + +```typescript +import { createSESSdk } from '@microfox/aws-ses'; + +const ses = createSESSdk(); + +async function sendWelcomeEmails() { + await ses.sendBulkTemplatedEmail({ + from: 'sender@your-verified-domain.com', + template: 'WelcomeUser', // The name of your SES template + destinations: [ + { + to: ['alice@example.com'], + // The keys here must match the placeholders in your template + templateData: { name: 'Alice' }, + }, + { + to: ['bob@example.com'], + templateData: { name: 'Bob' }, + }, + ], + // Optional: Default data if a destination is missing a value + defaultTemplateData: { name: 'Valued Customer' }, + }); +} +``` + +## Handling Attachments + +The `sendEmail` method accepts an `attachments` array to send files. Each attachment object in the array must have a `filename` and `content`. + +- **`filename`**: The name of the file as it should appear in the email client (e.g., `'invoice.pdf'`). +- **`content`**: The raw content of the file, typically as a `Buffer`. + +You can easily read a file from your disk and send it: +```typescript +import * as fs from 'fs'; + +const fileContent = fs.readFileSync('./path/to/your/file.pdf'); + +await ses.sendEmail({ + // ... to, from, subject ... + attachments: [ + { + filename: 'document.pdf', + content: fileContent, + }, + ], +}); +``` + +## Best Practices + +- **IAM Security:** Always follow the principle of least privilege. Create a dedicated IAM user for your application and attach a policy that *only* grants the SES permissions it needs (e.g., `ses:SendEmail`). Do not use your root AWS account credentials. +- **Sender Verification:** To improve deliverability and comply with anti-spam laws, verify your entire domain with SES, not just individual email addresses. This allows you to send from any address on that domain. +- **Monitor Your Sending Quotas:** AWS SES imposes sending quotas (e.g., emails per day, emails per second). You can monitor your usage in the SES console. If you need higher limits, you can request an increase from AWS support. +- **Handle Bounces and Complaints:** A high bounce or complaint rate can damage your sender reputation and lead to your account being suspended. Configure SES to send notifications about bounces and complaints to an SNS topic so you can process them and remove invalid email addresses from your mailing lists. +- **Use a "Reply-To" Address:** For transactional emails, it's good practice to set a `replyTo` address. This ensures that if a user replies to your email, it goes to a monitored inbox instead of a "no-reply" address. diff --git a/docs/micro-sdks/aws-ses/functions/sendBulkEmails.mdx b/docs/micro-sdks/aws-ses/functions/sendBulkEmails.mdx new file mode 100644 index 000000000..5afa1bfb1 --- /dev/null +++ b/docs/micro-sdks/aws-ses/functions/sendBulkEmails.mdx @@ -0,0 +1,55 @@ +--- +title: 'sendBulkEmails' +description: 'Sends multiple emails in parallel using the provided parameters.' +--- + +The `sendBulkEmails` function sends multiple emails to a list of recipients using the AWS SES service. Emails are sent in parallel for improved performance. + +## Usage + +```typescript +const responses = await ses.sendBulkEmails({ + sender: 'sender@example.com', + recipients: [ + 'recipient1@example.com', + 'recipient2@example.com', + 'recipient3@example.com', + ], + subject: 'Hello from SES!', + bodyText: 'This is a plain text email body.', +}); + +console.log(responses); +``` + +## Arguments + + + + Parameters for sending bulk emails. + + + Email address of the sender (must be verified in AWS SES). + + + List of recipient email addresses. + + + Display name for the sender (e.g., "John Doe" <john@example.com>). + + + Subject line of the email. + + + Plain text version of the email body. + + + HTML version of the email body. + + + + + +## Response + +This function returns a `Promise` which resolves to an array of `SendEmailResponse` objects, one for each recipient. diff --git a/docs/micro-sdks/aws-ses/functions/sendEmail.mdx b/docs/micro-sdks/aws-ses/functions/sendEmail.mdx new file mode 100644 index 000000000..641462608 --- /dev/null +++ b/docs/micro-sdks/aws-ses/functions/sendEmail.mdx @@ -0,0 +1,58 @@ +--- +title: 'sendEmail' +description: 'Sends a single email using the provided parameters.' +--- + +The `sendEmail` function sends a single email to a specified recipient using the AWS SES service. + +## Usage + +```typescript +// Sending a plain text email +await ses.sendEmail({ + sender: 'sender@example.com', + recipient: 'recipient@example.com', + subject: 'Hello from SES!', + bodyText: 'This is a plain text email body.', +}); + +// Sending an HTML email +await ses.sendEmail({ + sender: 'sender@example.com', + recipient: 'recipient@example.com', + subject: 'Hello from SES!', + bodyHtml: 'This is an HTML email body.', +}); +``` + +## Arguments + + + + Parameters for sending the email. + + + Email address of the sender (must be verified in AWS SES). + + + Email address of the recipient. + + + Display name for the sender (e.g., "John Doe" <john@example.com>). + + + Subject line of the email. + + + Plain text version of the email body. + + + HTML version of the email body. + + + + + +## Response + +This function returns a `Promise` which resolves to the `SendEmailResponse` object from the AWS SES API. diff --git a/docs/micro-sdks/aws-ses/getting-started.mdx b/docs/micro-sdks/aws-ses/getting-started.mdx new file mode 100644 index 000000000..bb283c377 --- /dev/null +++ b/docs/micro-sdks/aws-ses/getting-started.mdx @@ -0,0 +1,121 @@ +--- +title: 'Getting Started' +description: 'A step-by-step guide to configuring AWS and using the SDK to send your first emails with Amazon SES.' +--- + +## Prerequisites + +Before you begin, you must have the following set up in your AWS account: + +- **Node.js:** Version 16.x or later. +- **An AWS Account:** With billing enabled. +- **An IAM User:** You need an IAM user with programmatic access (an Access Key ID and Secret Access Key). +- **SES Permissions:** The IAM user must have a policy attached that grants permissions to use SES. A minimal policy would be `AmazonSESFullAccess`. For better security, create a custom policy with only the permissions you need, like `ses:SendEmail` and `ses:SendRawEmail`. +- **A Verified Sender Identity:** You cannot send emails *from* an address until you've proven you own it. In the Amazon SES console, go to **"Verified identities"** and verify either a specific email address or an entire domain. + +## 1. Configure Your AWS Credentials + +The SDK, like the official AWS SDK, will automatically look for credentials in your environment. The recommended way to provide them is by setting environment variables: + +```bash +export AWS_ACCESS_KEY_ID="YOUR_ACCESS_KEY_ID" +export AWS_SECRET_ACCESS_KEY="YOUR_SECRET_ACCESS_KEY" +export AWS_REGION="us-east-1" # Or your preferred AWS region +``` + +## 2. Installation + +Add the `@microfox/aws-ses` package to your project: + +```bash npm +npm install @microfox/aws-ses +``` +```bash yarn +yarn add @microfox/aws-ses +``` +```bash pnpm +pnpm add @microfox/aws-ses +``` + + +## 3. Your First Email Script + +Let's create a script that demonstrates how to send both a simple text email and a more complex HTML email with an attachment. + +Create a new file named `mailer.ts`. + +```typescript +import { createSESSdk } from '@microfox/aws-ses'; +import * as fs from 'fs'; +import * as path from 'path'; + +// 1. Initialize the SDK client. It automatically picks up credentials +// from your environment variables. +const ses = createSESSdk(); + +// IMPORTANT: Replace with your verified sender email address. +const SENDER_EMAIL = 'sender@your-verified-domain.com'; + +async function main() { + try { + // --- Example 1: Sending a simple text email --- + console.log('Sending a simple text email...'); + await ses.sendEmail({ + to: ['recipient1@example.com'], + from: SENDER_EMAIL, + subject: 'A Simple Test Email', + text: 'Hello from the @microfox/aws-ses SDK!', + }); + console.log('Simple email sent successfully.'); + + // --- Example 2: Sending an HTML email with an attachment --- + console.log('\nSending an HTML email with an attachment...'); + + // Create a dummy attachment file + const attachmentPath = path.join(__dirname, 'invoice.txt'); + fs.writeFileSync(attachmentPath, 'This is a test invoice.'); + + // Read the file into a buffer + const attachmentContent = fs.readFileSync(attachmentPath); + + await ses.sendEmail({ + to: ['recipient2@example.com'], + cc: ['cc-recipient@example.com'], + from: SENDER_EMAIL, + subject: 'Your Invoice is Attached', + html: ` +

Invoice Details

+

Hello,

+

Please find your invoice attached to this email.

+

Thank you!

+ `, + attachments: [ + { + filename: 'invoice.txt', + content: attachmentContent, + }, + ], + }); + console.log('HTML email with attachment sent successfully.'); + + } catch (error) { + console.error('An error occurred while sending email:', error); + } +} + +main(); +``` + +### Running the Script +Make sure your environment variables are set, then run the script with `ts-node`: + +```bash +ts-node mailer.ts +``` + +Check the recipient inboxes! You should see the two emails you sent. + +## Next Steps +You've now successfully configured your environment and sent emails using the SDK. +- **Explore the API:** Check out the **API Reference** for `sendEmail`, `sendBulkEmails`, and more. +- **Learn Advanced Concepts:** Read the **[Foundation](/micro-sdks/aws-ses/foundation)** guide to learn about sending templated emails and best practices for deliverability. diff --git a/docs/micro-sdks/aws-ses/index.mdx b/docs/micro-sdks/aws-ses/index.mdx new file mode 100644 index 000000000..c96ee1549 --- /dev/null +++ b/docs/micro-sdks/aws-ses/index.mdx @@ -0,0 +1,53 @@ +--- +title: 'Overview' +description: 'A modern, developer-friendly TypeScript SDK for Amazon Simple Email Service (SES).' +--- + +## A Simplified SDK for Amazon SES + +`@microfox/aws-ses` is a lightweight TypeScript SDK that provides a simplified, high-level interface for sending emails with Amazon Simple Email Service (SES). It's designed to make common email-sending tasks easier and more intuitive than using the official AWS SDK directly. + +### What Problem Does It Solve? +While the official AWS SDK for JavaScript (v3) is powerful and comprehensive, its modular design can be verbose for common tasks. Sending a simple email requires importing multiple command and client objects and understanding a complex configuration structure. + +This SDK streamlines the process by: +- **Providing a Simple, Unified Client:** A single `createSESSdk` function gives you a client ready to send emails. +- **Simplifying the API:** Methods like `sendEmail` and `sendBulkEmails` accept a single, intuitive options object. +- **Handling Boilerplate:** The SDK manages the creation and execution of the underlying SES commands for you. +- **Adding Type Safety:** Provides a clear, type-safe interface for all parameters and options. + +### Who Is This For? +This package is ideal for any developer using Node.js who needs to integrate email functionality into their application via Amazon SES. Common use cases include: +- Sending transactional emails (e.g., password resets, order confirmations). +- Distributing marketing or notification emails to a list of users. +- Building applications that need to send emails with complex HTML content and attachments. + +### High-Level Architecture +The SDK acts as a developer-friendly wrapper around the official `@aws-sdk/client-sesv2`. It simplifies the interface without sacrificing the power and reliability of the underlying AWS infrastructure. + +```mermaid +graph TD; + subgraph Your Application + A[Your Code] --> B{createSESSdk}; + B --> C{SES SDK Client} + end + + subgraph @microfox/aws-ses + C -- uses --> D["@aws-sdk/client-sesv2"]; + end + + subgraph AWS + D -- "Makes API Calls" --> E[(Amazon SES)]; + end + + style C fill:#ccf,stroke:#333,stroke-width:2px +``` + +### Key Features +- **Simplified Email Sending:** Send emails with just a few lines of code. +- **Bulk Email Support:** Easily send the same email to multiple recipients. +- **Attachment Handling:** A simple interface for adding attachments to your emails. +- **HTML & Text Content:** Supports both HTML and plain text email bodies. +- **Fully Typed:** Written entirely in TypeScript for a great developer experience. + +This documentation provides a comprehensive **[Getting Started](/micro-sdks/aws-ses/getting-started)** guide, an in-depth look at the package's **[Foundation](/micro-sdks/aws-ses/foundation)**, and a detailed API Reference to help you get the most out of `@microfox/aws-ses`. diff --git a/docs/micro-sdks/brave/constructors/createBraveSDK.mdx b/docs/micro-sdks/brave/constructors/createBraveSDK.mdx new file mode 100644 index 000000000..262e2298f --- /dev/null +++ b/docs/micro-sdks/brave/constructors/createBraveSDK.mdx @@ -0,0 +1,76 @@ +--- +title: 'createBraveSDK' +description: 'API reference for createBraveSDK' +--- + +## Function: `createBraveSDK` + +Creates a new instance of the Brave SDK. + +**Purpose:** +This function creates and returns a new instance of the `BraveSDK` class, which can be used to interact with the Brave Search API. + +**Parameters:** + +- `options` (BraveSDKOptions, optional): An object containing configuration options. + - `apiKey` (string, optional): Your Brave Search API key. If not provided, the constructor will try to use the `BRAVE_API_KEY` environment variable. + - `headers` (RequestHeaders, optional): Custom headers for web search requests. + - `localSearchHeaders` (LocalSearchHeaders, optional): Custom headers for local search requests. + - `enableRedisTracking` (boolean, optional): Enable Redis tracking for requests. + - `redisTrackingId` (string, optional): Custom ID for Redis tracking. + +**Return Value:** + +- `BraveSDK`: A new instance of the `BraveSDK` class. + +**Examples:** + +```typescript +// Example 1: Basic usage with API key +const braveSDK = createBraveSDK({ apiKey: process.env.BRAVE_API_KEY }); + +// Example 2: Using environment variable +const braveSDK = createBraveSDK(); // Assumes BRAVE_API_KEY is set in the environment + +// Example 4: With Redis tracking enabled +const braveSDK = createBraveSDK({ + apiKey: process.env.BRAVE_API_KEY, + enableRedisTracking: true, + redisTrackingId: 'custom-tracking-id', +}); +``` + +## Header Configuration + +### Manual Header Configuration + +You can manually configure headers by passing them directly to the SDK: + +```typescript +const brave = createBraveSDK({ + apiKey: 'your-api-key', + headers: { + 'X-Loc-State': 'NY', + 'X-Loc-Country': 'US', + }, +}); +``` + +## Available Header Options + +### Standard Headers + +- `User-Agent`: Browser/device identification +- `Api-Version`: API version (e.g., '2024-01-01') +- `Cache-Control`: Cache control directives + +### Location Headers + +- `X-Loc-Lat`: Latitude coordinate +- `X-Loc-Long`: Longitude coordinate +- `X-Loc-Timezone`: Timezone (e.g., 'America/New_York') +- `X-Loc-City`: City name +- `X-Loc-State`: State code +- `X-Loc-State-Name`: Full state name +- `X-Loc-Country`: Country code +- `X-Loc-Postal-Code`: Postal/ZIP code diff --git a/docs/micro-sdks/brave/foundation.mdx b/docs/micro-sdks/brave/foundation.mdx new file mode 100644 index 000000000..bf170d42e --- /dev/null +++ b/docs/micro-sdks/brave/foundation.mdx @@ -0,0 +1,135 @@ +--- +title: 'Foundation' +description: 'A deep dive into the advanced features of the @microfox/brave SDK, including batch processing and the middleware system.' +--- + +## Understanding Search Verticals + +The Brave Search API is not a single endpoint, but a collection of specialized "verticals," each tailored to a specific type of content. The `@microfox/brave` SDK provides a dedicated method for each of these verticals, ensuring that the parameters and response shapes are correctly typed for the content you're working with. + +- **`webSearch`**: The standard search engine for web pages. +- **`imageSearch`**: For finding images. +- **`videoSearch`**: For finding videos. +- **`newsSearch`**: For retrieving recent news articles. +- **`localPoiSearch` / `localDescriptionsSearch`**: For finding local businesses and points of interest. +- **`summarizerSearch`**: For getting an AI-powered summary of a URL. +- **`suggestSearch`**: For autocompletion and search suggestions. +- **`spellcheckSearch`**: For checking spelling. + +Each of these methods takes a specific set of parameters relevant to its vertical (e.g., `imageSearch` has parameters for aspect ratio and color that `webSearch` does not). Refer to the API reference for the specific options available for each. + +## Advanced Batch Processing + +The `batchProcess` method is designed to help you manage API usage and respect rate limits when you need to perform multiple searches in a short period. + +By default, it adds a 1-second (1000ms) delay between each request in the batch to prevent you from hitting the API too quickly. You can customize this delay and also monitor the progress. + +### Example: Custom Delay and Progress Tracking + +```typescript +import { createBraveSDK } from '@microfox/brave'; + +const brave = createBraveSDK(); + +async function runMonitoredBatch() { + const queries = ['AI', 'Web Dev', 'Serverless', 'TypeScript', 'Rust']; + + await brave.batchProcess( + // Map our array of strings to the request format + queries.map(q => ({ type: 'web', params: { q } })), + { + delay: 500, // Set a 500ms delay between requests + onProgress: (completed, total) => { + console.log(`Progress: ${completed} of ${total} searches complete.`); + }, + } + ); + + console.log('Batch processing finished.'); +} +``` + +## The Middleware System + +For truly advanced use cases, the SDK includes a middleware system that allows you to intercept and modify requests before they are sent. A middleware is a function that receives the request details and can return a modified version of them. + +This is incredibly powerful for implementing cross-cutting concerns like: +- Custom logging +- Caching +- Request retries +- Dynamically modifying request headers or parameters + +### Example: Creating a Logging Middleware + +Let's create a simple middleware that logs the URL of every outgoing request to the console. + +```typescript +import { createBraveSDK, MiddlewareFunction } from '@microfox/brave'; + +// Define our logging middleware +const loggingMiddleware: MiddlewareFunction = async (request) => { + console.log(`[Logger] Sending request to: ${request.url.toString()}`); + + // Middleware must always return the request object + return request; +}; + +const brave = createBraveSDK(); + +// Register the middleware with the client instance +brave.use(loggingMiddleware); + +// Now, every search call will be logged +await brave.webSearch({ q: 'hello world' }); +// Console output: [Logger] Sending request to: https://api.search.brave.com/res/v1/web/search?q=hello+world +``` + +### Example: A Caching Middleware (Conceptual) + +Here is a more advanced conceptual example of how you could implement a simple in-memory cache to avoid re-running the same search. + +```typescript +import { createBraveSDK, MiddlewareFunction } from '@microfox/brave'; + +const cache = new Map(); + +const cachingMiddleware: MiddlewareFunction = async (request) => { + const cacheKey = request.url.toString(); + + // If the result is in our cache, throw a special error + // to stop the request from being sent. + if (cache.has(cacheKey)) { + const cachedResult = cache.get(cacheKey); + // We can throw an error with a specific structure + // to pass the cached data back to our main logic. + throw { type: 'CACHE_HIT', data: cachedResult }; + } + + return request; +}; + +const brave = createBraveSDK(); +brave.use(cachingMiddleware); + +async function performCachedSearch(query: string) { + try { + const results = await brave.webSearch({ q: query }); + // If the request was successful, store the result in the cache + cache.set(`https://api.search.brave.com/res/v1/web/search?q=${query.replace(' ', '+')}`, results); + console.log('[API] Fetched from API'); + return results; + } catch (error) { + // If we get our special cache hit error, return the cached data + if (error.type === 'CACHE_HIT') { + console.log('[CACHE] Fetched from cache'); + return error.data; + } + // Otherwise, re-throw the error + throw error; + } +} + +await performCachedSearch('typescript'); // Fetches from API +await performCachedSearch('typescript'); // Fetches from cache +``` +This demonstrates the flexibility of the middleware system for implementing complex, custom logic. diff --git a/docs/micro-sdks/brave/functions/batchProcess.mdx b/docs/micro-sdks/brave/functions/batchProcess.mdx new file mode 100644 index 000000000..b9584274a --- /dev/null +++ b/docs/micro-sdks/brave/functions/batchProcess.mdx @@ -0,0 +1,195 @@ +--- +title: 'batchProcess' +description: 'API reference for batchProcess' +--- + +# Batch Processing + +The batch processing functionality allows you to process multiple requests sequentially with configurable delays and progress tracking. This is particularly useful for handling rate limits and managing multiple API calls efficiently. + +## Overview + +The `batchProcess` method is a core functionality that supports various types of batch operations: + +- Web searches +- Image searches +- Video searches +- News searches +- Suggest searches +- Spellcheck searches +- Summarizer searches +- Local POI searches +- Local descriptions searches + +## Usage + +### Basic Usage + +```typescript +const results = await braveSDK.batchProcess([ + { type: 'web', params: { q: 'TypeScript' } }, + { type: 'image', params: { q: 'cats' } }, + { type: 'news', params: { q: 'technology' } }, +]); +``` + +### With Options + +```typescript +const results = await braveSDK.batchProcess( + [ + { type: 'web', params: { q: 'React' } }, + { type: 'web', params: { q: 'Vue' } }, + { type: 'web', params: { q: 'Angular' } }, + ], + { + delay: 2000, // 2 second delay between requests + onProgress: (completed, total) => { + console.log(`Completed ${completed} of ${total} requests`); + }, + }, +); +``` + +## Convenience Methods + +The SDK provides convenience methods for common batch operations: + +### Batch Web Search + +```typescript +const results = await braveSDK.batchWebSearch([ + { q: 'TypeScript' }, + { q: 'JavaScript' }, + { q: 'Python' }, +]); +``` + +### Batch Image Search + +```typescript +const results = await braveSDK.batchImageSearch([ + { q: 'cats', count: 5 }, + { q: 'dogs', count: 5 }, +]); +``` + +### Batch Video Search + +```typescript +const results = await braveSDK.batchVideoSearch([ + { q: 'tutorials', count: 5 }, + { q: 'reviews', count: 5 }, +]); +``` + +### Batch News Search + +```typescript +const results = await braveSDK.batchNewsSearch([ + { q: 'technology', freshness: 'pd' }, + { q: 'sports', country: 'US' }, +]); +``` + +## Options + +The `BatchRequestOptions` interface provides the following configuration options: + +- `delay` (number, optional): Delay between requests in milliseconds. Default: 1000ms +- `onProgress` (function, optional): Callback function that receives progress updates + - Parameters: + - `completed`: Number of completed requests + - `total`: Total number of requests + +## Error Handling + +The batch process handles errors gracefully: + +- If a request fails, it will be included in the results with an error property +- The process continues with the next request +- All results are returned in the same order as the input requests + +```typescript +const results = await braveSDK.batchProcess(requests); +results.forEach((result, index) => { + if ('error' in result) { + console.error(`Request ${index + 1} failed:`, result.error); + } else { + // Process successful result + } +}); +``` + +## Best Practices + +1. **Rate Limiting** + + - Use appropriate delays between requests to respect API rate limits + - Free Plan: 1 request per second + - Pro Plan: 20 requests per second + +2. **Progress Tracking** + + - Implement the `onProgress` callback to monitor batch processing + - Use this for updating UI or logging progress + +3. **Error Handling** + + - Always check for errors in the results + - Implement appropriate error recovery strategies + +4. **Resource Management** + + - Consider the total number of requests in a batch + - Implement pagination or chunking for large batches + - Cache results when appropriate + +5. **Type Safety** + - Use TypeScript's type system to ensure correct request types and parameters + - Leverage the provided convenience methods for type-safe batch operations + + +## Examples + +### Complex Batch Processing + +```typescript +const batchRequests = [ + { type: 'web', params: { q: 'TypeScript', count: 10 } }, + { type: 'image', params: { q: 'cats', safesearch: 'moderate' } }, + { type: 'news', params: { q: 'technology', freshness: 'pw' } }, + { type: 'video', params: { q: 'tutorials', count: 5 } }, +]; + +const results = await braveSDK.batchProcess(batchRequests, { + delay: 1500, + onProgress: (completed, total) => { + const percentage = (completed / total) * 100; + console.log(`Progress: ${percentage.toFixed(1)}%`); + }, +}); + +// Process results +results.forEach((result, index) => { + if ('error' in result) { + console.error(`Request ${index + 1} failed:`, result.error); + return; + } + + switch (batchRequests[index].type) { + case 'web': + console.log('Web results:', result.search?.results); + break; + case 'image': + console.log('Image results:', result.mixed?.main); + break; + case 'news': + console.log('News results:', result.results); + break; + case 'video': + console.log('Video results:', result.results); + break; + } +}); +``` diff --git a/docs/micro-sdks/brave/functions/headers.mdx b/docs/micro-sdks/brave/functions/headers.mdx new file mode 100644 index 000000000..855fd7a41 --- /dev/null +++ b/docs/micro-sdks/brave/functions/headers.mdx @@ -0,0 +1,97 @@ +--- +title: 'headers' +description: 'API reference for headers' +--- + +# Brave SDK Headers Documentation + +## Overview + +The Brave SDK allows you to configure custom headers for API requests. Headers can be set either manually or using helper functions. + +## Header Configuration + +### Manual Header Configuration + +You can manually configure headers by passing them directly to the SDK: + +```typescript +const brave = createBraveSDK({ + apiKey: 'your-api-key', + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/', + 'X-Loc-Lat': 40.7128, + 'X-Loc-Long': -74.006, + 'X-Loc-Timezone': 'America/New_York', + 'X-Loc-City': 'New York', + 'X-Loc-State': 'NY', + 'X-Loc-State-Name': 'New York', + 'X-Loc-Country': 'US', + 'X-Loc-Postal-Code': '10001', + 'Api-Version': '2024-01-01', + 'Cache-Control': 'no-cache', + }, +}); +``` + +### Using Helper Functions + +For convenience, you can use the provided helper functions to create headers: + +```typescript +const brave = createBraveSDK({ + apiKey: 'your-api-key', + headers: createHeaders({ + apiKey: 'your-api-key', + platform: 'macos', + location: { + latitude: 40.7128, + longitude: -74.006, + timezone: 'America/New_York', + city: 'New York', + state: 'NY', + stateName: 'New York', + country: 'US', + postalCode: '10001', + }, + apiVersion: '2024-01-01', + cacheControl: 'no-cache', + }), +}); +``` + +## Local Search Headers + +For local search requests, you can configure separate headers: + +```typescript +const brave = createBraveSDK({ + apiKey: 'your-api-key', + // ... other configuration + localSearchHeaders: createLocalSearchHeaders({ + apiKey: 'your-api-key', + platform: 'ios', + apiVersion: '2024-01-01', + }), +}); +``` + +## Available Header Options + +### Standard Headers + +- `User-Agent`: Browser/device identification +- `Api-Version`: API version (e.g., '2024-01-01') +- `Cache-Control`: Cache control directives + +### Location Headers + +- `X-Loc-Lat`: Latitude coordinate +- `X-Loc-Long`: Longitude coordinate +- `X-Loc-Timezone`: Timezone (e.g., 'America/New_York') +- `X-Loc-City`: City name +- `X-Loc-State`: State code +- `X-Loc-State-Name`: Full state name +- `X-Loc-Country`: Country code +- `X-Loc-Postal-Code`: Postal/ZIP code diff --git a/docs/micro-sdks/brave/functions/imageSearch.mdx b/docs/micro-sdks/brave/functions/imageSearch.mdx new file mode 100644 index 000000000..33000967f --- /dev/null +++ b/docs/micro-sdks/brave/functions/imageSearch.mdx @@ -0,0 +1,208 @@ +--- +title: 'imageSearch' +description: 'API reference for imageSearch' +--- + +# Brave Search API: Image Search + +The `imageSearch` function enables image search capabilities through the Brave Search API, allowing you to find and retrieve images based on search queries. + +## Function Signature + +```typescript +imageSearch(params: ImageSearchParams): Promise +``` + +## Parameters + +The function accepts a single parameter object with the following properties: + +### Required Parameters + +- `q` (string): The search query + - Maximum length: 400 characters + - Maximum words: 50 + +### Optional Parameters + +- `country` (string): Two-letter country code (e.g., 'US', 'GB') +- `search_lang` (string): Language for search results +- `ui_lang` (string): Language for user interface elements +- `count` (number): Number of results to return (maximum: 20) +- `offset` (number): Pagination offset (maximum: 9) +- `safesearch` (enum): Content filtering level + - Values: 'off' | 'moderate' | 'strict' +- `freshness` (enum | string): Result freshness filter + - Preset values: 'pd' (past day), 'pw' (past week), 'pm' (past month), 'py' (past year) + - Custom string values also supported +- `text_decorations` (boolean): Include text decoration markers in results +- `spellcheck` (boolean): Enable automatic spell checking +- `result_filter` (string): Comma-separated list of result types to include +- `goggles` (string[]): Array of goggle definitions +- `units` (enum): Measurement units + - Values: 'metric' | 'imperial' + +## Return Value + +Returns a Promise that resolves to an `ImageSearchApiResponse` object containing the search results. + +### Response Type Structure + +```typescript +interface ImageSearchApiResponse { + query: { + original: string; + show_strict_warning: boolean; + is_navigational: boolean; + is_news_breaking: boolean; + spellcheck_off: boolean; + country: string; + bad_results: boolean; + should_fallback: boolean; + postal_code: string; + city: string; + header_country: string; + more_results_available: boolean; + state: string; + }; + mixed: { + type: string; + main: ImageResult[]; + }; + type: string; +} + +interface ImageResult { + title: string; + url: string; + source: string; + source_favicon: string; + source_domain: string; + thumbnail: { + src: string; + original: string; + width: number; + height: number; + }; + size: { + width: number; + height: number; + }; + age: string; + type: string; + description: string; + family_friendly: boolean; +} +``` + +## Example Usage + +```typescript +// Basic image search +const results = await braveSDK.imageSearch({ + q: 'cats', + count: 10, + safesearch: 'moderate', +}); + +// Advanced image search with multiple parameters +const results = await braveSDK.imageSearch({ + q: 'sunset beach', + country: 'US', + search_lang: 'en', + count: 15, + freshness: 'pw', + safesearch: 'strict', + units: 'metric', +}); + +// Processing the results +const processResults = async () => { + try { + const response = await braveSDK.imageSearch({ q: 'landscape' }); + + // Access the image results + const images = response.mixed.main; + + // Process each image + images.forEach(image => { + console.log(`Title: ${image.title}`); + console.log(`Source: ${image.source}`); + console.log(`Image URL: ${image.url}`); + console.log(`Thumbnail: ${image.thumbnail.src}`); + console.log(`Dimensions: ${image.size.width}x${image.size.height}`); + console.log(`Description: ${image.description}`); + }); + + // Check if more results are available + if (response.query.more_results_available) { + console.log('More results are available'); + } + + // Get query metadata + console.log(`Original query: ${response.query.original}`); + console.log(`Country: ${response.query.country}`); + } catch (error) { + console.error('Error performing image search:', error); + } +}; + +// Batch image search example +const batchImageSearchExample = async () => { + try { + // Define multiple image search queries + const searchQueries = [ + { q: 'cats', count: 5, safesearch: 'moderate' }, + { q: 'dogs', count: 5, safesearch: 'moderate' }, + { q: 'birds', count: 5, safesearch: 'moderate' }, + ]; + + // Process batch search with progress tracking + const results = await braveSDK.batchImageSearch(searchQueries); + + // Process results for each query + results.forEach((response, index) => { + console.log(`\nResults for query "${searchQueries[index].q}":`); + const images = response.mixed.main; + + images.forEach(image => { + console.log(`- ${image.title} (${image.source})`); + }); + }); + } catch (error) { + console.error('Error performing batch image search:', error); + } +}; +``` + +## Notes + +- The function is similar to `webSearch` but specifically optimized for image results +- Results are paginated using the `count` and `offset` parameters +- Safe search is enabled by default with 'moderate' level +- All parameters except `q` are optional + +## Best Practices + +1. **Error Handling** + + - Always wrap API calls in try-catch blocks + - Handle rate limiting and network errors appropriately + +2. **Result Processing** + + - Check for `more_results_available` before attempting to fetch more results + - Use the `thumbnail` property for preview images + - Consider the `family_friendly` flag when filtering results + - Use `size` information to filter by image dimensions + +3. **Performance** + + - Use appropriate `count` values to avoid over-fetching + - Implement pagination using `offset` for large result sets + - Cache results when appropriate + +4. **Image Quality** + - Use `size` information to ensure images meet your requirements + - Consider using `freshness` parameter for time-sensitive content + - Use `safesearch` to filter inappropriate content diff --git a/docs/micro-sdks/brave/functions/newsSearch.mdx b/docs/micro-sdks/brave/functions/newsSearch.mdx new file mode 100644 index 000000000..1f5b8f312 --- /dev/null +++ b/docs/micro-sdks/brave/functions/newsSearch.mdx @@ -0,0 +1,163 @@ +--- +title: 'newsSearch' +description: 'API reference for newsSearch' +--- + +## Function: `newsSearch` + +Performs a news search using the Brave Search API. + +**Purpose:** +This function allows you to search for news articles using the Brave Search API. + +**Parameters:** + +- `params` (NewsSearchParams, required): An object containing the search parameters. This is the same as `WebSearchParams` but without the `summary` and `extra_snippets` properties. + - `q` (string, required): The search query (max 400 characters, 50 words). + - `country` (string, optional): 2-letter country code. Defaults to 'US'. + - `search_lang` (string, optional): Search language. Defaults to 'en'. + - `ui_lang` (string, optional): UI language. Defaults to 'en-US'. + - `count` (number, optional): Number of results (1-20). Defaults to 20. + - `offset` (number, optional): Offset for pagination (0-9). Defaults to 0. + - `safesearch` (enum, optional): Safe search level ('off', 'moderate', 'strict'). Defaults to 'moderate'. + - `freshness` (enum | string, optional): Freshness of results. Can be: + - 'pd': Last 24 hours + - 'pw': Last 7 days + - 'pm': Last 31 days + - 'py': Last 365 days + - Custom date range in format 'YYYY-MM-DDtoYYYY-MM-DD' + - `text_decorations` (boolean, optional): Include decoration markers. Defaults to true. + - `spellcheck` (boolean, optional): Enable spellchecking. Defaults to true. + - `result_filter` (string, optional): Comma-separated list of result types. + - `goggles` (array\, optional): Goggle definitions for custom re-ranking. + - `units` (enum, optional): Units for measurements ('metric', 'imperial'). + +**Return Value:** + +- `Promise`: A promise that resolves to the news search response containing: + - `type`: Always 'news' + - `query`: Query information including original, altered, and cleaned versions + - `results`: Array of news results, each containing: + - `type`: Always 'news_result' + - `url`: Article URL + - `title`: Article title + - `description`: Article description + - `age`: Article age + - `page_age`: Page age + - `page_fetched`: When the page was fetched + - `breaking`: Whether it's breaking news + - `thumbnail`: Thumbnail information + - `meta_url`: Meta URL information + - `extra_snippets`: Additional article snippets + +**Examples:** + +```typescript +// Basic news search +const results = await braveSDK.newsSearch({ q: 'current events' }); + +// Advanced news search with multiple parameters +const results = await braveSDK.newsSearch({ + q: 'technology news', + country: 'US', + search_lang: 'en', + count: 15, + freshness: 'pw', + safesearch: 'moderate', + text_decorations: true, + spellcheck: true, +}); + +// Processing the results +if (results.type === 'news') { + // Access query information + console.log('Original query:', results.query.original); + console.log('Cleaned query:', results.query.cleaned); + + // Process each news result + results.results.forEach((article, index) => { + console.log(`\nArticle ${index + 1}:`); + console.log('Title:', article.title); + console.log('URL:', article.url); + console.log('Description:', article.description); + console.log('Age:', article.age); + console.log('Breaking News:', article.breaking); + + // Access thumbnail if available + if (article.thumbnail) { + console.log('Thumbnail URL:', article.thumbnail.src); + console.log( + 'Thumbnail dimensions:', + article.thumbnail.width, + 'x', + article.thumbnail.height, + ); + } + + // Access meta URL information if available + if (article.meta_url) { + console.log('Source:', article.meta_url.source); + console.log('Source Domain:', article.meta_url.domain); + } + + // Access extra snippets if available + if (article.extra_snippets && article.extra_snippets.length > 0) { + console.log('Additional snippets:', article.extra_snippets); + } + }); +} + +// Example: Filtering breaking news +const breakingNews = results.results.filter(article => article.breaking); + +// Example: Sorting by age +const sortedByAge = results.results.sort((a, b) => { + const ageA = new Date(a.age).getTime(); + const ageB = new Date(b.age).getTime(); + return ageA - ageB; +}); + +// Example: Grouping by source +const articlesBySource = results.results.reduce( + (acc, article) => { + const source = article.meta_url?.source || 'Unknown'; + if (!acc[source]) { + acc[source] = []; + } + acc[source].push(article); + return acc; + }, + {} as Record, +); + +// Example: Batch news search +const batchResults = await braveSDK.batchNewsSearch( + [ + { q: 'technology news', freshness: 'pd' }, + { q: 'sports news', country: 'US', count: 10 }, + { q: 'business news', freshness: 'pw' }, + ], + { + delay: 1000, // 1 second delay between requests + onProgress: (completed, total) => { + console.log(`Completed ${completed} of ${total} requests`); + }, + }, +); + +// Process batch results +batchResults.forEach((result, index) => { + if ('error' in result) { + console.error(`Error in request ${index + 1}:`, result.error); + return; + } + + console.log(`\nResults for search ${index + 1}:`); + result.results.forEach((article, articleIndex) => { + console.log(`\nArticle ${articleIndex + 1}:`); + console.log('Title:', article.title); + console.log('Source:', article.meta_url?.source); + console.log('Age:', article.age); + }); +}); +``` diff --git a/docs/micro-sdks/brave/functions/videoSearch.mdx b/docs/micro-sdks/brave/functions/videoSearch.mdx new file mode 100644 index 000000000..3bf4a0ec1 --- /dev/null +++ b/docs/micro-sdks/brave/functions/videoSearch.mdx @@ -0,0 +1,185 @@ +--- +title: 'videoSearch' +description: 'API reference for videoSearch' +--- + +## Function: `videoSearch` + +Performs a video search using the Brave Search API. + +**Purpose:** +This function allows you to search for videos using the Brave Search API, retrieving video results with detailed metadata and thumbnails. + +**Parameters:** + +- `params` (VideoSearchParams, required): An object containing the search parameters. This is the same as `WebSearchParams` but without the `summary` and `extra_snippets` properties. + - `q` (string, required): The search query (max 400 characters, 50 words). + - `country` (string, optional): 2-letter country code. Defaults to 'US'. + - `search_lang` (string, optional): Search language. Defaults to 'en'. + - `ui_lang` (string, optional): UI language. Defaults to 'en-US'. + - `count` (number, optional): Number of results (1-20). Defaults to 20. + - `offset` (number, optional): Offset for pagination (0-9). Defaults to 0. + - `safesearch` (enum, optional): Safe search level ('off', 'moderate', 'strict'). Defaults to 'moderate'. + - `freshness` (enum | string, optional): Freshness of results. Can be: + - 'pd': Last 24 hours + - 'pw': Last 7 days + - 'pm': Last 31 days + - 'py': Last 365 days + - Custom date range in format 'YYYY-MM-DDtoYYYY-MM-DD' + - `text_decorations` (boolean, optional): Include decoration markers. Defaults to true. + - `spellcheck` (boolean, optional): Enable spellchecking. Defaults to true. + - `result_filter` (string, optional): Comma-separated list of result types. + - `goggles` (array\, optional): Goggle definitions for custom re-ranking. + - `units` (enum, optional): Units for measurements ('metric', 'imperial'). + +**Return Value:** + +- `Promise`: A promise that resolves to the video search response containing: + - `type`: Always 'videos' + - `query`: Query information including: + - `original`: Original search query + - `altered`: Modified version of the query + - `cleaned`: Cleaned version of the query + - `spellcheck_off`: Whether spellchecking was disabled + - `show_strict_warning`: Warning message for strict safe search + - `results`: Array of video results, each containing: + - `type`: Always 'video_result' + - `url`: Video URL + - `title`: Video title + - `description`: Video description + - `age`: Video age + - `page_age`: Page age + - `page_fetched`: When the page was fetched + - `thumbnail`: Thumbnail information + - `video`: Video metadata including: + - `duration`: Video duration + - `views`: Number of views + - `creator`: Video creator + - `publisher`: Video publisher + - `tags`: Array of video tags + - `author`: Author profile information + - `requires_subscription`: Whether the video requires a subscription + - `meta_url`: Meta URL information + +**Examples:** + +```typescript +// Basic video search +const results = await braveSDK.videoSearch({ q: 'cats playing' }); + +// Advanced video search with multiple parameters +const results = await braveSDK.videoSearch({ + q: 'cute puppies', + country: 'US', + search_lang: 'en', + count: 15, + freshness: 'pw', + safesearch: 'moderate', + text_decorations: true, + spellcheck: true, +}); + +// Processing the results +if (results.type === 'videos') { + // Access query information + console.log('Original query:', results.query.original); + console.log('Cleaned query:', results.query.cleaned); + + // Process each video result + results.results.forEach((video, index) => { + console.log(`\nVideo ${index + 1}:`); + console.log('Title:', video.title); + console.log('URL:', video.url); + console.log('Description:', video.description); + console.log('Age:', video.age); + + // Access video metadata + if (video.video) { + console.log('Duration:', video.video.duration); + console.log('Views:', video.video.views); + console.log('Creator:', video.video.creator); + console.log('Publisher:', video.video.publisher); + if (video.video.tags) { + console.log('Tags:', video.video.tags.join(', ')); + } + } + + // Access thumbnail if available + if (video.thumbnail) { + console.log('Thumbnail URL:', video.thumbnail.src); + console.log( + 'Thumbnail dimensions:', + video.thumbnail.width, + 'x', + video.thumbnail.height, + ); + } + + // Access meta URL information if available + if (video.meta_url) { + console.log('Source:', video.meta_url.source); + console.log('Source Domain:', video.meta_url.domain); + } + }); +} + +// Example: Filtering videos by duration +const shortVideos = results.results.filter(video => { + const duration = video.video?.duration; + return duration && parseInt(duration) < 60; // Videos under 1 minute +}); + +// Example: Sorting by views +const sortedByViews = results.results.sort((a, b) => { + const viewsA = parseInt(a.video?.views || '0'); + const viewsB = parseInt(b.video?.views || '0'); + return viewsB - viewsA; +}); + +// Example: Grouping by creator +const videosByCreator = results.results.reduce( + (acc, video) => { + const creator = video.video?.creator || 'Unknown'; + if (!acc[creator]) { + acc[creator] = []; + } + acc[creator].push(video); + return acc; + }, + {} as Record, +); + +// Example: Batch video search +const batchResults = await braveSDK.batchVideoSearch( + [ + { q: 'cute puppies', count: 5 }, + { q: 'funny cats', count: 5 }, + { q: 'baby animals', count: 5 }, + ], + { + delay: 1000, // 1 second delay between requests + onProgress: (completed, total) => { + console.log(`Completed ${completed} of ${total} searches`); + }, + }, +); + +// Process batch results +batchResults.forEach((result, index) => { + if ('error' in result) { + console.error(`Search ${index + 1} failed:`, result.error); + return; + } + + console.log(`\nResults for search ${index + 1}:`); + result.results.forEach((video, videoIndex) => { + console.log(`\nVideo ${videoIndex + 1}:`); + console.log('Title:', video.title); + console.log('URL:', video.url); + if (video.video) { + console.log('Duration:', video.video.duration); + console.log('Views:', video.video.views); + } + }); +}); +``` diff --git a/docs/micro-sdks/brave/functions/webSearch.mdx b/docs/micro-sdks/brave/functions/webSearch.mdx new file mode 100644 index 000000000..651365d66 --- /dev/null +++ b/docs/micro-sdks/brave/functions/webSearch.mdx @@ -0,0 +1,133 @@ +--- +title: 'webSearch' +description: 'API reference for webSearch' +--- + +## Function: `webSearch` + +Performs a web search using the Brave Search API. + +**Purpose:** +This function allows you to perform web searches and retrieve results from the Brave Search API, including web pages, news, videos, discussions, FAQs, and more. + +**Parameters:** + +- `params` (WebSearchParams, required): An object containing the search parameters. + - `q` (string, required): The search query (max 400 characters, 50 words). + - `country` (string, optional): 2-letter country code. Defaults to 'US'. + - `search_lang` (string, optional): Search language. Defaults to 'en'. + - `ui_lang` (string, optional): UI language. Defaults to 'en-US'. + - `count` (number, optional): Number of results (1-20). Defaults to 20. + - `offset` (number, optional): Offset for pagination (0-9). Defaults to 0. + - `safesearch` (enum, optional): Safe search level ('off', 'moderate', 'strict'). Defaults to 'moderate'. + - `freshness` (enum | string, optional): Freshness of results. Can be: + - 'pd': Last 24 hours + - 'pw': Last 7 days + - 'pm': Last 31 days + - 'py': Last 365 days + - Custom date range in format 'YYYY-MM-DDtoYYYY-MM-DD' + - `text_decorations` (boolean, optional): Include decoration markers. Defaults to true. + - `spellcheck` (boolean, optional): Enable spellchecking. Defaults to true. + - `result_filter` (string, optional): Comma-separated list of result types. Available values: discussions, faq, infobox, news, query, summarizer, videos, web, locations. + - `goggles` (array\, optional): Goggle definitions for custom re-ranking. + - `units` (enum, optional): Units for measurements ('metric', 'imperial'). + - `extra_snippets` (boolean, optional): Include up to 5 additional, alternative excerpts. + - `summary` (boolean, optional): Enable summary key generation for use with summarizer. + +**Return Value:** + +- `Promise`: A promise that resolves to the web search response containing: + - `type`: Always 'search' + - `discussions`: Optional discussions results + - `faq`: Optional FAQ results + - `infobox`: Optional infobox results + - `locations`: Optional location results + - `mixed`: Optional mixed results + - `news`: Optional news results + - `query`: Query information + - `rich_data`: Optional rich data + - `web`: Optional web search results + - `videos`: Optional video results + +**Examples:** + +```typescript +// Basic web search +const results = await braveSDK.webSearch({ q: 'TypeScript' }); + +// Advanced web search with multiple parameters +const results = await braveSDK.webSearch({ + q: 'JavaScript frameworks', + country: 'US', + search_lang: 'en', + count: 15, + freshness: 'pw', + safesearch: 'moderate', + text_decorations: true, + spellcheck: true, + result_filter: 'web,news,videos', + units: 'metric', + extra_snippets: true, + summary: true, +}); + +// Processing the results +if (results.type === 'search') { + // Access search results + if (results.web) { + results.web.results.forEach((result, index) => { + console.log(`Result ${index + 1}:`, result.title); + console.log('URL:', result.url); + console.log('Description:', result.description); + }); + } + + // Access news results if available + if (results.news) { + console.log('News Results:', results.news.results); + } + + // Access video results if available + if (results.videos) { + console.log('Video Results:', results.videos.results); + } + + // Access FAQ results if available + if (results.faq) { + console.log('FAQ Results:', results.faq.results); + } +} + +// Batch web search example +const batchResults = await braveSDK.batchProcess([ + { type: 'web', params: { q: 'TypeScript' } }, + { type: 'web', params: { q: 'JavaScript', count: 10 } }, + { type: 'web', params: { q: 'Python', freshness: 'pw' } }, +]); + +// Process batch results +batchResults.forEach((result, index) => { + if (result.type === 'search' && result.web) { + console.log(`Batch ${index + 1} Results:`); + result.web.results.forEach((searchResult, resultIndex) => { + console.log(` Result ${resultIndex + 1}:`, searchResult.title); + console.log(' URL:', searchResult.url); + }); + } +}); + +// Batch search with progress tracking +const batchResultsWithProgress = await braveSDK.batchProcess( + [ + { type: 'web', params: { q: 'React' } }, + { type: 'web', params: { q: 'Vue' } }, + { type: 'web', params: { q: 'Angular' } }, + ], + { + delay: 2000, // 2 second delay between requests + onProgress: (completed, total) => { + console.log(`Completed ${completed} of ${total} searches`); + }, + }, +); +``` diff --git a/docs/micro-sdks/brave/getting-started.mdx b/docs/micro-sdks/brave/getting-started.mdx new file mode 100644 index 000000000..ffd1cf32b --- /dev/null +++ b/docs/micro-sdks/brave/getting-started.mdx @@ -0,0 +1,144 @@ +--- +title: 'Getting Started' +description: 'A step-by-step guide to installing and using the @microfox/brave SDK to perform your first web searches.' +--- + +## Prerequisites + +Before you begin, you will need: +- **Node.js:** Version 16.x or later. +- **A Brave Search API Key:** You can get a free API key from the [Brave Developer portal](https://brave.com/search/api/). + +## 1. Installation + +Add the `@microfox/brave` package to your project using your preferred package manager. + + +```bash npm +npm install @microfox/brave +``` +```bash yarn +yarn add @microfox/brave +``` +```bash pnpm +pnpm add @microfox/brave +``` + + +## 2. Setting Up the SDK Client + +Create a new file for your script (e.g., `search.ts`). It's best practice to provide your API key via an environment variable. + +```typescript +import { createBraveSDK } from '@microfox/brave'; + +// The client will automatically use the BRAVE_API_KEY environment variable. +// You can also pass it directly: createBraveSDK({ apiKey: 'YOUR_KEY' }) +const brave = createBraveSDK(); +``` + +## 3. Performing Searches + +The SDK provides a separate method for each search vertical. Let's try a few. + +### Example 1: Web Search +Let's find the top 3 search results for "Latest AI developments". + +```typescript +async function performWebSearch() { + console.log('--- Performing Web Search ---'); + try { + const results = await brave.webSearch({ + q: 'Latest AI developments', + count: 3, + }); + // The results object is fully typed and contains a wealth of information + results.web?.results?.forEach((result) => { + console.log(`- ${result.title} (${result.url})`); + }); + } catch (error) { + console.error('Web search failed:', error); + } +} +``` + +### Example 2: Image Search +Now let's find a picture of a cat. + +```typescript +async function performImageSearch() { + console.log('\n--- Performing Image Search ---'); + try { + const results = await brave.imageSearch({ + q: 'cute cat', + safesearch: 'strict', + }); + // The first result's source URL + const firstImageUrl = results.results?.[0]?.properties?.url; + console.log('Found an image:', firstImageUrl); + } catch (error) { + console.error('Image search failed:', error); + } +} +``` + +## 4. Using Batch Processing + +The `batchProcess` method is a powerful feature for running multiple queries of different types in a controlled sequence. This is useful for respecting API rate limits. + +Let's search for both news about "Tesla" and videos about "Web Development" in one operation. + +```typescript +async function performBatchSearch() { + console.log('\n--- Performing Batch Search ---'); + try { + const [newsResults, videoResults] = await brave.batchProcess([ + { + type: 'news', + params: { q: 'Tesla' }, + }, + { + type: 'video', + params: { q: 'Web Development' }, + }, + ]); + + console.log(`Found ${newsResults.results?.length} news articles about Tesla.`); + console.log(`Found a video about Web Development: ${videoResults.results?.[0]?.url}`); + + } catch (error) { + console.error('Batch search failed:', error); + } +} +``` + +### Running the Script +Create a main function to run all examples, set your environment variable, and run the script with `ts-node`. + +`search.ts` (full file) +```typescript +import { createBraveSDK } from '@microfox/brave'; + +const brave = createBraveSDK(); + +// ... (paste the 3 async functions from above here) + +async function main() { + await performWebSearch(); + await performImageSearch(); + await performBatchSearch(); +} + +main(); +``` + +```bash +export BRAVE_API_KEY="YOUR_API_KEY" +ts-node search.ts +``` +You'll see the output from all three search operations printed to your console. + +## Next Steps +You have now successfully performed single and batch searches with the `@microfox/brave` SDK. +- **Explore Search Verticals:** Check out the API Reference for all the other search types and their specific parameters. +- **Learn Advanced Concepts:** Read the **[Foundation](/micro-sdks/brave/foundation)** guide to learn about the middleware system for advanced request handling. diff --git a/docs/micro-sdks/brave/index.mdx b/docs/micro-sdks/brave/index.mdx new file mode 100644 index 000000000..354e398f9 --- /dev/null +++ b/docs/micro-sdks/brave/index.mdx @@ -0,0 +1,53 @@ +--- +title: 'Overview' +description: 'A powerful and type-safe TypeScript SDK for the Brave Search API, providing access to web, image, video, news, and local search.' +--- + +## A Unified SDK for the Brave Search API + +`@microfox/brave` is a comprehensive TypeScript SDK for interacting with the Brave Search API. It provides a simple, unified interface for all of Brave's search verticals, including web, images, videos, news, and more, all with the benefits of strong type safety. + +### What Problem Does It Solve? +The Brave Search API offers a wide range of search capabilities across different endpoints. Interacting with them directly requires managing an HTTP client, handling authentication for each request, and manually typing the various query parameters and response structures for each search vertical. + +This SDK simplifies this entire process by: +- **Providing a Single Client:** A single, easy-to-configure client provides access to all search types. +- **Offering a Type-Safe Interface:** All methods, parameters, and responses are fully typed, enabling autocompletion and preventing common bugs. +- **Including Built-in Batch Processing:** A powerful `batchProcess` method makes it easy to run multiple queries sequentially while respecting API rate limits. +- **Featuring a Middleware System:** Allows for custom logic, such as logging or caching, to be injected into the request pipeline. + +### Who Is This For? +This package is ideal for developers who want to integrate real-world search capabilities into their applications, such as: +- AI agents that need to browse the web to find up-to-date information. +- Applications that need to search for images, videos, or news articles. +- Developer tools that require search suggestions or spell-checking functionality. +- Services that need to find local points of interest or business information. + +### High-Level Architecture +The SDK provides a central client that routes requests to the appropriate Brave Search API endpoint based on the method you call. + +```mermaid +graph TD; + subgraph Your Application + A[Your Code] --> B{BraveSDK Client}; + end + + B --> C[/web/search]; + B --> D[/images/search]; + B --> E[/videos/search]; + B --> F[...] + + subgraph Brave Search API + C & D & E & F + end + + style B fill:#ccf,stroke:#333,stroke-width:2px +``` + +### Key Features +- **Complete API Coverage:** Supports all Brave Search API verticals: Web, Image, Video, News, Suggest, Spellcheck, Summarizer, and Local Search. +- **Robust Batch Processing:** Execute multiple queries of different types in a single batch operation with configurable delays. +- **Extensible Middleware:** Intercept and modify requests using a flexible middleware system. +- **TypeScript First:** Written in TypeScript with Zod schemas for robust validation and type safety. + +This documentation provides a comprehensive **[Getting Started](/micro-sdks/brave/getting-started)** guide and an in-depth look at the package's **[Foundation](/micro-sdks/brave/foundation)** to help you master the SDK's advanced features. diff --git a/docs/micro-sdks/db-upstash/constructors/CrudHash.mdx b/docs/micro-sdks/db-upstash/constructors/CrudHash.mdx new file mode 100644 index 000000000..bd5367f4e --- /dev/null +++ b/docs/micro-sdks/db-upstash/constructors/CrudHash.mdx @@ -0,0 +1,41 @@ +--- +title: 'CrudHash' +description: 'API reference for CrudHash' +--- + +# CrudHash Constructor + +Initializes a new instance of the `CrudHash` class, a generic CRUD helper for Upstash Redis that uses a Redis hash for each item. + +This class is ideal for managing individual objects where atomic updates on specific fields are common, such as user profiles, session data, or feature flags. + +## Signature + +```ts +constructor(redis: Redis, keyPrefix: string) +``` + +## Parameters + +- **`redis`** (`Redis`): An instance of the Redis client from `@upstash/redis`. This client is used to execute all commands against your Redis database. +- **`keyPrefix`** (`string`): A string prefix for all keys created by this instance (e.g., `user`). This helps to namespace your data within Redis. + +## Throws + +- `ConfigurationError`: Thrown if the `redis` client or `keyPrefix` is not provided. + +## Example + +```ts +import { Redis } from '@upstash/redis'; +import { CrudHash } from '@pkg/db-upstash'; + +const redis = new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, +}); + +const userStore = new CrudHash(redis, 'users'); + +// Now you can use userStore to manage user data +``` diff --git a/docs/micro-sdks/db-upstash/constructors/CrudStore.mdx b/docs/micro-sdks/db-upstash/constructors/CrudStore.mdx new file mode 100644 index 000000000..a4530c0b5 --- /dev/null +++ b/docs/micro-sdks/db-upstash/constructors/CrudStore.mdx @@ -0,0 +1,54 @@ +--- +title: 'CrudStore' +description: 'API reference for CrudStore' +--- + +# CrudStore Constructor + +Initializes a new instance of the `CrudStore` class, a generic CRUD helper for Upstash Redis that is optimized for querying and listing large numbers of items efficiently using multiple types of indexes. + +This class is ideal for managing collections of objects where you need to efficiently query, sort, and paginate using different criteria, such as a list of blog posts that can be sorted by `createdAt` (a sorted index) or filtered by `status: 'published'` (a matched index). + +## Signature + +```ts +constructor( + redis: Redis, + keyPrefix: string, + config: { + sortFields: StoreField[]; + matchFields?: StoreField[]; + } +) +``` + +## Parameters + +- **`redis`** (`Redis`): An instance of the Redis client from `@upstash/redis`. +- **`keyPrefix`** (`string`): A string prefix for all keys created by this instance (e.g., `posts`). +- **`config`** (`object`): An object to configure the store's indexes. + - **`sortFields`** (`StoreField[]`): An array of field names to use for numeric, sorted indexes. These are ideal for sorting and range queries (e.g., by timestamp, price, or score). The value of these fields must be a number or a string that can be parsed into a number. + - **`matchFields`** (`StoreField[]`, optional): An array of field names to use for exact-match indexes. These are ideal for filtering results by a specific value (e.g., by status, category, or tag). + +## Throws + +- `ConfigurationError`: Thrown if the `redis` client, `keyPrefix`, or `config.sortFields` are not provided. + +## Example + +```ts +import { Redis } from '@upstash/redis'; +import { CrudStore } from '@pkg/db-upstash'; + +const redis = new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, +}); + +const postStore = new CrudStore(redis, 'posts', { + sortFields: ['createdAt', 'viewCount'], + matchFields: ['status', 'authorId'], +}); + +// Now you can use postStore to manage and query blog posts +``` diff --git a/docs/micro-sdks/db-upstash/constructors/Paginator.mdx b/docs/micro-sdks/db-upstash/constructors/Paginator.mdx new file mode 100644 index 000000000..5e71d3a54 --- /dev/null +++ b/docs/micro-sdks/db-upstash/constructors/Paginator.mdx @@ -0,0 +1,444 @@ +--- +title: 'Paginator' +description: 'API reference for Paginator' +--- + +# Paginator Constructor + +The `Paginator` class provides a robust way to manage and track the state of long-running pagination or indexing tasks using Upstash Redis. It is designed to be generic and can handle any progress state structure. + +## Instantiation + +To create a new `Paginator` instance, you need to provide an Upstash Redis client instance and a unique ID for the pagination task. + +```typescript +new Paginator(redis: Redis, id: string) +``` + +### Parameters + +- `redis` (`Redis`): An active Upstash Redis client instance from `@upstash/redis`. The `Paginator` uses this client to communicate with your Redis database. +- `id` (`string`): A unique identifier for the pagination task. This ID is used to create a unique key in Redis, ensuring that different pagination tasks do not interfere with each other. +- `T` (`generic`): A TypeScript generic type that defines the shape of the `progress` object. This allows you to customize the state you want to track. It defaults to `Record`. + +### Example + +```typescript +import { Redis } from '@upstash/redis'; +import { Paginator } from '@microfox/db-upstash'; + +// Define the shape of your pagination progress +interface MyScraperProgress { + page: number; + lastId: string | null; + itemsProcessed: number; +} + +// Initialize the Redis client +const redis = new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, +}); + +// Create a new Paginator instance for a specific scraping task +const myScraperPaginator = new Paginator( + redis, + 'my-scraper-task-123' +); +``` + +```` + +### Method Documentation +Here are the markdown files for each of the public methods, which I'll place in `packages/db-upstash/docs/functions/`: + +
+startNewIndexing.md + +```markdown:packages/db-upstash/docs/functions/startNewIndexing.md +# startNewIndexing + +Initializes a new indexing or pagination task. This method sets the initial state in Redis, marks the status as 'running', and records the start time. + +## Signature +```typescript +startNewIndexing(initialProgress: T): Promise> +```` + +### Parameters + +- `initialProgress` (`T`): An object representing the starting progress of the task. The shape of this object must match the generic type `T` provided to the `Paginator` constructor. + +### Returns + +- (`Promise>`): A promise that resolves to the newly created pagination status object. + +### Example + +```typescript +const initialProgress = { + page: 1, + lastId: null, + itemsProcessed: 0, +}; + +const status = await myScraperPaginator.startNewIndexing(initialProgress); +console.log(status); +// { +// status: 'running', +// progress: { page: 1, lastId: null, itemsProcessed: 0 }, +// startedAt: 1678886400000, +// lastUpdatedAt: 1678886400000 +// } +``` + +```` + +
+ +
+updateIndexingStatus.md + +```markdown:packages/db-upstash/docs/functions/updateIndexingStatus.md +# updateIndexingStatus + +Updates the progress of an ongoing indexing task. This is the most common method to call during a scraping loop to save the latest state. + +## Signature +```typescript +updateIndexingStatus(newProgress: Partial): Promise> +```` + +### Parameters + +- `newProgress` (`Partial`): An object containing the progress fields to update. It will be merged with the existing progress state. + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object. + +### Example + +```typescript +const status = await myScraperPaginator.updateIndexingStatus({ + page: 2, + itemsProcessed: 100, +}); + +console.log('Progress updated:', status.progress); +// Progress updated: { page: 2, lastId: 'some-last-id', itemsProcessed: 100 } +``` + +```` + +
+ +
+completeIndexing.md + +```markdown:packages/db-upstash/docs/functions/completeIndexing.md +# completeIndexing + +Marks an indexing task as 'completed'. This is the final state for a successful task. + +## Signature +```typescript +completeIndexing(): Promise> +```` + +### Returns + +- (`Promise>`): A promise that resolves to the final pagination status object with a `status` of `'completed'`. + +### Example + +```typescript +await myScraperPaginator.completeIndexing(); +console.log('Scraping job completed!'); +``` + +```` + +
+ +
+pauseIndexing.md + +```markdown:packages/db-upstash/docs/functions/pauseIndexing.md +# pauseIndexing + +Pauses a running indexing task. This sets the status to `'paused'`. + +## Signature +```typescript +pauseIndexing(): Promise> +```` + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object with a `status` of `'paused'`. + +### Example + +```typescript +await myScraperPaginator.pauseIndexing(); +console.log('Scraper has been paused.'); +``` + +```` + +
+ +
+resumeIndexing.md + +```markdown:packages/db-upstash/docs/functions/resumeIndexing.md +# resumeIndexing + +Resumes a paused indexing task by setting its status back to `'running'`. + +## Signature +```typescript +resumeIndexing(): Promise> +```` + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object with a `status` of `'running'`. + +### Example + +```typescript +const status = await myScraperPaginator.resumeIndexing(); +console.log(`Resumed at page: ${status.progress.page}`); +``` + +```` + +
+ +
+failIndexing.md + +```markdown:packages/db-upstash/docs/functions/failIndexing.md +# failIndexing + +Marks an indexing task as 'failed' and stores an error message. This is useful for formally recording a failure state. + +## Signature +```typescript +failIndexing(error: Record | string): Promise> +```` + +### Parameters + +- `error` (`Record | string`): The error to be stored. Can be a simple string or a more detailed object. + +### Returns + +- (`Promise>`): A promise that resolves to the final pagination status object with a `status` of `'failed'`. + +### Example + +```typescript +try { + // some failing operation + throw new Error('API rate limit exceeded'); +} catch (e: any) { + await myScraperPaginator.failIndexing({ message: e.message, code: 500 }); + console.log('Scraper job failed.'); +} +``` + +```` + +
+ +
+getCurrentStatus.md + +```markdown:packages/db-upstash/docs/functions/getCurrentStatus.md +# getCurrentStatus + +Retrieves the current status and progress of the indexing task from Redis. + +## Signature +```typescript +getCurrentStatus(): Promise | null> +```` + +### Returns + +- (`Promise | null>`): A promise that resolves to the current pagination status object, or `null` if the task has not been started. + +### Example + +```typescript +const status = await myScraperPaginator.getCurrentStatus(); + +if (status) { + console.log(`Current status: ${status.status}`); +} else { + console.log('No job found. Starting a new one.'); +} +``` + +```` + +
+ +
+isStale.md + +```markdown:packages/db-upstash/docs/functions/isStale.md +# isStale + +Checks if an indexing task is stale by comparing its `lastUpdatedAt` timestamp with a given timeout. This is useful for detecting and restarting stuck or dead worker processes. + +## Signature +```typescript +isStale(timeoutSeconds: number): Promise +```` + +### Parameters + +- `timeoutSeconds` (`number`): The timeout in seconds. If the task has not been updated within this period, it is considered stale. + +### Returns + +- (`Promise`): A promise that resolves to `true` if the task is stale, otherwise `false`. + +### Example + +```typescript +// Check if the scraper has been silent for more than 5 minutes +const isStuck = await myScraperPaginator.isStale(300); + +if (isStuck) { + console.log('Warning: Scraper job appears to be stale!'); +} +``` + +```` + +
+ +
+resetIndexing.md + +```markdown:packages/db-upstash/docs/functions/resetIndexing.md +# resetIndexing + +Deletes all data associated with the indexing task from Redis. This is a destructive operation and should be used with care. + +## Signature +```typescript +resetIndexing(): Promise +```` + +### Returns + +- (`Promise`): A promise that resolves when the data has been deleted. + +### Example + +```typescript +// Clean up the task state after it's no longer needed +await myScraperPaginator.resetIndexing(); +console.log('Paginator state has been reset.'); +``` + +```` + +
+ +### Package README +Finally, I'll create a `README.md` for the `@microfox/db-upstash` package to provide an overview and a clear example. + +```markdown:packages/db-upstash/README.md +# @microfox/db-upstash + +A TypeScript SDK for interacting with Upstash Redis, providing helpful utilities and classes. + +## Features + +- **Paginator**: A generic class for managing the state of long-running pagination or indexing tasks. + +## Installation + +```bash +npm install @microfox/db-upstash @upstash/redis +```` + +## Usage + +### Paginator + +The `Paginator` class helps you track the progress of background jobs, scrapers, or any task that involves pagination. It can store, update, pause, and resume stateful progress information in Redis. + +#### Example + +```typescript +import { Redis } from '@upstash/redis'; +import { Paginator } from '@microfox/db-upstash'; + +// Define the structure for your task's progress +interface ScraperProgress { + page: number; + itemsScraped: number; +} + +const redis = new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL!, + token: process.env.UPSTASH_REDIS_REST_TOKEN!, +}); + +const scraperId = 'user-profiles-scraper'; +const paginator = new Paginator(redis, scraperId); + +async function runScraper() { + // Start a new scraping task or resume a previous one + let status = await paginator.getCurrentStatus(); + if (!status) { + console.log('Starting new scraper job...'); + status = await paginator.startNewIndexing({ page: 1, itemsScraped: 0 }); + } else if (status.status === 'paused') { + console.log('Resuming scraper job...'); + status = await paginator.resumeIndexing(); + } + + if (!status || status.status === 'completed' || status.status === 'failed') { + console.log('Job is already completed or has failed.'); + return; + } + + console.log(`Scraping page ${status.progress.page}...`); + + // Simulate scraping + await new Promise((resolve) => setTimeout(resolve, 2000)); + const newItems = 10; + + // Update the progress + const updatedStatus = await paginator.updateIndexingStatus({ + page: status.progress.page + 1, + itemsScraped: status.progress.itemsScraped + newItems, + }); + + console.log('Progress updated:', updatedStatus.progress); + + // Complete the job after a few pages + if (updatedStatus.progress.page > 5) { + await paginator.completeIndexing(); + console.log('Scraping job completed!'); + } +} + +runScraper().catch(console.error); +``` + +For detailed API documentation, please refer to the files in the `docs/` directory. + +``` + +These files should provide comprehensive documentation for your new `Paginator` class. Let me know if you'd like any adjustments +``` diff --git a/docs/micro-sdks/db-upstash/foundation.mdx b/docs/micro-sdks/db-upstash/foundation.mdx new file mode 100644 index 000000000..c61d3c0ba --- /dev/null +++ b/docs/micro-sdks/db-upstash/foundation.mdx @@ -0,0 +1,134 @@ +--- +title: 'Foundation' +description: 'A deep dive into the data structures, indexing strategies, and best practices for using @microfox/db-upstash effectively.' +--- + +## Core Concepts: Data Modeling in Redis + +`@microfox/db-upstash` abstracts away the low-level Redis commands, but understanding the underlying data structures it uses is key to leveraging the library effectively. + +### `CrudHash`: Simple Object Storage + +The `CrudHash` class uses a single Redis **Hash** for each item. + +- **Key:** `{keyPrefix}:{id}` (e.g., `sessions:sess_123`) +- **Value:** A Redis Hash where each field in your object is a field in the hash. + +```mermaid +graph TD; + A["Key: 'sessions:sess_123'"] --> B{Redis Hash}; + subgraph B + C["id: 'sess_123'"]; + D["userId: 'user_abc'"]; + E["expiresAt: '167...'"]; + end +``` + +**Pros:** +- Very efficient for `get`, `set`, and `del` operations (O(1)). +- Supports atomic partial updates of individual fields using `HSET`. + +**Cons:** +- **Inefficient for querying.** The `list` and `query` methods use the `SCAN` command to iterate over *all* keys matching the prefix. This is a slow, O(N) operation and should not be used on large datasets in production. + +### `CrudStore`: Indexed Object Storage + +`CrudStore` is more sophisticated. It uses a combination of Redis data structures to enable efficient querying. + +1. **Item Data (Hash):** Each item is still stored in its own Hash, just like with `CrudHash`. + - Key: `{keyPrefix}:item:{id}` (e.g., `products:item:prod_001`) + +2. **Sortable Indexes (Sorted Sets):** For each field in your `sortFields` config, a **Sorted Set** is created. + - Key: `{keyPrefix}:zset:{sortField}` (e.g., `products:zset:price`) + - Member: The item's ID (e.g., `prod_001`) + - Score: The numeric value of the `sortField` (e.g., `1200`) + - This allows for incredibly fast range queries (e.g., "get all products with price between 1000 and 1500") and sorting. + +3. **Matchable Indexes (Sets):** For each field in your `matchFields` config, a **Set** is created for *each unique value*. + - Key: `{keyPrefix}:match:{field}:{value}` (e.g., `products:match:category:electronics`) + - Members: The IDs of all items that have this exact field-value pair. + - This allows for fast lookups of all items with a specific property. + +```mermaid +graph LR; + subgraph Item Data + A["'products:item:prod_001' (Hash)"] + end + subgraph Sort Index + B["'products:zset:price' (Sorted Set)"] + B -- "Score: 1200" --> C("Member: 'prod_001'"); + end + subgraph Match Index + D["'products:match:category:electronics' (Set)"] + D --> E("Member: 'prod_001'"); + end + A -- Indexed by --> B; + A -- Indexed by --> D; +``` + +When you call `set`, `update`, or `del` on a `CrudStore`, it uses a Redis Pipeline to update the item's Hash and all of its associated indexes in a single, atomic batch operation. + +## Indexing Strategy and Best Practices + +- **Choose `sortFields` wisely:** Only use fields that are numbers (like timestamps, prices, or counts) and that you will need to sort by or query within a range. +- **Use `matchFields` for low-cardinality data:** "Cardinality" refers to the number of unique values. Fields like `category`, `status`, or `type` are excellent candidates for `matchFields`. Avoid using them on high-cardinality data like `email` or `username`, as this would create a huge number of individual Set keys. +- **Keep your `id` unique and simple:** The `id` is the core of your item and is used as the member in all indexes. +- **Plan your `keyPrefix`:** Use a descriptive prefix to create a clean namespace for your data within Redis (e.g., `users`, `products`, `articles`). + +## Error Handling + +The library provides custom error types that extend the base `Error` class, allowing you to handle specific failure scenarios gracefully. + +- **`DbUpstashError`**: The base error class for this library. +- **`ConfigurationError`**: Thrown if there is an issue with the setup (e.g., missing Redis client). +- **`ItemNotFoundError`**: Thrown by `update` or `del` if the target item doesn't exist. +- **`InvalidFieldError`**: Thrown by `update` if you attempt to change an item's `id` or provide a non-numeric value for a `sortField`. + +```typescript +import { ItemNotFoundError } from '@microfox/db-upstash'; + +try { + await userStore.update('non-existent-user', { name: 'New Name' }); +} catch (error) { + if (error instanceof ItemNotFoundError) { + // Handle the specific case of the user not being found + console.error('Could not update user because they do not exist.'); + } else { + // Handle other potential errors + console.error('An unexpected error occurred:', error); + } +} +``` + +## `Paginator`: Tracking Long-Running Jobs + +The `Paginator` class is a utility for managing the state of a multi-step or long-running process. It's not for paginating data from a list, but for tracking the progress of something like a data import, a report generation, or a multi-page scraping job. + +It stores a simple state object in Redis and provides methods to update it atomically. + +### Use Case Example +Imagine you're indexing a large number of documents. +```typescript +import { Paginator } from '@microfox/db-upstash'; +import { redis } from './lib/redis'; + +// Each indexing job gets a unique ID +const paginator = new Paginator<{ documentsProcessed: number, totalDocuments: number }>(redis, 'indexing-job-123'); + +async function runJob() { + await paginator.startNewIndexing({ documentsProcessed: 0, totalDocuments: 1000 }); + + for (let i = 1; i <= 1000; i++) { + // ... process document i ... + await paginator.updateIndexingStatus({ documentsProcessed: i }); + } + + await paginator.completeIndexing(); +} +``` +You can then check the status from another process: +```typescript +const status = await paginator.getCurrentStatus(); +console.log(`${status.progress.documentsProcessed} / ${status.progress.totalDocuments} processed.`); +``` +This is useful for building resumable jobs, monitoring progress from a UI, and handling failures gracefully. diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.del.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.del.mdx new file mode 100644 index 000000000..b07df0a79 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.del.mdx @@ -0,0 +1,31 @@ +--- +title: 'CrudHash.del' +description: 'API reference for CrudHash.del' +--- + +# CrudHash.del() + +Deletes an item from the store by its ID. This operation is idempotent; it will not throw an error if the item does not exist. + +## Signature + +```ts +del(id: string): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item to delete. + +## Returns + +- `Promise`: A promise that resolves when the delete operation is complete. + +## Example + +```ts +const userIdToDelete = 'user-456'; + +await userStore.del(userIdToDelete); +// The hash at key `users:user-456` is now deleted. +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.get.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.get.mdx new file mode 100644 index 000000000..9ed9393f2 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.get.mdx @@ -0,0 +1,35 @@ +--- +title: 'CrudHash.get' +description: 'API reference for CrudHash.get' +--- + +# CrudHash.get() + +Gets a single item from the store by its ID. + +## Signature + +```ts +get(id: string): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item to retrieve. + +## Returns + +- `Promise`: A promise that resolves to the item object if found, or `null` if the item does not exist. + +## Example + +```ts +const userId = 'user-123'; +const user = await userStore.get(userId); + +if (user) { + console.log(user.name); // 'Jane Doe' +} else { + console.log('User not found.'); +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.getFields.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.getFields.mdx new file mode 100644 index 000000000..77e997af4 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.getFields.mdx @@ -0,0 +1,39 @@ +--- +title: 'CrudHash.getFields' +description: 'API reference for CrudHash.getFields' +--- + +# CrudHash.getFields() + +Atomically gets one or more specific fields of an item without retrieving the entire object. This is more efficient than `get()` if you only need a subset of the data. + +## Signature + +```ts +getFields(id: string, fields: K[]): Promise | null> +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item. +- **`fields`** (`K[]`): An array of strings representing the field names you want to retrieve. + +## Returns + +- `Promise | null>`: A promise that resolves to: + - A partial item object containing only the requested fields that exist. + - An empty object `{}` if the item exists but none of the requested fields do. + - `null` if the item with the specified `id` does not exist. + +## Example + +```ts +const userId = 'user-123'; +const partialUser = await userStore.getFields(userId, ['name', 'lastLogin']); + +if (partialUser) { + console.log(partialUser.name); // 'Jane Doe' + console.log(partialUser.lastLogin); // 1678886400000 + console.log(partialUser.email); // undefined +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.list.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.list.mdx new file mode 100644 index 000000000..e36475e30 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.list.mdx @@ -0,0 +1,36 @@ +--- +title: 'CrudHash.list' +description: 'API reference for CrudHash.list' +--- + +# CrudHash.list() + +Lists all items in the store. This is a convenience method that is equivalent to calling `query('*')`. + +**Note:** This method uses `SCAN` and can be slow and resource-intensive on large datasets. It is intended for smaller collections or administrative purposes. + +## Signature + +```ts +list(options?: { count?: number; offset?: number }): Promise +``` + +## Parameters + +- **`options`** (`object`, optional): An object containing pagination options. + - **`count`** (`number`, optional): The number of items to return per page. + - **`offset`** (`number`, optional): The number of items to skip for pagination. Defaults to `0`. + +## Returns + +- `Promise`: A promise that resolves to an array of all items, respecting the pagination options. + +## Example + +```ts +// Get the first 10 users +const firstTenUsers = await userStore.list({ count: 10 }); + +// Get the next 10 users +const nextTenUsers = await userStore.list({ count: 10, offset: 10 }); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.query.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.query.mdx new file mode 100644 index 000000000..ab9ab5519 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.query.mdx @@ -0,0 +1,37 @@ +--- +title: 'CrudHash.query' +description: 'API reference for CrudHash.query' +--- + +# CrudHash.query() + +Queries for items using a glob-style pattern on the ID part of the Redis key. This method uses the `SCAN` command, making it safe for production use without blocking the server, but it can be slow on very large datasets. + +The implementation is memory-efficient and will not load an unbounded number of keys into memory. + +## Signature + +```ts +query(pattern: string, options?: { count?: number; offset?: number }): Promise +``` + +## Parameters + +- **`pattern`** (`string`): A glob-style pattern to match against item IDs. For example, `user-*` would find all items where the ID starts with `user-`. +- **`options`** (`object`, optional): An object containing pagination options. + - **`count`** (`number`, optional): The number of items to return. + - **`offset`** (`number`, optional): The number of items to skip. Defaults to `0`. + +## Returns + +- `Promise`: A promise that resolves to an array of items that match the pattern, respecting the pagination options. + +## Example + +```ts +// Find all users who are members of 'org-abc' +const orgUsers = await userStore.query('org-abc:*'); + +// Find the first 5 users with a 'guest-' prefix +const guestUsers = await userStore.query('guest-*', { count: 5 }); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.set.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.set.mdx new file mode 100644 index 000000000..775b1a2b5 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.set.mdx @@ -0,0 +1,37 @@ +--- +title: 'CrudHash.set' +description: 'API reference for CrudHash.set' +--- + +# CrudHash.set() + +Sets (creates or overwrites) an item in the store. The item is stored in a Redis Hash. This is an atomic operation that replaces the entire hash. + +## Signature + +```ts +set(id: string, value: T): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item. This will be part of the Redis key. +- **`value`** (`T`): The item data to store. It must be an object containing at least an `id` property. + +## Returns + +- `Promise`: A promise that resolves to the item that was set. + +## Example + +```ts +const user = { + id: 'user-123', + name: 'Jane Doe', + email: 'jane.doe@example.com', + lastLogin: Date.now(), +}; + +await userStore.set(user.id, user); +// The user object is now stored in a hash at key `users:user-123` +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudHash.update.mdx b/docs/micro-sdks/db-upstash/functions/CrudHash.update.mdx new file mode 100644 index 000000000..63d26fe4e --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudHash.update.mdx @@ -0,0 +1,48 @@ +--- +title: 'CrudHash.update' +description: 'API reference for CrudHash.update' +--- + +# CrudHash.update() + +Atomically updates one or more properties of an existing item. This operation is performed using a Lua script to ensure that the check for the item's existence and the update itself are a single, uninterruptible operation, preventing race conditions. + +If the item does not exist, an error is thrown. + +## Signature + +```ts +update(id: string, value: Partial): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item to update. +- **`value`** (`Partial`): An object containing the fields and new values to update. + +## Returns + +- `Promise`: A promise that resolves to the full, updated item object after the update is complete. + +## Throws + +- `ItemNotFoundError`: Thrown if the item with the specified `id` does not exist. + +## Example + +```ts +const userId = 'user-123'; +const updates = { + lastLogin: Date.now(), + loginAttempts: 5, +}; + +try { + const updatedUser = await userStore.update(userId, updates); + console.log(updatedUser.lastLogin); // The new timestamp +} catch (e) { + if (e instanceof ItemNotFoundError) { + console.error('Cannot update user, as they do not exist.'); + } +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.del.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.del.mdx new file mode 100644 index 000000000..a45f8f2c9 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.del.mdx @@ -0,0 +1,33 @@ +--- +title: 'CrudStore.del' +description: 'API reference for CrudStore.del' +--- + +# CrudStore.del() + +Deletes an item by its ID. This method atomically removes the item's hash and its entries from all configured `sortFields` and `matchFields` indexes. + +This operation is idempotent and will not throw an error if the item does not exist. + +## Signature + +```ts +del(id: string): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item to delete. + +## Returns + +- `Promise`: A promise that resolves when the delete operation is complete. + +## Example + +```ts +const postIdToDelete = 'post-001'; + +await postStore.del(postIdToDelete); +// The hash for 'post-001' is deleted, and its ID is removed from all relevant indexes. +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.experimental_reIndex.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.experimental_reIndex.mdx new file mode 100644 index 000000000..6b5b10a9d --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.experimental_reIndex.mdx @@ -0,0 +1,39 @@ +--- +title: 'CrudStore.experimental_reIndex' +description: 'API reference for CrudStore.experimental_reIndex' +--- + +# CrudStore.experimental_reIndex() + +**[EXPERIMENTAL]** + +Deletes all existing `sortField` and `matchField` indexes for the store and rebuilds them from the source-of-truth data in the item hashes. + +This is a potentially long-running and expensive operation that should be used with caution, primarily for maintenance, migrations, or to repair corrupted indexes. It processes all items in batches to avoid overwhelming the server. + +## Signature + +```ts +experimental_reIndex(options?: { + batchSize?: number; +}): Promise<{ reIndexedCount: number }> +``` + +## Parameters + +- **`options`** (`object`, optional): Configuration for the re-indexing process. + - **`batchSize`** (`number`, optional): The number of items to process in each batch. Defaults to `100`. + +## Returns + +- `Promise<{ reIndexedCount: number }>`: A promise that resolves to an object containing the total count of items that were successfully re-indexed. + +## Example + +```ts +// Rebuild all indexes for the postStore +// This is a major operation and should be run from a maintenance script. +const result = await postStore.experimental_reIndex({ batchSize: 200 }); + +console.log(`Successfully re-indexed ${result.reIndexedCount} posts.`); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.get.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.get.mdx new file mode 100644 index 000000000..6b19df664 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.get.mdx @@ -0,0 +1,35 @@ +--- +title: 'CrudStore.get' +description: 'API reference for CrudStore.get' +--- + +# CrudStore.get() + +Gets a single item from the store by its ID. This method fetches the item directly from its hash and does not interact with the indexes. + +## Signature + +```ts +get(id: string): Promise +``` + +## Parameters + +- **`id`** (`string`): The unique ID of the item to retrieve. + +## Returns + +- `Promise`: A promise that resolves to the item object if found, or `null` if the item does not exist. + +## Example + +```ts +const postId = 'post-001'; +const post = await postStore.get(postId); + +if (post) { + console.log(post.title); // 'Hello World' +} else { + console.log('Post not found.'); +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.list.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.list.mdx new file mode 100644 index 000000000..a740f1f64 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.list.mdx @@ -0,0 +1,49 @@ +--- +title: 'CrudStore.list' +description: 'API reference for CrudStore.list' +--- + +# CrudStore.list() + +Lists items from the store, sorted by a specified `sortField`. This method is highly efficient for paginating through ordered data. + +## Signature + +```ts +list(options: { + sortField: StoreField; + offset?: number; + count?: number; + desc?: boolean; +}): Promise +``` + +## Parameters + +- **`options`** (`object`): An object containing sorting and pagination options. + - **`sortField`** (`StoreField`): The name of the field to sort by. This field must have been included in the `sortFields` array in the constructor. + - **`offset`** (`number`, optional): The number of items to skip for pagination. Defaults to `0`. + - **`count`** (`number`, optional): The number of items to return. Defaults to `10`. + - **`desc`** (`boolean`, optional): If `true`, the items will be sorted in descending order. Defaults to `false` (ascending). + +## Returns + +- `Promise`: A promise that resolves to a sorted and paginated array of items. + +## Example + +```ts +// Get the 10 most recent posts +const recentPosts = await postStore.list({ + sortField: 'createdAt', + desc: true, + count: 10, +}); + +// Get the 10 most popular posts +const popularPosts = await postStore.list({ + sortField: 'viewCount', + desc: true, + count: 10, +}); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.queryByComposite.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByComposite.mdx new file mode 100644 index 000000000..6fc7c91ad --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByComposite.mdx @@ -0,0 +1,6 @@ +--- +title: 'CrudStore.queryByComposite' +description: 'API reference for CrudStore.queryByComposite' +--- + + \ No newline at end of file diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.queryByField.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByField.mdx new file mode 100644 index 000000000..251fb3a6f --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByField.mdx @@ -0,0 +1,53 @@ +--- +title: 'CrudStore.queryByField' +description: 'API reference for CrudStore.queryByField' +--- + +--- +title: CrudStore.queryByField() +--- + +# CrudStore.queryByField() + +Queries for items where a field has an exact value. This provides an efficient way to filter a collection by a specific property, like a status or a category. + +The field must be included in the `matchFields` array in the constructor configuration. + +## Signature + +```ts +queryByField(options: { + field: StoreField; + value: string | number; +}): Promise +``` + +## Parameters + +- **`options`** (`object`): The query options. + - **`field`** (`StoreField`): The field to query against. + - **`value`** (`string | number`): The exact value to match. + +## Returns + +- `Promise`: A promise that resolves to an array of items that match the value. + +## Throws + +- `ConfigurationError`: Thrown if the specified `field` was not registered in the `matchFields` config. + +## Example + +```ts +// Get all posts that are in 'draft' status +const draftPosts = await postStore.queryByField({ + field: 'status', + value: 'draft', +}); + +// Get all posts by a specific author +const authorPosts = await postStore.queryByField({ + field: 'authorId', + value: 'user-xyz-789', +}); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.queryByFieldIn.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByFieldIn.mdx new file mode 100644 index 000000000..dab4beac8 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByFieldIn.mdx @@ -0,0 +1,43 @@ +--- +title: 'CrudStore.queryByFieldIn' +description: 'API reference for CrudStore.queryByFieldIn' +--- + +# CrudStore.queryByFieldIn() + +Queries for items where a field's value matches any of the values in a given array. This is the equivalent of a SQL `IN` clause and is highly efficient, using the `SUNION` command in Redis. + +The field must be included in the `matchFields` array in the constructor configuration. + +## Signature + +```ts +queryByFieldIn(options: { + field: StoreField; + values: (string | number)[]; +}): Promise +``` + +## Parameters + +- **`options`** (`object`): The query options. + - **`field`** (`StoreField`): The field to query against. + - **`values`** (`(string | number)[]`): An array of possible values to match. + +## Returns + +- `Promise`: A promise that resolves to an array of items that match any of the provided values. + +## Throws + +- `ConfigurationError`: Thrown if the specified `field` was not registered in the `matchFields` config. + +## Example + +```ts +// Get all posts that are either 'archived' or 'flagged' +const postsToReview = await postStore.queryByFieldIn({ + field: 'status', + values: ['archived', 'flagged'], +}); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.queryByLex.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByLex.mdx new file mode 100644 index 000000000..1234462a5 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByLex.mdx @@ -0,0 +1,20 @@ +--- +title: 'CrudStore.queryByLex' +description: 'API reference for CrudStore.queryByLex' +--- + +# CrudStore.queryByLex() + +**[TODO - Not Implemented]** + +This method is a placeholder for future functionality. When implemented, it will query for items using lexicographical (alphabetical) sorting on a specified index. This will allow for powerful autocomplete-style queries and alphabetical range searches. + +## Signature + +```ts +queryByLex(_options: any): Promise +``` + +## Throws + +- `Error`: This method is not yet implemented and will throw an error if called. diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.queryByScore.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByScore.mdx new file mode 100644 index 000000000..fb5c6554f --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.queryByScore.mdx @@ -0,0 +1,52 @@ +--- +title: 'CrudStore.queryByScore' +description: 'API reference for CrudStore.queryByScore' +--- + +# CrudStore.queryByScore() + +Queries for items within a given score range on a specified `sortField`. This is useful for fetching items based on numeric criteria, such as timestamps, prices, or counts. + +## Signature + +```ts +queryByScore(options: { + sortField: StoreField; + min: number; + max: number; + offset?: number; + count?: number; +}): Promise +``` + +## Parameters + +- **`options`** (`object`): An object containing the query options. + - **`sortField`** (`StoreField`): The indexed field to query against. + - **`min`** (`number`): The minimum score for the range (inclusive). + - **`max`** (`number`): The maximum score for the range (inclusive). + - **`offset`** (`number`, optional): The number of items to skip within the matched range. Defaults to `0`. + - **`count`** (`number`, optional): The maximum number of items to return from the matched range. Defaults to `10`. + +## Returns + +- `Promise`: A promise that resolves to an array of items matching the score range. + +## Example + +```ts +// Get all posts created in the last 24 hours +const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; +const newPosts = await postStore.queryByScore({ + sortField: 'createdAt', + min: oneDayAgo, + max: Date.now(), +}); + +// Find posts with 100 to 500 views +const popularPosts = await postStore.queryByScore({ + sortField: 'viewCount', + min: 100, + max: 500, +}); +``` diff --git a/docs/micro-sdks/db-upstash/functions/CrudStore.set.mdx b/docs/micro-sdks/db-upstash/functions/CrudStore.set.mdx new file mode 100644 index 000000000..fccea2813 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/CrudStore.set.mdx @@ -0,0 +1,45 @@ +--- +title: 'CrudStore.set' +description: 'API reference for CrudStore.set' +--- + +# CrudStore.set() + +Sets (creates or overwrites) an item in the store. This method atomically updates the item's hash and all of its configured indexes (`sortFields` and `matchFields`) in a single batched operation. + +For `matchFields`, this involves reading the old item state first to ensure indexes are cleaned up correctly. + +## Signature + +```ts +set(item: T): Promise +``` + +## Parameters + +- **`item`** (`T`): The item data to store. It must contain an `id` and all fields specified in the `sortFields` and `matchFields` configuration. + +## Returns + +- `Promise`: A promise that resolves to the item that was set. + +## Throws + +- `InvalidFieldError`: Thrown if a `sortField` has a value that is null, undefined, or not a parsable number. + +## Example + +```ts +const newPost = { + id: 'post-001', + title: 'Hello World', + content: '...', + status: 'published', + authorId: 'user-123', + createdAt: Date.now(), + viewCount: 0, +}; + +await postStore.set(newPost); +// The post is now stored and indexed by createdAt, viewCount, status, and authorId. +``` diff --git a/docs/micro-sdks/db-upstash/functions/completeIndexing.mdx b/docs/micro-sdks/db-upstash/functions/completeIndexing.mdx new file mode 100644 index 000000000..f18b3ab7c --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/completeIndexing.mdx @@ -0,0 +1,25 @@ +--- +title: 'completeIndexing' +description: 'API reference for completeIndexing' +--- + +# completeIndexing + +Marks an indexing task as 'completed'. This is the final state for a successful task. + +## Signature + +```typescript +completeIndexing(): Promise> +``` + +### Returns + +- (`Promise>`): A promise that resolves to the final pagination status object with a `status` of `'completed'`. + +### Example + +```typescript +await myScraperPaginator.completeIndexing(); +console.log('Scraping job completed!'); +``` diff --git a/docs/micro-sdks/db-upstash/functions/failIndexing.mdx b/docs/micro-sdks/db-upstash/functions/failIndexing.mdx new file mode 100644 index 000000000..64975d806 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/failIndexing.mdx @@ -0,0 +1,34 @@ +--- +title: 'failIndexing' +description: 'API reference for failIndexing' +--- + +# failIndexing + +Marks an indexing task as 'failed' and stores an error message. This is useful for formally recording a failure state. + +## Signature + +```typescript +failIndexing(error: Record | string): Promise> +``` + +### Parameters + +- `error` (`Record | string`): The error to be stored. Can be a simple string or a more detailed object. + +### Returns + +- (`Promise>`): A promise that resolves to the final pagination status object with a `status` of `'failed'`. + +### Example + +```typescript +try { + // some failing operation + throw new Error('API rate limit exceeded'); +} catch (e: any) { + await myScraperPaginator.failIndexing({ message: e.message, code: 500 }); + console.log('Scraper job failed.'); +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/getCurrentStatus.mdx b/docs/micro-sdks/db-upstash/functions/getCurrentStatus.mdx new file mode 100644 index 000000000..4ad41e82b --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/getCurrentStatus.mdx @@ -0,0 +1,30 @@ +--- +title: 'getCurrentStatus' +description: 'API reference for getCurrentStatus' +--- + +# getCurrentStatus + +Retrieves the current status and progress of the indexing task from Redis. + +## Signature + +```typescript +getCurrentStatus(): Promise | null> +``` + +### Returns + +- (`Promise | null>`): A promise that resolves to the current pagination status object, or `null` if the task has not been started. + +### Example + +```typescript +const status = await myScraperPaginator.getCurrentStatus(); + +if (status) { + console.log(`Current status: ${status.status}`); +} else { + console.log('No job found. Starting a new one.'); +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/isStale.mdx b/docs/micro-sdks/db-upstash/functions/isStale.mdx new file mode 100644 index 000000000..bd0ab2662 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/isStale.mdx @@ -0,0 +1,33 @@ +--- +title: 'isStale' +description: 'API reference for isStale' +--- + +# isStale + +Checks if an indexing task is stale by comparing its `lastUpdatedAt` timestamp with a given timeout. This is useful for detecting and restarting stuck or dead worker processes. + +## Signature + +```typescript +isStale(timeoutSeconds: number): Promise +``` + +### Parameters + +- `timeoutSeconds` (`number`): The timeout in seconds. If the task has not been updated within this period, it is considered stale. + +### Returns + +- (`Promise`): A promise that resolves to `true` if the task is stale, otherwise `false`. + +### Example + +```typescript +// Check if the scraper has been silent for more than 5 minutes +const isStuck = await myScraperPaginator.isStale(300); + +if (isStuck) { + console.log('Warning: Scraper job appears to be stale!'); +} +``` diff --git a/docs/micro-sdks/db-upstash/functions/pauseIndexing.mdx b/docs/micro-sdks/db-upstash/functions/pauseIndexing.mdx new file mode 100644 index 000000000..2d1a4fd5f --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/pauseIndexing.mdx @@ -0,0 +1,25 @@ +--- +title: 'pauseIndexing' +description: 'API reference for pauseIndexing' +--- + +# pauseIndexing + +Pauses a running indexing task. This sets the status to `'paused'`. + +## Signature + +```typescript +pauseIndexing(): Promise> +``` + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object with a `status` of `'paused'`. + +### Example + +```typescript +await myScraperPaginator.pauseIndexing(); +console.log('Scraper has been paused.'); +``` diff --git a/docs/micro-sdks/db-upstash/functions/resetIndexing.mdx b/docs/micro-sdks/db-upstash/functions/resetIndexing.mdx new file mode 100644 index 000000000..ba97e675d --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/resetIndexing.mdx @@ -0,0 +1,26 @@ +--- +title: 'resetIndexing' +description: 'API reference for resetIndexing' +--- + +# resetIndexing + +Deletes all data associated with the indexing task from Redis. This is a destructive operation and should be used with care. + +## Signature + +```typescript +resetIndexing(): Promise +``` + +### Returns + +- (`Promise`): A promise that resolves when the data has been deleted. + +### Example + +```typescript +// Clean up the task state after it's no longer needed +await myScraperPaginator.resetIndexing(); +console.log('Paginator state has been reset.'); +``` diff --git a/docs/micro-sdks/db-upstash/functions/resumeIndexing.mdx b/docs/micro-sdks/db-upstash/functions/resumeIndexing.mdx new file mode 100644 index 000000000..f8a78127c --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/resumeIndexing.mdx @@ -0,0 +1,25 @@ +--- +title: 'resumeIndexing' +description: 'API reference for resumeIndexing' +--- + +# resumeIndexing + +Resumes a paused indexing task by setting its status back to `'running'`. + +## Signature + +```typescript +resumeIndexing(): Promise> +``` + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object with a `status` of `'running'`. + +### Example + +```typescript +const status = await myScraperPaginator.resumeIndexing(); +console.log(`Resumed at page: ${status.progress.page}`); +``` diff --git a/docs/micro-sdks/db-upstash/functions/startNewIndexing.mdx b/docs/micro-sdks/db-upstash/functions/startNewIndexing.mdx new file mode 100644 index 000000000..12d3563af --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/startNewIndexing.mdx @@ -0,0 +1,41 @@ +--- +title: 'startNewIndexing' +description: 'API reference for startNewIndexing' +--- + +# startNewIndexing + +Initializes a new indexing or pagination task. This method sets the initial state in Redis, marks the status as 'running', and records the start time. + +## Signature + +```typescript +startNewIndexing(initialProgress: T): Promise> +``` + +### Parameters + +- `initialProgress` (`T`): An object representing the starting progress of the task. The shape of this object must match the generic type `T` provided to the `Paginator` constructor. + +### Returns + +- (`Promise>`): A promise that resolves to the newly created pagination status object. + +### Example + +```typescript +const initialProgress = { + page: 1, + lastId: null, + itemsProcessed: 0, +}; + +const status = await myScraperPaginator.startNewIndexing(initialProgress); +console.log(status); +// { +// status: 'running', +// progress: { page: 1, lastId: null, itemsProcessed: 0 }, +// startedAt: 1678886400000, +// lastUpdatedAt: 1678886400000 +// } +``` diff --git a/docs/micro-sdks/db-upstash/functions/updateIndexingStatus.mdx b/docs/micro-sdks/db-upstash/functions/updateIndexingStatus.mdx new file mode 100644 index 000000000..ae7c521d8 --- /dev/null +++ b/docs/micro-sdks/db-upstash/functions/updateIndexingStatus.mdx @@ -0,0 +1,34 @@ +--- +title: 'updateIndexingStatus' +description: 'API reference for updateIndexingStatus' +--- + +# updateIndexingStatus + +Updates the progress of an ongoing indexing task. This is the most common method to call during a scraping loop to save the latest state. + +## Signature + +```typescript +updateIndexingStatus(newProgress: Partial): Promise> +``` + +### Parameters + +- `newProgress` (`Partial`): An object containing the progress fields to update. It will be merged with the existing progress state. + +### Returns + +- (`Promise>`): A promise that resolves to the updated pagination status object. + +### Example + +```typescript +const status = await myScraperPaginator.updateIndexingStatus({ + page: 2, + itemsProcessed: 100, +}); + +console.log('Progress updated:', status.progress); +// Progress updated: { page: 2, lastId: 'some-last-id', itemsProcessed: 100 } +``` diff --git a/docs/micro-sdks/db-upstash/getting-started.mdx b/docs/micro-sdks/db-upstash/getting-started.mdx new file mode 100644 index 000000000..4c44cf54f --- /dev/null +++ b/docs/micro-sdks/db-upstash/getting-started.mdx @@ -0,0 +1,171 @@ +--- +title: 'Getting Started' +description: 'A step-by-step tutorial to install, configure, and use @microfox/db-upstash to manage data and rate limit your application.' +--- + +## Prerequisites + +Before you begin, ensure you have the following: +- **Node.js:** Version 16.x or later. +- **An Upstash Account:** You'll need a Redis database. You can create one for free at [Upstash](https://upstash.com/). +- **Environment Variables:** After creating your database, find your `UPSTASH_REDIS_REST_URL` and `UPSTASH_REDIS_REST_TOKEN` from the Upstash console. + +## 1. Installation + +First, you need both the `@upstash/redis` client and our SDK. Install them using your preferred package manager: + + +```bash npm +npm install @upstash/redis @microfox/db-upstash +``` +```bash yarn +yarn add @upstash/redis @microfox/db-upstash +``` +```bash pnpm +pnpm add @upstash/redis @microfox/db-upstash +``` + + +## 2. Setting Up the Redis Client + +Create a file to initialize and export your Redis client. This ensures you use a single, shared connection across your application. + +`lib/redis.ts` +```typescript +import { Redis } from '@upstash/redis'; + +if (!process.env.UPSTASH_REDIS_REST_URL || !process.env.UPSTASH_REDIS_REST_TOKEN) { + throw new Error('Upstash Redis credentials are not set in environment variables.'); +} + +export const redis = new Redis({ + url: process.env.UPSTASH_REDIS_REST_URL, + token: process.env.UPSTASH_REDIS_REST_TOKEN, +}); +``` + +## 3. Defining Your Data Model and Store + +Now, let's create a `CrudStore` to manage a collection of products. + +First, define the TypeScript interface for a product: +`models/product.ts` +```typescript +export interface Product { + id: string; // Must have an 'id' of type string + name: string; + price: number; + category: 'electronics' | 'clothing' | 'books'; + createdAt: number; // Unix timestamp +} +``` + +Next, create an instance of `CrudStore` for your products. This is where you define your indexes. +`lib/product-store.ts` +```typescript +import { CrudStore } from '@microfox/db-upstash'; +import { redis } from './redis'; +import { Product } from '../models/product'; + +export const productStore = new CrudStore(redis, 'products', { + // Fields for efficient sorting and range queries (must be numbers) + sortFields: ['price', 'createdAt'], + // Fields for efficient exact-match filtering + matchFields: ['category'] +}); +``` + +### Understanding the `CrudStore` Config +- The first argument is the `redis` client. +- The second argument, `'products'`, is a `keyPrefix`. All Redis keys created by this store will start with `products:`. +- The third argument configures the indexes: + - `sortFields`: We can now efficiently sort our products by `price` or `createdAt`. + - `matchFields`: We can now efficiently find all products where `category` is `'electronics'`, for example. + +## 4. Using the Store + +Now you can use `productStore` to manage your data. Let's create a script to add some products and then query them. + +`seed.ts` +```typescript +import { productStore } from './lib/product-store'; +import { Product } from './models/product'; + +async function main() { + console.log('Adding products...'); + + await productStore.set({ + id: 'prod_001', + name: 'Laptop Pro', + price: 1200, + category: 'electronics', + createdAt: Date.now() + }); + + await productStore.set({ + id: 'prod_002', + name: 'Classic T-Shirt', + price: 25, + category: 'clothing', + createdAt: Date.now() - 10000 + }); + + console.log('Products added.'); + + console.log('\nQuerying for all electronics...'); + const electronics = await productStore.queryByField({ + field: 'category', + value: 'electronics' + }); + console.log(electronics); + + console.log('\nListing all products sorted by price (cheapest first)...'); + const cheapestFirst = await productStore.list({ + sortField: 'price', + desc: false // ascending + }); + console.log(cheapestFirst); +} + +main(); +``` + +Run this script with `ts-node seed.ts` to see it in action. + +## 5. Adding Rate Limiting + +Let's protect an imaginary API endpoint using the `Ratelimit` class. + +`lib/ratelimit.ts` +```typescript +import { Ratelimit } from '@microfox/db-upstash'; +import { redis } from './redis'; + +// Allow 5 requests every 10 seconds +export const ratelimit = new Ratelimit(redis, { + requests: 5, + window: 10 +}); +``` +Now, you could use this in your API route: +```typescript +import { ratelimit } from './lib/ratelimit'; + +async function handleApiRequest(request: Request) { + const ip = request.headers.get('x-forwarded-for') ?? '127.0.0.1'; + + const { allowed, remaining } = await ratelimit.limit(`ip:${ip}`); + + if (!allowed) { + return new Response('Too many requests', { status: 429 }); + } + + // Your API logic here... + return new Response('Success!'); +} +``` + +## Next Steps +You now have a solid foundation for using `@microfox/db-upstash`. +- **Explore the API:** Dive into the **API Reference** for `CrudStore`, `CrudHash`, and other classes. +- **Learn Advanced Concepts:** Read the **[Foundation](/micro-sdks/db-upstash/foundation)** guide to understand the underlying Redis data structures and advanced indexing strategies. diff --git a/docs/micro-sdks/db-upstash/index.mdx b/docs/micro-sdks/db-upstash/index.mdx new file mode 100644 index 000000000..382cc44c8 --- /dev/null +++ b/docs/micro-sdks/db-upstash/index.mdx @@ -0,0 +1,34 @@ +--- +title: 'Overview' +description: 'A high-level, object-oriented SDK for Upstash Redis that provides powerful data structures and utilities like CRUD, indexing, and rate limiting.' +--- + +## High-Level Data Structures for Upstash Redis + +`@microfox/db-upstash` is a powerful TypeScript SDK that supercharges your Upstash Redis database. It moves beyond simple key-value operations by providing a suite of high-level, object-oriented classes that enable structured, queryable, and efficient data management. + +### What Problem Does It Solve? + +While Redis is incredibly fast and versatile, using it effectively for complex applications requires careful data modeling and often a lot of boilerplate code. Developers frequently need to: +- Manually serialize and deserialize JSON objects. +- Build and maintain their own secondary indexes for querying. +- Implement common patterns like rate limiting or paginated job tracking from scratch. +- Handle the complexities of atomic multi-step operations. + +`@microfox/db-upstash` abstracts these challenges away, providing robust, pre-built solutions for common data patterns, allowing you to build faster and with more confidence. + +### Who Is This For? +This package is ideal for developers building applications on a serverless stack (or any Node.js environment) who need more than just a basic cache from Redis. Use it if you want to: +- Use Upstash Redis as a primary database for structured objects (e.g., user profiles, product catalogs, articles). +- Efficiently query and sort your data without implementing custom indexing logic. +- Implement robust, Redis-backed rate limiting for your APIs. +- Track the progress of long-running, stateful operations. + +### Key Features & When to Use Which Class + +- **`CrudStore`**: The most powerful class. Use this when you need to manage a collection of objects that require efficient querying, sorting, and pagination. It uses secondary indexes for performance. **(Recommended for most use cases)**. +- **`CrudHash`**: A simpler class for managing individual objects. Use this for things like user sessions or shopping carts where you primarily access items by their ID and perform atomic field updates. Avoid this for large collections that need to be queried. +- **`Ratelimit`**: A straightforward implementation of a fixed-window rate limiter. Use this to protect your API endpoints from abuse. +- **`Paginator`**: A utility for tracking the state of long-running, multi-step processes or background jobs. + +This documentation provides a comprehensive **[Getting Started](/micro-sdks/db-upstash/getting-started)** guide, an in-depth look at the package's **[Foundation](/micro-sdks/db-upstash/foundation)**, and a detailed API Reference to help you get the most out of `@microfox/db-upstash`. diff --git a/docs/micro-sdks/index.mdx b/docs/micro-sdks/index.mdx index 096864264..1d10abd44 100644 --- a/docs/micro-sdks/index.mdx +++ b/docs/micro-sdks/index.mdx @@ -1,6 +1,29 @@ --- -title: 'Micro SDKs' -description: 'Byte-sized SDKs for serverless environments - only import what you need' +title: 'Overview' +description: 'Discover our collection of Micro SDKs.' --- -Coming Soon.. +Welcome to the Micro SDKs documentation. Here you will find a collection of lightweight, purpose-built SDKs to help you integrate with various services and platforms. + +## Available SDKs + + + + A lightweight SDK for Amazon Simple Email Service (SES). + + + An SDK for interacting with the Brave Search API. + + + A TypeScript SDK for Upstash Database Utils. + + + A serverless-ready Puppeteer wrapper for browser automation. + + + An SDK for building RAG applications with Upstash Vector. + + + A collection of utilities for building Slack apps. + + diff --git a/docs/micro-sdks/puppeteer-sls/foundation.mdx b/docs/micro-sdks/puppeteer-sls/foundation.mdx new file mode 100644 index 000000000..9cf6e0626 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/foundation.mdx @@ -0,0 +1,52 @@ +--- +title: 'Foundation' +description: 'A deep dive into the core concepts, architecture, and best practices of @microfox/puppeteer-sls.' +--- + +## The Challenge of Puppeteer in Serverless + +Headless browsers are powerful but resource-intensive. Running them in ephemeral, resource-constrained serverless environments presents several significant challenges: + +- **Deployment Package Size:** AWS Lambda has a 250 MB (unzipped) deployment package size limit. A standard Puppeteer installation, which includes a full Chromium browser, far exceeds this limit. +- **Missing System Dependencies:** Serverless Linux runtimes are minimal and lack the numerous shared libraries (like `libnss3.so`, `libgconf-2.so.4`, etc.) that Chromium needs to run. +- **No Writable `/tmp` Space:** While serverless environments provide some temporary disk space, it's limited and ephemeral. Chromium needs a writable file system for user data profiles and temporary files. +- **Performance and Cold Starts:** The time it takes to launch a new browser instance (a "cold start") can add significant latency to function execution, impacting user experience and increasing costs. +- **Memory Consumption:** Running a full browser is memory-intensive. Serverless functions have strict memory limits, and exceeding them can cause the function to crash. + +## The `@microfox/puppeteer-sls` Architecture + +`@microfox/puppeteer-sls` is specifically designed to overcome these challenges through a carefully considered architecture. + +```mermaid +graph TD; + A[Your Serverless Function] --> B{PuppeteerSLS}; + B --> C[@sparticuz/chromium]; + B --> D[puppeteer-core]; + C --> E[Optimized Chromium Binary]; + D --> E; + E --> F[Target Webpage]; +``` + +### Core Components: +- **`puppeteer-core`**: We use `puppeteer-core` instead of the full `puppeteer` package. `puppeteer-core` is a version of Puppeteer that does not download a browser by default, giving us the flexibility to provide our own. +- **`@sparticuz/chromium`**: This is the key to solving the package size and dependency problems. It provides a Brotli-compressed, self-contained Chromium binary that is stripped of non-essential features and is designed to run in AWS Lambda and other serverless environments. It also bundles the required shared library dependencies. + +### How It Solves the Problems: +1. **Size:** The compressed Chromium from `@sparticuz/chromium` is small enough to fit comfortably within serverless deployment limits. +2. **Dependencies:** The bundled binary includes the necessary shared libraries, making the package self-sufficient. +3. **Configuration:** `@microfox/puppeteer-sls` automatically configures `puppeteer-core` with the optimal launch arguments for a serverless environment, including pointing it to the correct Chromium executable and setting flags to disable the sandbox, use `/tmp` for user data, and optimize performance. +4. **Performance:** While cold starts are unavoidable, the lightweight binary and optimized launch arguments minimize the startup time. + +## Best Practices for Serverless Usage + +- **Manage Memory:** When configuring your serverless function, allocate sufficient memory (at least 1024 MB is a good starting point for browser automation). Monitor your function's memory usage and adjust as needed. +- **Set Timeouts Appropriately:** Browser operations can be slow. Set a generous timeout for your serverless function (e.g., 30 seconds or more) to prevent it from terminating prematurely. +- **Graceful Error Handling:** Always wrap your browser automation logic in `try...catch...finally` blocks. Ensure that `browser.close()` is called in the `finally` block to clean up resources, even if an error occurs. This is critical for preventing orphaned Chromium processes that can consume resources. +- **Prefer Individual Functions for Simple Tasks:** For single, discrete operations like `takeSnapShot` or `extractLinks`, importing the function directly is efficient. The library manages the browser lifecycle for you. +- **Use `openPage` for Complex Workflows:** If you need to perform multiple sequential actions on a single page (e.g., fill out a form, click a button, then scrape the results), use `openPage` to get a persistent `page` object. This avoids the overhead of launching a new browser for each step. + +## Limitations + +- **Concurrency:** Be mindful of the concurrency limits of your serverless platform and any downstream services you are interacting with. Scaling up too aggressively can lead to rate-limiting or IP bans. +- **No Persistent State:** Each serverless function invocation is stateless. The browser is new every time, with no cookies, cache, or session data from previous runs. +- **Resource Intensive:** Despite optimizations, running a headless browser is still more resource-intensive than typical serverless tasks. This can impact cost and performance at very high scales. diff --git a/docs/micro-sdks/puppeteer-sls/functions/colorExtract.mdx b/docs/micro-sdks/puppeteer-sls/functions/colorExtract.mdx new file mode 100644 index 000000000..21dcd108b --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/colorExtract.mdx @@ -0,0 +1,40 @@ +--- +title: 'colorExtract' +description: 'The `colorExtract` function identifies the dominant colors of a webpage.' +--- + +## Function Signature + +```typescript +async function colorExtract( + url: string +): Promise +``` + +The `colorExtract` function analyzes the visual elements of a webpage and identifies its dominant colors. This is useful for design analysis, theme extraction, or building tools that adapt to a site's color palette. + +## Parameters + +- **`url`** (string, required): The URL of the webpage to analyze. + +## Returns + +- **`Promise`**: A Promise that resolves to an array of strings, where each string is a dominant color in hexadecimal format (e.g., `#1A2B3C`). + +## Example Usage + +```typescript +import { colorExtract } from '@microfox/puppeteer-sls'; + +async function getWebsiteColors() { + const url = 'https://www.google.com'; + try { + const colors = await colorExtract(url); + console.log('Dominant Colors:', colors); + } catch (error) { + console.error('An error occurred:', error); + } +} + +getWebsiteColors(); +``` diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractHTML.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractHTML.mdx new file mode 100644 index 000000000..7e1ae726a --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractHTML.mdx @@ -0,0 +1,40 @@ +--- +title: 'extractHTML' +description: 'The `extractHTML` function retrieves the full HTML content of a specified webpage.' +--- + +## Function Signature + +```typescript +async function extractHTML( + url: string +): Promise +``` + +The `extractHTML` function navigates to a given URL and returns its complete HTML source code. This is useful when you need the raw HTML for parsing or analysis. + +## Parameters + +- **`url`** (string, required): The URL of the webpage from which you want to extract the HTML. + +## Returns + +- **`Promise`**: A Promise that resolves to the full HTML content of the page. + +## Example Usage + +```typescript +import { extractHTML } from '@microfox/puppeteer-sls'; + +async function getPageSource() { + const url = 'https://www.google.com'; + try { + const html = await extractHTML(url); + console.log('HTML Source:\n', html); + } catch (error) { + console.error('An error occurred:', error); + } +} + +getPageSource(); +``` diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractImages.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractImages.mdx new file mode 100644 index 000000000..f0bc842a7 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractImages.mdx @@ -0,0 +1,44 @@ +--- +title: 'extractImages' +description: 'The `extractImages` function scrapes all the image sources from a given webpage.' +--- + +## Function Signature + +```typescript +async function extractImages( + url: string +): Promise +``` + +The `extractImages` function navigates to a URL and extracts the `src` attribute from every image (``) tag on the page. This is useful for building image scrapers, archiving visual content, or analyzing a webpage's imagery. + +## Parameters + +- **`url`** (string, required): The URL of the webpage from which to extract image sources. + +## Returns + +- **`Promise`**: A Promise that resolves to an array of strings, where each string is an image URL found on the page. + +## Example Usage + +```typescript +import { extractImages } from '@microfox/puppeteer-sls'; + +async function scrapeImages() { + const url = 'https://www.google.com'; + try { + const images = await extractImages(url); + console.log('Found Images:', images); + } catch (error) { + console.error('An error occurred:', error); + } +} + +scrapeImages(); +``` + +## Related Functions + +- `extractImagesFromURL`: This is an alias for `extractImages`. diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractImagesFromURL.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractImagesFromURL.mdx new file mode 100644 index 000000000..4337ff7b8 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractImagesFromURL.mdx @@ -0,0 +1,43 @@ +--- +title: 'extractImagesFromURL' +description: 'Extracts image data from a webpage.' +--- + +The `extractImagesFromURL` function is a convenience function that opens a URL, extracts images, and closes the browser. Use this for single-use scraping tasks. + +## Usage + +```typescript +import { extractImagesFromURL } from '@microfox/puppeteer-sls'; + +async function main() { + const images = await extractImagesFromURL({ + url: 'https://www.pinterest.com/search/pins/?q=cyberpunk', + isLocal: true, + headless: true, + }); + console.log(images); +} + +main(); +``` + +## Arguments + + + + Inherits all options from `openPage`. The following options are also available: + + + When `true`, the function will attempt to scroll through the page to trigger lazy-loaded images and extract background images from CSS. Defaults to `false`. + + + Defines when to consider navigation successful. Defaults to `networkidle2`. + + + + + +## Response + +This function returns a `Promise` that resolves to an array of `ExtractedImage` objects. diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractLinks.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractLinks.mdx new file mode 100644 index 000000000..09a8a8187 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractLinks.mdx @@ -0,0 +1,44 @@ +--- +title: 'extractLinks' +description: 'The `extractLinks` function scrapes all the hyperlinks from a given webpage.' +--- + +## Function Signature + +```typescript +async function extractLinks( + url: string +): Promise +``` + +The `extractLinks` function navigates to a URL and extracts the `href` attribute from every anchor (``) tag on the page. This is useful for web crawling, link checking, or sitemap generation. + +## Parameters + +- **`url`** (string, required): The URL of the webpage from which to extract links. + +## Returns + +- **`Promise`**: A Promise that resolves to an array of strings, where each string is a URL found on the page. + +## Example Usage + +```typescript +import { extractLinks } from '@microfox/puppeteer-sls'; + +async function scrapeLinks() { + const url = 'https://www.google.com'; + try { + const links = await extractLinks(url); + console.log('Found Links:', links); + } catch (error) { + console.error('An error occurred:', error); + } +} + +scrapeLinks(); +``` + +## Related Functions + +- `extractLinksFromUrl`: This is an alias for `extractLinks`. diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractLinksFromUrl.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractLinksFromUrl.mdx new file mode 100644 index 000000000..6563c7840 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractLinksFromUrl.mdx @@ -0,0 +1,35 @@ +--- +title: 'extractLinksFromUrl' +description: 'Extracts hyperlink data from a webpage.' +--- + +A convenience wrapper that opens a URL, extracts links, and closes the browser. + +## Usage + +```typescript +import { extractLinksFromUrl } from '@microfox/puppeteer-sls'; + +async function main() { + const links = await extractLinksFromUrl({ + url: 'https://developers.pinterest.com/docs/api/v5/introduction', + isLocal: true, + headless: true, + }); + console.log(links); +} + +main(); +``` + +## Arguments + + + + Inherits all options from `openPage`. + + + +## Response + +This function returns a `Promise` that resolves to an array of `ExtractedLink` objects. diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractPureText.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractPureText.mdx new file mode 100644 index 000000000..1fe418fcd --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractPureText.mdx @@ -0,0 +1,40 @@ +--- +title: 'extractPureText' +description: 'The `extractPureText` function extracts all the text content from a webpage, excluding HTML tags.' +--- + +## Function Signature + +```typescript +async function extractPureText( + url: string +): Promise +``` + +The `extractPureText` function fetches a webpage and extracts all its text content, stripping out all HTML tags. This is useful for natural language processing, content analysis, or creating a clean text-only version of a webpage. + +## Parameters + +- **`url`** (string, required): The URL of the webpage from which to extract the text. + +## Returns + +- **`Promise`**: A Promise that resolves to a single string containing all the text from the webpage. + +## Example Usage + +```typescript +import { extractPureText } from '@microfox/puppeteer-sls'; + +async function getPageText() { + const url = 'https://example-blog.com/some-article'; + try { + const text = await extractPureText(url); + console.log('Page Text:\n', text); + } catch (error) { + console.error('An error occurred:', error); + } +} + +getPageText(); +``` diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractToMarkdown.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractToMarkdown.mdx new file mode 100644 index 000000000..363866721 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractToMarkdown.mdx @@ -0,0 +1,40 @@ +--- +title: 'extractToMarkdown' +description: 'The `extractToMarkdown` function converts the content of a webpage into a clean, readable Markdown format.' +--- + +## Function Signature + +```typescript +async function extractToMarkdown( + url: string +): Promise +``` + +The `extractToMarkdown` function fetches a webpage and converts its main content into Markdown. This is particularly useful for content extraction, archiving, or feeding webpage data into language models. + +## Parameters + +- **`url`** (string, required): The URL of the webpage you want to convert to Markdown. + +## Returns + +- **`Promise`**: A Promise that resolves to the Markdown representation of the webpage's content. + +## Example Usage + +```typescript +import { extractToMarkdown } from '@microfox/puppeteer-sls'; + +async function getArticleInMarkdown() { + const url = 'https://example-blog.com/some-article'; + try { + const markdownContent = await extractToMarkdown(url); + console.log('Markdown Content:\n', markdownContent); + } catch (error) { + console.error('An error occurred:', error); + } +} + +getArticleInMarkdown(); +``` diff --git a/docs/micro-sdks/puppeteer-sls/functions/extractVideos.mdx b/docs/micro-sdks/puppeteer-sls/functions/extractVideos.mdx new file mode 100644 index 000000000..ac21802f2 --- /dev/null +++ b/docs/micro-sdks/puppeteer-sls/functions/extractVideos.mdx @@ -0,0 +1,44 @@ +--- +title: 'extractVideos' +description: 'The `extractVideos` function scrapes all video sources from a given webpage.' +--- + +## Function Signature + +```typescript +async function extractVideos( + url: string +): Promise +``` + +The `extractVideos` function navigates to a URL and extracts the `src` attribute from every video (`