Merge branch 'main' into fern/link-updates

.github/scripts/check-mdx-frontmatter.js → .github/scripts/check-mdx-frontmatter.cjs

@@ -10,9 +10,16 @@ let totalFilesChecked = 0;
 let totalFilesValid = 0;
 let totalFilesInvalid = 0;
 
+// List of validators to run
+const validators = [checkDescriptionLength, checkTitleLength];
+
 // List of folders to exclude (relative to mdxDir)
 const excludedFolders = ["-ARCHIVE-", "api-reference", "llm-university"];
 
+function logInvalidMessage(message) {
+    console.error(`[INVALID]: ${message}`);
+}
+
 function shouldExcludeFolder(dirPath) {
   return excludedFolders.some((excludedFolder) => {
     return path.relative(mdxDir, dirPath).startsWith(excludedFolder);
@@ -25,36 +32,68 @@ async function shouldExcludeFile(filePath) {
     const { data } = matter(fileContent);
     return data.hidden === true;
   } catch (error) {
-    console.error(`Error reading file "${filePath}":`, error);
+    console.error(`[ERROR]: Error reading file "${filePath}":`, error);
     return false; // In case of error, don't exclude the file
   }
 }
 
 async function checkDescriptionLength(filePath) {
-  totalFilesChecked++;
   const fileContent = await fs.readFile(filePath, "utf8");
   const { data } = matter(fileContent);
+  const minDescriptionLength = 50;
+  const maxDescriptionLength = 160;
 
   if (!data.description) {
-    console.log(`File "${filePath}" is missing a description.`);
-    totalFilesInvalid++;
+    logInvalidMessage(`File "${filePath}" is missing a description.`);
     return false;
   }
 
   const descriptionLength = data.description.length;
 
-  if (descriptionLength < 50 || descriptionLength > 160) {
-    console.log(
-      `File "${filePath}" has an invalid description length: ${descriptionLength} characters.`
+  if (descriptionLength < minDescriptionLength || descriptionLength > maxDescriptionLength) {
+    logInvalidMessage(
+      `File "${filePath}" has an invalid description length: ${descriptionLength} characters. ` +
+      `Description should be between ${minDescriptionLength}-${maxDescriptionLength} characters.`
     );
-    totalFilesInvalid++;
     return false;
   }
 
-  totalFilesValid++;
   return true;
 }
 
+
+async function checkTitleLength(filePath) {
+    // these two files are layout files 
+    // and we don't expect to have title in them
+    const filesToExclude = ["index.mdx", "cookbooks.mdx"];
+
+    const fileContent = await fs.readFile(filePath, "utf8");
+    const { data } = matter(fileContent);
+    const minTitleLength = 30;
+    const maxTitleLength = 60;
+
+    filePath = path.relative(mdxDir, filePath);
+
+    if (!data.title) {
+        if (filesToExclude.includes(filePath)) {
+            return true;
+        }
+        logInvalidMessage(`File "${filePath}" is missing a title.`);
+        return false;
+    }
+
+    const titleLength = data.title.length;
+    if (titleLength < minTitleLength || titleLength > maxTitleLength) {
+        console.warn(
+            `File "${filePath}" has an invalid title length: ${titleLength} characters. ` +
+            `Title should be between ${minTitleLength}-${maxTitleLength} characters.`
+        );
+        return true;
+    }
+
+    return true;
+}
+
 async function checkMDXFiles(dirPath) {
   let allFilesValid = true;
   const files = await fs.readdir(dirPath);
@@ -77,9 +116,22 @@ async function checkMDXFiles(dirPath) {
         console.log(`Skipping excluded file: ${fullPath}`);
         continue;
       }
-      const isValid = await checkDescriptionLength(fullPath);
+      let isValid = true;
+
+      for (const validate of validators) {
+        const fileIsValid = await validate(fullPath);
+        if (!fileIsValid) {
+            isValid = false; 
+        }
+      }
+
+      totalFilesChecked++;
       if (!isValid) {
         allFilesValid = false;
+        totalFilesInvalid++;
+      }
+      else {
+        totalFilesValid++;
       }
     }
   }
@@ -98,12 +150,12 @@ async function checkMDXFiles(dirPath) {
 
   if (!allFilesValid) {
     console.error(
-      "Some files have invalid or missing descriptions. Meta description needing to be 50-160 characters"
+      "Some files have invalid or missing content."
     );
     process.exit(1); // Fail if any file is invalid
   } else {
     console.log(
-      "All files have a valid description length in the frontmatter."
+      "All files a valid for frontmatter."
     );
   }
 })();

.github/scripts/check_python_code_snippets.py

@@ -0,0 +1,99 @@
+import os
+import re
+from pathlib import Path
+import black
+
+DEFAULT_LINE_LENGTH = 70
+BASE_DIR = Path(__file__).resolve().parent
+MDX_DIR = BASE_DIR / "../../fern/pages"
+FILE_PATTERN = re.compile(r"\.mdx$")
+
+
+def find_files_by_pattern(directory, pattern):
+    """
+    Finds all files in the given directory that match the provided regex pattern.
+    """
+    directory = Path(directory).resolve()
+    if not directory.is_dir():
+        raise ValueError(f"Provided directory {directory} is not valid.")
+    return [f for f in directory.rglob('*') if f.is_file() and pattern.search(f.name)]
+
+
+def format_python_snippets_in_mdx(file_path, line_length=DEFAULT_LINE_LENGTH):
+    """
+    Formats Python code snippets inside MDX files using Black.
+    """
+    black_mode = black.FileMode(line_length=line_length)
+    code_block_pattern = re.compile(r"```python\n(.*?)\n```", re.DOTALL)
+
+    with open(file_path, 'r', encoding='utf-8') as file:
+        original_content = file.read()
+
+    def format_with_black(match):
+        code = match.group(1)
+
+        # Comment out lines starting with '!'
+        processed_code = re.sub(r"^\s*!(.*)", r"# TEMP_COMMENT !\1", code, flags=re.MULTILINE)
+
+        try:
+            # Format the code with Black
+            formatted_code = black.format_str(processed_code, mode=black_mode)
+        except black.NothingChanged:
+            # If Black doesn't change anything, use original
+            formatted_code = processed_code  
+
+        # Revert the commented lines starting with '!'
+        reverted_code = re.sub(r"^\s*# TEMP_COMMENT !(.*)", r"!\1", formatted_code, flags=re.MULTILINE)
+
+        return f"```python\n{reverted_code.strip()}\n```"
+
+    new_content = code_block_pattern.sub(format_with_black, original_content)
+
+    with open(file_path, 'w', encoding='utf-8') as file:
+        file.write(new_content)
+
+    return original_content, new_content
+
+
+def process_mdx_files(directory, file_pattern, line_length=DEFAULT_LINE_LENGTH, check_changes=False):
+    """
+    Processes all MDX files in the directory, formatting Python code snippets.
+
+    Args:
+        directory (Path or str): Path to the directory containing MDX files.
+        file_pattern (re.Pattern): Regex pattern to match MDX files.
+        line_length (int): Line length to use for Black formatting.
+        check_changes (bool): If True, raises an exception if changes are detected.
+    """
+    matching_files = find_files_by_pattern(directory, file_pattern)
+    files_changed = []
+
+    for file_path in matching_files:
+        original_content, new_content = format_python_snippets_in_mdx(file_path, line_length)
+
+        if original_content != new_content:
+            files_changed.append(file_path)
+
+    if check_changes and files_changed:
+        raise RuntimeError(
+            f"The following files were modified during the run:\n"
+            + "\n".join(str(file) for file in files_changed)
+        )
+
+
+if __name__ == "__main__":
+    import sys
+
+    path = sys.argv[1] if len(sys.argv) > 1 else MDX_DIR
+    line_length = int(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_LINE_LENGTH
+    check_changes = os.getenv("CI") == "true"  # Set to True in CI pipeline
+
+    if Path(path).is_dir():
+        process_mdx_files(path, FILE_PATTERN, line_length, check_changes)
+    elif Path(path).is_file():
+        if FILE_PATTERN.search(path):
+            process_mdx_files(Path(path).parent, FILE_PATTERN, line_length, check_changes)
+        else:
+            print("The specified file does not match the MDX pattern.")
+    else:
+        print("Provided path is not valid.")

.github/workflows/check-mdx-frontmatter.yml

@@ -26,4 +26,4 @@ jobs:
         run: pnpm install
 
       - name: Run MDX frontmatter check
-        run: node .github/scripts/check-mdx-frontmatter.js
+        run: node .github/scripts/check-mdx-frontmatter.cjs

.github/workflows/check-python-code-snippets.yml

@@ -0,0 +1,39 @@
+name: check-python-code-snippets
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'fern/pages/**/*.mdx'
+      - 'fern/pages/**/**/*.mdx'
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        shell: bash
+        run: |
+          pipx install poetry
+
+      - name: Install Project Dependencies with Poetry
+        shell: bash
+        run: |
+          poetry install
+
+      - name: Run Python MDX Snippet Formatter
+        shell: bash
+        env:
+          CI: true
+        run: poetry run python .github/scripts/check_python_code_snippets.py fern/pages

.github/workflows/create-sdk-releases.yml

@@ -0,0 +1,46 @@
+name: create-sdk-releases
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump_type:
+        description: 'The type of version bump to make'
+        required: true
+        type: choice
+        options:
+          - 'major'
+          - 'minor'
+          - 'patch'
+      language:
+        description: 'If you want to just release a specific language'
+        type: choice
+        options:
+          - 'all'
+          - 'go'
+          - 'java'
+          - 'python'
+          - 'typescript'
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install             
+
+      - name: Run snippet tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_OWNER: ${{ github.repository_owner }}
+          GITHUB_REPO: ${{ github.event.repository.name }}
+          BUMP_TYPE: ${{ github.event.inputs.bump_type }}
+          LANGUAGE: ${{ github.event.inputs.language }}
+        run: pnpm run --filter autorelease release

.github/workflows/delete-n-releases.yml

@@ -0,0 +1,33 @@
+name: delete-n-releases
+
+on:
+  workflow_dispatch:
+    inputs:
+      n:
+        description: 'Number of releases to delete'
+        required: true
+        default: "1"
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install
+
+      - name: Run snippet tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_OWNER: ${{ github.repository_owner }}
+          GITHUB_REPO: ${{ github.event.repository.name }}
+          N_VERSIONS: ${{ github.event.inputs.n }}
+        run: pnpm run --filter autorelease delete

.github/workflows/preview-docs.yml

@@ -1,20 +1,28 @@
 name: preview-docs
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - main
 
 jobs:
   run:
     runs-on: ubuntu-latest
-    permissions: write-all
+    permissions:
+      pull-requests: write  # Only for commenting
+      contents: read       # For checking out code
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         # with:
         #   ref: "${{ github.event.pull_request.merge_commit_sha }}"
 
+      - name: Checkout PR
+        if: github.event_name == 'pull_request_target'
+        run: |
+          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-${{ github.event.pull_request.number }}
+          git checkout pr-${{ github.event.pull_request.number }}
+
       - name: Setup pnpm
         uses: pnpm/action-setup@v2
         with: