diff --git a/.github/workflows/bot-pr-checks.yml b/.github/workflows/bot-pr-checks.yml new file mode 100644 index 000000000..95ab19e26 --- /dev/null +++ b/.github/workflows/bot-pr-checks.yml @@ -0,0 +1,50 @@ +name: Bot PR Checks + +on: + pull_request_target: + types: [ opened, synchronize, reopened, labeled ] + branches: + - "develop" + +jobs: + bot-check: + runs-on: ubuntu-latest + if: contains(github.triggering_actor, '[bot]') + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup environment + uses: ./.github/actions/setup-environment + + - name: Run tests + timeout-minutes: 5 + run: | + uv run pytest \ + -n auto \ + --cov src \ + --timeout 15 \ + -o junit_suite_name="bot-unit-tests" \ + tests/unit + + - uses: ./.github/actions/report + with: + flag: bot-unit-tests + codecov_token: ${{ secrets.CODECOV_TOKEN }} + + # Add a job to handle the Build & Release workflow for bot PRs + bot-build: + runs-on: ubuntu-latest + if: contains(github.triggering_actor, '[bot]') + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Skip build for bot PRs + run: echo "Skipping build for bot PRs" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 4d3c5c9e8..4662ff63d 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -1,7 +1,7 @@ name: Mypy Checks on: - pull_request: + pull_request_target: branches: - "develop" @@ -19,6 +19,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} - name: Setup environment uses: ./.github/actions/setup-environment diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 153fe1d9b..c58ff3786 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,7 +1,7 @@ name: pre-commit on: - pull_request: + pull_request_target: branches: - "develop" push: @@ -21,6 +21,7 @@ jobs: with: fetch-depth: 0 token: ${{ env.REPO_SCOPED_TOKEN || github.token }} + ref: ${{ github.event.pull_request.head.sha }} - name: Setup environment uses: ./.github/actions/setup-environment diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c850ad3c1..3751a14c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,8 +25,26 @@ permissions: contents: read jobs: + # Add a check to skip the build for bot PRs + check-bot: + runs-on: ubuntu-latest + outputs: + is_bot: ${{ steps.check-bot.outputs.is_bot }} + steps: + - name: Check if user is bot + id: check-bot + run: | + if [[ "${{ github.triggering_actor }}" == *"[bot]" ]]; then + echo "is_bot=true" >> $GITHUB_OUTPUT + else + echo "is_bot=false" >> $GITHUB_OUTPUT + fi + build: name: Build 3.${{ matrix.python }} ${{ matrix.os }} + needs: check-bot + # Skip this job if the PR is from a bot + if: needs.check-bot.outputs.is_bot != 'true' runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e500b424..06b7396d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,17 @@ jobs: access-check: runs-on: ubuntu-latest steps: + - name: Check if user is bot + id: check-bot + run: | + if [[ "${{ github.triggering_actor }}" == *"[bot]" ]]; then + echo "is_bot=true" >> $GITHUB_OUTPUT + else + echo "is_bot=false" >> $GITHUB_OUTPUT + fi + - uses: actions-cool/check-user-permission@v2 + if: steps.check-bot.outputs.is_bot != 'true' with: require: write username: ${{ github.triggering_actor }} diff --git a/organize_codebase.py b/organize_codebase.py new file mode 100644 index 000000000..d12d4f660 --- /dev/null +++ b/organize_codebase.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +""" +Codebase Organizer Script + +This script helps organize a codebase by analyzing file contents and moving +related files into appropriate directories based on their functionality. +""" + +import os +import re +import shutil +from pathlib import Path +from typing import Dict, List, Set, Tuple + +# Define categories and their related patterns +CATEGORIES = { + "analyzers": [ + r"analyzer", r"analysis", r"analyze" + ], + "code_quality": [ + r"code_quality", r"quality", r"lint" + ], + "context": [ + r"context", r"codebase_context" + ], + "dependencies": [ + r"dependenc", r"import" + ], + "issues": [ + r"issue", r"error" + ], + "visualization": [ + r"visual", r"display", r"render" + ], +} + +def read_file_content(file_path: str) -> str: + """Read the content of a file.""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return f.read() + except Exception as e: + print(f"Error reading {file_path}: {e}") + return "" + +def categorize_file(file_path: str, categories: Dict[str, List[str]]) -> List[str]: + """Categorize a file based on its content and name.""" + file_categories = [] + content = read_file_content(file_path) + filename = os.path.basename(file_path) + + # Check filename and content against category patterns + for category, patterns in categories.items(): + for pattern in patterns: + if re.search(pattern, filename, re.IGNORECASE) or re.search(pattern, content, re.IGNORECASE): + file_categories.append(category) + break + + return file_categories + +def analyze_imports(file_path: str) -> Set[str]: + """Analyze imports in a Python file.""" + imports = set() + content = read_file_content(file_path) + + # Find import statements + import_patterns = [ + r'import\s+([a-zA-Z0-9_\.]+)', + r'from\s+([a-zA-Z0-9_\.]+)\s+import' + ] + + for pattern in import_patterns: + for match in re.finditer(pattern, content): + imports.add(match.group(1)) + + return imports + +def build_dependency_graph(files: List[str]) -> Dict[str, Set[str]]: + """Build a dependency graph for the files.""" + graph = {} + module_to_file = {} + + # Map module names to files + for file_path in files: + if not file_path.endswith('.py'): + continue + + module_name = os.path.splitext(os.path.basename(file_path))[0] + module_to_file[module_name] = file_path + + # Build the graph + for file_path in files: + if not file_path.endswith('.py'): + continue + + imports = analyze_imports(file_path) + graph[file_path] = set() + + for imp in imports: + # Check if this is a local import + parts = imp.split('.') + if parts[0] in module_to_file: + graph[file_path].add(module_to_file[parts[0]]) + + return graph + +def find_related_files(graph: Dict[str, Set[str]], file_path: str) -> Set[str]: + """Find files related to the given file based on the dependency graph.""" + related = set() + + # Files that this file imports + if file_path in graph: + related.update(graph[file_path]) + + # Files that import this file + for other_file, deps in graph.items(): + if file_path in deps: + related.add(other_file) + + return related + +def organize_files(directory: str, dry_run: bool = True) -> Dict[str, List[str]]: + """ + Organize files in the directory into categories. + + Args: + directory: The directory containing the files to organize + dry_run: If True, only print the planned changes without making them + + Returns: + A dictionary mapping categories to lists of files + """ + # Get all Python files + py_files = [os.path.join(directory, f) for f in os.listdir(directory) + if f.endswith('.py') and os.path.isfile(os.path.join(directory, f))] + + # Build dependency graph + graph = build_dependency_graph(py_files) + + # Categorize files + categorized_files = {} + for category in CATEGORIES: + categorized_files[category] = [] + + # Special case for README and init files + categorized_files["root"] = [] + + for file_path in py_files: + filename = os.path.basename(file_path) + + # Keep some files in the root directory + if filename in ['__init__.py', 'README.md']: + categorized_files["root"].append(file_path) + continue + + # Categorize the file + categories = categorize_file(file_path, CATEGORIES) + + if not categories: + # If no category found, use related files to determine category + related = find_related_files(graph, file_path) + for related_file in related: + related_categories = categorize_file(related_file, CATEGORIES) + categories.extend(related_categories) + + # Remove duplicates + categories = list(set(categories)) + + if not categories: + # If still no category, put in a default category based on filename + if "analyzer" in filename: + categories = ["analyzers"] + elif "context" in filename: + categories = ["context"] + elif "issue" in filename or "error" in filename: + categories = ["issues"] + elif "visual" in filename: + categories = ["visualization"] + elif "depend" in filename: + categories = ["dependencies"] + elif "quality" in filename: + categories = ["code_quality"] + else: + # Default to analyzers if nothing else matches + categories = ["analyzers"] + + # Use the first category (most relevant) + primary_category = categories[0] + categorized_files[primary_category].append(file_path) + + # Print and execute the organization plan + for category, files in categorized_files.items(): + if not files: + continue + + print(f"\nCategory: {category}") + for file_path in files: + print(f" - {os.path.basename(file_path)}") + + if not dry_run and category != "root": + # Create the category directory if it doesn't exist + category_dir = os.path.join(directory, category) + os.makedirs(category_dir, exist_ok=True) + + # Move files to the category directory + for file_path in files: + if category != "root": + dest_path = os.path.join(category_dir, os.path.basename(file_path)) + shutil.move(file_path, dest_path) + print(f" Moved to {dest_path}") + + return categorized_files + +def main(): + """Main function to organize the codebase.""" + import argparse + + parser = argparse.ArgumentParser(description='Organize a codebase by categorizing files.') + parser.add_argument('directory', help='The directory containing the files to organize') + parser.add_argument('--execute', action='store_true', help='Execute the organization plan (default is dry run)') + + args = parser.parse_args() + + print(f"Analyzing files in {args.directory}...") + organize_files(args.directory, dry_run=not args.execute) + + if not args.execute: + print("\nThis was a dry run. Use --execute to actually move the files.") + else: + print("\nFiles have been organized.") + +if __name__ == "__main__": + main() + diff --git a/organize_specific_codebase.py b/organize_specific_codebase.py new file mode 100644 index 000000000..cfe8f534d --- /dev/null +++ b/organize_specific_codebase.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +Specific Codebase Organizer + +This script organizes the specific codebase structure shown in the screenshot, +with 5 folders and 21 Python files in the root directory. +""" + +import os +import re +import shutil +from pathlib import Path +from typing import Dict, List, Set + +# Define the organization structure based on the files in the screenshot +ORGANIZATION_PLAN = { + "analyzers": [ + "analyzer.py", + "analyzer_manager.py", + "base_analyzer.py", + "code_quality_analyzer.py", + "codebase_analyzer.py", + "dependency_analyzer.py", + "error_analyzer.py", + "unified_analyzer.py" + ], + "code_quality": [ + "code_quality.py" + ], + "context": [ + "codebase_context.py", + "context_codebase.py", + "current_code_codebase.py" + ], + "issues": [ + "issue_analyzer.py", + "issue_types.py", + "issues.py" + ], + "dependencies": [ + "dependencies.py" + ], + # Files to keep in root + "root": [ + "__init__.py", + "api.py", + "README.md" + ] +} + +def organize_specific_codebase(directory: str, dry_run: bool = True) -> None: + """ + Organize the specific codebase structure. + + Args: + directory: The directory containing the files to organize + dry_run: If True, only print the planned changes without making them + """ + print(f"Organizing codebase in {directory}...") + + # Create directories if they don't exist (unless dry run) + if not dry_run: + for category in ORGANIZATION_PLAN: + if category != "root": + os.makedirs(os.path.join(directory, category), exist_ok=True) + + # Process each file according to the plan + for category, files in ORGANIZATION_PLAN.items(): + print(f"\nCategory: {category}") + + for filename in files: + source_path = os.path.join(directory, filename) + + # Skip if file doesn't exist + if not os.path.exists(source_path): + print(f" - {filename} (not found, skipping)") + continue + + print(f" - {filename}") + + # Move the file if not a dry run and not in root category + if not dry_run and category != "root": + dest_path = os.path.join(directory, category, filename) + shutil.move(source_path, dest_path) + print(f" Moved to {dest_path}") + + # Handle any remaining Python files not explicitly categorized + all_planned_files = [f for files in ORGANIZATION_PLAN.values() for f in files] + remaining_files = [f for f in os.listdir(directory) + if f.endswith('.py') and os.path.isfile(os.path.join(directory, f)) + and f not in all_planned_files] + + if remaining_files: + print("\nRemaining Python files (not categorized):") + for filename in remaining_files: + print(f" - {filename}") + + # Try to categorize based on filename + if "analyzer" in filename.lower(): + category = "analyzers" + elif "context" in filename.lower() or "codebase" in filename.lower(): + category = "context" + elif "visual" in filename.lower(): + category = "visualization" + elif "issue" in filename.lower() or "error" in filename.lower(): + category = "issues" + elif "depend" in filename.lower(): + category = "dependencies" + elif "quality" in filename.lower(): + category = "code_quality" + else: + # Default to analyzers + category = "analyzers" + + print(f" Suggested category: {category}") + + # Move the file if not a dry run + if not dry_run: + os.makedirs(os.path.join(directory, category), exist_ok=True) + dest_path = os.path.join(directory, category, filename) + shutil.move(os.path.join(directory, filename), dest_path) + print(f" Moved to {dest_path}") + +def main(): + """Main function to organize the specific codebase.""" + import argparse + + parser = argparse.ArgumentParser(description='Organize the specific codebase structure.') + parser.add_argument('directory', help='The directory containing the files to organize') + parser.add_argument('--execute', action='store_true', help='Execute the organization plan (default is dry run)') + + args = parser.parse_args() + + organize_specific_codebase(args.directory, dry_run=not args.execute) + + if not args.execute: + print("\nThis was a dry run. Use --execute to actually move the files.") + else: + print("\nFiles have been organized according to the plan.") + + print("\nAfter organizing, you may need to update imports in your code.") + print("You can use the Codegen SDK to automatically update imports:") + print(""" + # Example code to update imports after moving files + from codegen.sdk import Codebase + + # Initialize the codebase + codebase = Codebase("path/to/your/codebase") + + # Commit the changes to ensure the codebase is up-to-date + codebase.commit() + """) + +if __name__ == "__main__": + main() + diff --git a/organize_with_codegen_sdk.py b/organize_with_codegen_sdk.py new file mode 100644 index 000000000..263947c1b --- /dev/null +++ b/organize_with_codegen_sdk.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +Codebase Organizer using Codegen SDK + +This script uses the Codegen SDK to programmatically organize a codebase by +moving symbols between files and updating imports automatically. +""" + +import os +import sys +from typing import Dict, List, Set, Optional + +try: + from codegen.sdk import Codebase +except ImportError: + print("Error: Codegen SDK not found. Please install it with:") + print("pip install codegen-sdk") + sys.exit(1) + +# Define the organization structure based on the files in the screenshot +ORGANIZATION_PLAN = { + "analyzers": [ + "analyzer.py", + "analyzer_manager.py", + "base_analyzer.py", + "code_quality_analyzer.py", + "codebase_analyzer.py", + "dependency_analyzer.py", + "error_analyzer.py", + "unified_analyzer.py" + ], + "code_quality": [ + "code_quality.py" + ], + "context": [ + "codebase_context.py", + "context_codebase.py", + "current_code_codebase.py" + ], + "issues": [ + "issue_analyzer.py", + "issue_types.py", + "issues.py" + ], + "dependencies": [ + "dependencies.py" + ], + # Files to keep in root + "root": [ + "__init__.py", + "api.py", + "README.md" + ] +} + +def organize_with_codegen_sdk(directory: str, dry_run: bool = True) -> None: + """ + Organize the codebase using Codegen SDK. + + Args: + directory: The directory containing the files to organize + dry_run: If True, only print the planned changes without making them + """ + print(f"Organizing codebase in {directory} using Codegen SDK...") + + # Initialize the codebase + codebase = Codebase(directory) + + # Create directories if they don't exist (unless dry run) + if not dry_run: + for category in ORGANIZATION_PLAN: + if category != "root": + os.makedirs(os.path.join(directory, category), exist_ok=True) + + # Process each file according to the plan + for category, files in ORGANIZATION_PLAN.items(): + if category == "root": + continue # Skip files that should stay in root + + print(f"\nCategory: {category}") + + for filename in files: + source_path = os.path.join(directory, filename) + + # Skip if file doesn't exist + if not os.path.exists(source_path): + print(f" - {filename} (not found, skipping)") + continue + + print(f" - {filename}") + + # Move the file if not a dry run + if not dry_run: + try: + # Get the source file + source_file = codebase.get_file(filename) + + # Create the destination file path + dest_path = os.path.join(category, filename) + + # Create the destination file if it doesn't exist + if not os.path.exists(os.path.join(directory, dest_path)): + dest_file = codebase.create_file(dest_path) + else: + dest_file = codebase.get_file(dest_path) + + # Move all symbols from source to destination + for symbol in source_file.symbols: + print(f" Moving symbol: {symbol.name}") + symbol.move_to_file( + dest_file, + include_dependencies=True, + strategy="update_all_imports" + ) + + # Commit changes to ensure the codebase is up-to-date + codebase.commit() + + print(f" Moved to {dest_path} with imports updated") + except Exception as e: + print(f" Error moving {filename}: {e}") + + # Handle any remaining Python files not explicitly categorized + all_planned_files = [f for files in ORGANIZATION_PLAN.values() for f in files] + remaining_files = [f for f in os.listdir(directory) + if f.endswith('.py') and os.path.isfile(os.path.join(directory, f)) + and f not in all_planned_files] + + if remaining_files: + print("\nRemaining Python files (not categorized):") + for filename in remaining_files: + print(f" - {filename}") + + # Try to categorize based on filename + if "analyzer" in filename.lower(): + category = "analyzers" + elif "context" in filename.lower() or "codebase" in filename.lower(): + category = "context" + elif "visual" in filename.lower(): + category = "visualization" + elif "issue" in filename.lower() or "error" in filename.lower(): + category = "issues" + elif "depend" in filename.lower(): + category = "dependencies" + elif "quality" in filename.lower(): + category = "code_quality" + else: + # Default to analyzers + category = "analyzers" + + print(f" Suggested category: {category}") + + # Move the file if not a dry run + if not dry_run: + try: + # Get the source file + source_file = codebase.get_file(filename) + + # Create the destination file path + dest_path = os.path.join(category, filename) + + # Create the destination file if it doesn't exist + if not os.path.exists(os.path.join(directory, dest_path)): + dest_file = codebase.create_file(dest_path) + else: + dest_file = codebase.get_file(dest_path) + + # Move all symbols from source to destination + for symbol in source_file.symbols: + print(f" Moving symbol: {symbol.name}") + symbol.move_to_file( + dest_file, + include_dependencies=True, + strategy="update_all_imports" + ) + + # Commit changes to ensure the codebase is up-to-date + codebase.commit() + + print(f" Moved to {dest_path} with imports updated") + except Exception as e: + print(f" Error moving {filename}: {e}") + +def main(): + """Main function to organize the codebase using Codegen SDK.""" + import argparse + + parser = argparse.ArgumentParser(description='Organize the codebase using Codegen SDK.') + parser.add_argument('directory', help='The directory containing the files to organize') + parser.add_argument('--execute', action='store_true', help='Execute the organization plan (default is dry run)') + + args = parser.parse_args() + + organize_with_codegen_sdk(args.directory, dry_run=not args.execute) + + if not args.execute: + print("\nThis was a dry run. Use --execute to actually move the files.") + else: + print("\nFiles have been organized according to the plan.") + +if __name__ == "__main__": + main() +