WIP

Author: ofeki-neosec

.gitignore

@@ -5,6 +5,8 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
+.venv
+.idea
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

.vscode/launch.json

@@ -1,6 +1,16 @@
 {
     "version": "0.2.0",
     "configurations": [
+        
+        {
+            "name": "Attach by Process ID",
+            "processId": "${command:PickProcess}",
+            "request": "attach",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node"
+        },
         {
             "name": "Pyright CLI",
             "type": "node",

.vscode/settings.json

@@ -8,7 +8,7 @@
         "editor.formatOnSave": true
     },
     "editor.codeActionsOnSave": {
-        "source.fixAll.eslint": true
+        "source.fixAll.eslint": "explicit"
     },
     "typescript.tsdk": "node_modules/typescript/lib"
 }

Dockerfile

@@ -0,0 +1,43 @@
+# Start from a Node.js base image
+FROM node:16-alpine
+
+# Install necessary dependencies for building Python
+RUN apk add --no-cache \
+    git \
+    build-base \
+    openssl-dev \
+    zlib-dev \
+    bzip2-dev \
+    readline-dev \
+    sqlite-dev \
+    xz-dev \
+    tk-dev \
+    libffi-dev \
+    ncurses-dev \
+    linux-headers
+
+# Download, extract, and install Python 3.12
+RUN wget https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tar.xz && \
+    tar -xf Python-3.12.0.tar.xz && \
+    cd Python-3.12.0 && \
+    ./configure --enable-optimizations && \
+    make -j$(nproc) && \
+    make install && \
+    cd .. && \
+    rm -rf Python-3.12.0 Python-3.12.0.tar.xz && \
+    ln -sf /usr/local/bin/python3.12 /usr/local/bin/python && \
+    ln -sf /usr/local/bin/pip3.12 /usr/local/bin/pip
+
+# Upgrade pip
+RUN pip3.12 install --upgrade pip
+
+# Add your required files
+ADD packages/pyright-scip/sourcegraph-scip-python-0.6.0.tgz /
+
+RUN npm --prefix /package install
+COPY index.py /
+
+WORKDIR /projects/data
+
+# Set entrypoint
+ENTRYPOINT ["python", "/index.py"]

Dockerfile.autoindex

@@ -0,0 +1,84 @@
+import argparse
+import subprocess
+from pathlib import Path
+import tomllib  # Use tomllib for Python 3.11+, or install toml for older versions
+
+
+def matches_pattern(package, patterns):
+    """Check if a package matches any of the specified patterns."""
+    return any(pattern.lower() in package.lower() for pattern in patterns)
+
+
+def install_package(package_with_constraints):
+    """Attempt to install a package with constraints using pip."""
+    print(f"Installing package: {package_with_constraints}")
+    try:
+        subprocess.run(["pip", "install", package_with_constraints], check=True)
+    except subprocess.CalledProcessError:
+        print(f"Failed to install {package_with_constraints}, continuing...")
+
+
+def extract_dependencies(data, patterns):
+    """Recursively search for dependencies in nested data structures."""
+    if isinstance(data, list):
+        for item in data:
+            # Match package names in a list
+            if isinstance(item, str):
+                package_name = item.split(" ", 1)[0]
+                if matches_pattern(package_name, patterns):
+                    yield item
+    elif isinstance(data, dict):
+        for key, value in data.items():
+            # Recurse into dictionaries
+            yield from extract_dependencies(value, patterns)
+
+
+def process_pyproject_file(pyproject_file, patterns):
+    """Process a pyproject.toml file and install matching packages."""
+    print(f"Processing pyproject.toml: {pyproject_file}")
+    with open(pyproject_file, "rb") as file:
+        pyproject_data = tomllib.load(file)
+
+    # Extract all dependencies recursively
+    for dependency in extract_dependencies(pyproject_data, patterns):
+        install_package(dependency)
+
+
+def process_requirements_file(req_file, patterns):
+    """Process a requirements file and install matching packages."""
+    print(f"Processing file: {req_file}")
+    with open(req_file, "r") as file:
+        for line in file:
+            package = line.strip()
+            # Ignore comments and empty lines
+            if not package or package.startswith("#"):
+                continue
+            # Install only if package matches any of the patterns
+            if matches_pattern(package, patterns):
+                install_package(package)
+
+
+def main(index_name, patterns):
+    """Main function to process files and run SCIP indexing command."""
+    # Process requirements-like files
+    for req_file in Path(".").rglob("requirements*.txt"):
+        process_requirements_file(req_file, patterns)
+
+    # Process pyproject.toml files
+    for pyproject_file in Path(".").rglob("pyproject.toml"):
+        process_pyproject_file(pyproject_file, patterns)
+
+    # Run the SCIP indexing command
+    print("Running SCIP indexing command...")
+    subprocess.run(["node", "/package/index.js", "index", ".", 
+                    "--project-version=0.1.0", f"--output={index_name}"], check=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Install specific packages from requirements and pyproject.toml files")
+    parser.add_argument("index_name", help="The index name for SCIP indexing command")
+    parser.add_argument("patterns", nargs="+", help="Patterns to match package names (e.g., 'flask')")
+    args = parser.parse_args()
+
+    # Run main with index name and patterns
+    main(args.index_name, args.patterns)

index-old.py

@@ -0,0 +1,147 @@
+import argparse
+import subprocess
+import os
+import venv
+from pathlib import Path
+import tomllib  # Use tomllib for Python 3.11+, or install toml for older versions
+
+
+def matches_pattern(package, patterns):
+    """Check if a package matches any of the specified patterns."""
+    return any(pattern.lower() in package.lower() for pattern in patterns)
+
+
+
+def setup_virtual_environment(project_path):
+    """Create a virtual environment in the specified directory."""
+    venv_path = project_path / '.venv'
+    print(f"Creating virtual environment at: {venv_path}")
+    venv.create(venv_path, with_pip=True)
+    # subprocess.run([sys.executable, '-m', 'venv', str(venv_path)], check=True)
+    return venv_path
+
+def activate_virtual_environment(venv_path):
+    """Activate the virtual environment by adjusting the environment variables."""
+    venv_bin = venv_path / 'bin'
+    os.environ['VIRTUAL_ENV'] = str(venv_path)
+    os.environ['PATH'] = f"{venv_bin}:{os.environ['PATH']}"
+    print(f"Virtual environment activated: {os.environ['VIRTUAL_ENV']}")
+    print(f"Updated PATH: {os.environ['PATH']}")
+    return venv_bin / 'python'
+
+def install_package(package_with_constraints, venv_python):
+    """Attempt to install a package with constraints using pip within the virtual environment."""
+    print(f"Installing package: {package_with_constraints}")
+    try:
+        result = subprocess.run(
+            [venv_python, "-m", "pip", "install", package_with_constraints],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        print(result.stdout)
+        print(result.stderr)
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to install {package_with_constraints}. Error: {e.stderr}")
+
+
+def extract_dependencies(data, patterns):
+    """Recursively search for dependencies in nested data structures."""
+    if isinstance(data, list):
+        for item in data:
+            # Match package names in a list
+            if isinstance(item, str):
+                if matches_pattern(item, patterns):
+                    yield item
+    elif isinstance(data, dict):
+        for key, value in data.items():
+            yield from extract_dependencies(key, patterns)
+            # Recurse into dictionaries
+            yield from extract_dependencies(value, patterns)
+    elif isinstance(data, str):
+        if matches_pattern(data, patterns):
+            yield data
+
+
+def process_pyproject_file(pyproject_file, patterns, venv_python):
+    """Process a pyproject.toml file and install matching packages."""
+    print(f"Processing pyproject.toml: {pyproject_file}")
+    with open(pyproject_file, "rb") as file:
+        pyproject_data = tomllib.load(file)
+
+    # Extract all dependencies recursively
+    for dependency in extract_dependencies(pyproject_data, patterns):
+        install_package(dependency, venv_python)
+
+
+def process_requirements_file(req_file, patterns, venv_python):
+    """Process a requirements file and install matching packages."""
+    print(f"Processing file: {req_file}")
+    with open(req_file, "r") as file:
+        for line in file:
+            package = line.strip()
+            # Ignore comments and empty lines
+            if not package or package.startswith("#"):
+                continue
+            # Install only if package matches any of the patterns
+            if matches_pattern(package, patterns):
+                install_package(package, venv_python)
+                
+    
+def process_project(path, patterns):
+    """Process the project: set up venv, install packages, and run Node.js script."""
+    project_path = Path(path).resolve()
+    venv_path = setup_virtual_environment(project_path)
+    venv_python = activate_virtual_environment(venv_path)
+
+    # Copy the current environment and update it for the virtual environment
+    env = os.environ.copy()
+
+    try:
+        # Install a test package or requirement to verify pip functionality
+        print("Verifying pip functionality in the virtual environment...")
+        subprocess.run(
+            [venv_python, "-m", "pip", "--version"],
+            check=True,
+            capture_output=True,
+            env=env,
+        )
+        
+        # Process requirements-like files
+        for req_file in Path(path).rglob("requirements*.txt"):
+            process_requirements_file(req_file, patterns, venv_python)
+
+        # Process pyproject.toml files
+        for pyproject_file in Path(path).rglob("pyproject.toml"):
+            process_pyproject_file(pyproject_file, patterns, venv_python)
+
+        # Run the indexer
+        print("Running SCIP indexing command...")
+        result = subprocess.run(
+            ["node", "/package/index.js", "index", ".", "--project-version=0.1.0", "--output=python.scip"],
+            cwd=path,
+            env=env,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        print("Indexer output:")
+        print(result.stdout)
+        print(result.stderr)
+
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    finally:
+        print("Deactivating virtual environment.")
+        os.environ.pop('VIRTUAL_ENV', None)
+        os.environ['PATH'] = os.environ['PATH'].split(":", 1)[1]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Install specific packages from requirements and pyproject.toml files")
+    parser.add_argument("path", help="Relative path to project")
+    parser.add_argument("patterns", nargs="+", help="Patterns to match package names (e.g., 'flask')")
+    args = parser.parse_args()
+
+    # Run main with index name and patterns
+    process_project(args.path, args.patterns)

index.py

@@ -0,0 +1,3 @@
+
+C
+scip-python0.6.0)file:///Users/oinger/Projects/scip-python 

index.scip

@@ -12,6 +12,7 @@
         "build": "tsc",
         "clean": "shx rm -rf ./dist ./out",
         "test": "jest --forceExit",
+        "typecheck": "tsc --noEmit",
         "test:coverage": "jest --forceExit --reporters=jest-junit --reporters=default --coverage --coverageReporters=cobertura --coverageReporters=html --coverageReporters=json"
     },
     "dependencies": {

packages/pyright-internal/package-lock.json

@@ -0,0 +1,22 @@
+MIT License
+
+Pyright - A static type checker for the Python language
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE