File store operations, refactor and ruff fixes

.github/workflows/python-app.yml

@@ -30,11 +30,13 @@ jobs:
 
     - name: Test with pytest
       run: |
-        uv run pytest --ignore=submit_ce/implementations/pubsub/tests \
-        --cov=submit_ce --cov-report=term-missing --cov-fail-under=80
+        # Much of the coverage settings are in pyproject.toml
+        uv run pytest \
+        --cov=submit_ce --cov-report=term-missing \
+        --ignore=submit_ce/implementations/pubsub
 
     - name: Ruff lint check
       run: |
         uv pip install ruff
         uv run ruff check --output-format=github submit_ce
-      continue-on-error: true
+      continue-on-error: true

.gitignore

@@ -246,3 +246,4 @@ legacy.db
 \#*\#
 .\#*
 .aider*
+/.agent-shell/

README.md

@@ -12,24 +12,32 @@ sudo apt-get install cmake libprotobuf-dev protobuf-compiler
 #   $ brew search protobuf
 #   protobuf@21 ✔ (deprecated)
 
-# this uses uv instead of pipenv or poetry
 uv sync
 
-# make sqlite dev db
-python submit_ce/make_test_db.py bootstrap_db
-
+uv run python submit_ce/make_test_db.py bootstrap_db
 # this will give you an Authorization token, save that and use a browser extension
 # like modheader to add Authorization=eyJhb...
 
-flask --app submit_ce.ui.factory:create_web_app run
+uv run flask --app submit_ce.ui.factory:create_web_app run -p 8000
+open http://localhost:8000
 # mac notes: pick another port if 5000 used for music:
-#   flask --app submit_ce.ui.factory:create_web_app run -p 5000
-
-google-chrome localhost:5000
-open localhost:8001
+#   uv run flask --app submit_ce.ui.factory:create_web_app run -p 5000
 
 ```
 
+On the mac:
+```
+```bash
+# mac notes: don't use a version of protobuf greater than 21:
+brew search protobuf
+brew install protobuf@21
+uv sync
+uv run python submit_ce/make_test_db.py bootstrap_db
+# this will give you an Authorization token, save that and use a browser extension
+# like modheader to add Authorization=eyJhb...
+uv run flask --app submit_ce.ui.factory:create_web_app run -p 8000
+open http://localhost:8000
+```
 
 ## Build Docker Image
 

lint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+uv pip install ruff
+uv run ruff check --output-format=github submit_ce

pyproject.toml

@@ -103,34 +103,9 @@ dependencies = [
     "werkzeug==3.0.4",
     "wrapt==1.16.0",
     "wtforms==3.1.2",
+    "yarl>=1.23.0",
 ]
 
-[tool.black]
-line-length = 88
-exclude = '''
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-  )/
-)
-'''
-
-[tool.isort]
-profile = "black"
-skip = [
-    '.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox',
-    '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv',
-]
-skip_gitignore = true
 
 [tool.uv.sources]
 arxiv-base = { git = "https://github.com/arXiv/arxiv-base.git", branch = "develop" }
@@ -193,22 +168,19 @@ relative_files = true
 source = ["submit_ce"]
 omit = [
     # Ignore tests
+    "**/conftest.py",
     "**/tests/**",
     "**/test_*.py",
     "**/*_test.py",
 
     # Ignore questionable code or code in the process of reimplementation
     # Fastly code not currently used
-    "submit_ce/fastapi/test_*.py",
-    "submit_ce/fastapi/auth.py",
-    "submit_ce/fastapi/routes.py",
+    "submit_ce/fastapi",
     #
     "submit_ce/implementations/compile/common.py",
     "submit_ce/implementations/compile/compile_at_gcp.py", # migrate existing tests or eliminate script
     "submit_ce/implementations/file_store/gs_file_store.py",
     "submit_ce/implementations/file_store/legacy_file_store.py",
-    "submit_ce/implementations/legacy_implementation/**",
-    "submit_ce/implementations/pubsub/**",  # disabled - omit until feature is enabled
     "submit_ce/ui/filters/**", # need rewrite
     "submit_ce/ui/compile_sass.py",
     "submit_ce/ui/controllers/new/create.py",
@@ -222,7 +194,8 @@ omit = [
 ]
 
 [tool.coverage.report]
-
+fail_under = 80
+skip_covered = false
 omit = [
     # Ignore tests
     "**/tests/**",
@@ -232,15 +205,14 @@ omit = [
 
     # Ignore questionable code or code in the process of reimplementation
     # Fastly code not currently used
-    "submit_ce/fastapi/test_*.py",
-    "submit_ce/fastapi/auth.py",
-    "submit_ce/fastapi/routes.py",
+    "submit_ce/fastapi",
     "submit_ce/implementations/compile/common.py", # migrate existing tests or eliminate script
     "submit_ce/implementations/compile/compile_at_gcp.py", # migrate existing tests or eliminate script
     "submit_ce/implementations/file_store/legacy_file_store.py", # add tests
     "submit_ce/implementations/file_store/gs_file_store.py", # add tests
-    "submit_ce/implementations/legacy_implementation/**",
-    "submit_ce/implementations/pubsub/**",  # disabled - omit until feature is enabled
+    "submit_ce/implementations/legacy_implementation/interpolate.py",
+    "submit_ce/implementations/legacy_implementation/patch.py",
+    "submit_ce/implementations/legacy_implementation/util.py",
     "submit_ce/ui/filters/**", # need rewrite
     "submit_ce/ui/compile_sass.py",
     "submit_ce/ui/controllers/new/create.py",
@@ -267,10 +239,9 @@ exclude_also = [
     ]
 
 
-# Fail threshold (tune after you see the new baseline)
-fail_under = 80
-skip_covered = true
-
+[tool.ruff]
+target-version="py311"
+exclude = [
+    ".git",
+]
 
-[tool.pytest.ini_options]
-addopts = "-q --maxfail=1 -ra --cov=submit_ce --cov-report=term-missing"

submit_ce/api/CompileService.py

@@ -1,13 +1,16 @@
 """API for CompileService."""
+
+from __future__ import annotations
 from abc import ABC, abstractmethod
-from typing import Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
-from submit_ce.api import Submission, User, Client
-from submit_ce.api.domain.event.process import Result
-from submit_ce.api.domain.process import ProcessStatus
 
 if TYPE_CHECKING:
-    from submit_ce.api import SubmitApi
+    from submit_ce.api.submit import SubmitApi
+    from submit_ce.domain import Submission, User, Client
+    from submit_ce.domain.event.process import Result
+    from submit_ce.domain.process import ProcessStatus
+
 
 class CompileService(ABC):
     """
@@ -16,7 +19,7 @@ class CompileService(ABC):
 
     @abstractmethod
     def start_compile(self, submission: Submission, user: User, client: Client,
-                      api: 'SubmitApi',
+                      api: SubmitApi,
                       source_package_id: Optional[str] = None) -> Result:
         """
         Start a compile process.
@@ -53,4 +56,4 @@ def check(self, process_id: str, user: User, client: Client) -> ProcessStatus:
         Returns
         -------
         ProcessStatus
-        """
+        """

submit_ce/api/__init__.py

@@ -1,10 +1,12 @@
 """Core persistence methods for submissions and submission events."""
 
-from .domain import (Event,
-                     Submission,
-                     License,
-                     User, Client, PublicUser, StaffUser, System, ServiceAgent, agent_factory,
-                     HttpClient, InternalClient, user_from_session,
-                     Upload)
-from .file_store import SubmissionFileStore, SubmitFile
-from .submit import SubmitApi, SubmitFile
+
+from .file_store import SubmissionFileStore
+from .submit import SubmitApi
+from .CompileService import CompileService
+
+__all__ = [
+    SubmissionFileStore,
+    SubmitApi,
+    CompileService
+]