SUBMISSION-79: Initial Use of Author's Work (was License) UI updates.

.github/workflows/python-app.yml

@@ -0,0 +1,42 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    branches: [ "develop" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+
+    - name: Install protobuf compiler
+      run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+    - name: Install the project
+      run: uv sync --locked --all-extras --dev
+
+    - name: Test with pytest
+      run: |
+        # Much of the coverage settings are in pyproject.toml
+        uv run pytest \
+        --cov=submit_ce --cov-report=term-missing \
+        --ignore=submit_ce/implementations/pubsub
+
+    - name: Ruff lint check
+      run: |
+        uv pip install ruff
+        uv run ruff check --output-format=github submit_ce
+      continue-on-error: true

.gitignore

@@ -246,3 +246,4 @@ legacy.db
 \#*\#
 .\#*
 .aider*
+/.agent-shell/

CHANGELOG.md

@@ -11,7 +11,7 @@ There is a generated client in `client`. It can be built with `clitools.py`. I'm
 starting to wire up the UI to use that.
 
 I took the NG submission-core and submission-ui and put them in the same project
-and then refactored the pacakges. 
+and then refactored the packages. 
 
 # 2024-09-24 Start of UI
 Brian Caruso
@@ -73,8 +73,8 @@ without flask. This will need to be merged to arxiv-base master soon.
 ### Current state
 *WORKING*
 - creates a submission
-- uplaod and unpacks tar.gz
-- license, policy and author attestation
+- upload and unpacks tar.gz
+- submission agreement, and license acceptance
 - metadata: title abstract etc
 - docker file
 - tests

Dockerfile

@@ -16,7 +16,11 @@ ENV PYTHONFAULTHANDLER=1 \
     PATH="/usr/sbin:/usr/local/bin:/usr/bin:/bin"
 
 RUN apt-get -q update && apt-get -y -q upgrade && \
-    apt-get -y install default-libmysqlclient-dev
+    apt-get -y install default-libmysqlclient-dev \
+                       cmake \
+                       libprotobuf-dev \
+                       protobuf-compiler
+
 
 RUN useradd --create-home e-prints
 USER e-prints

README.md

@@ -6,26 +6,38 @@ arXiv paper submission system
 To run the server, please execute the following from the root directory:
 
 ```bash
-# setup venv in your preferred way
-python --version
-# 3.11
+# Install gcld3 dependencies needed by arxiv-base metadata checks
+sudo apt-get install cmake libprotobuf-dev protobuf-compiler
+# mac notes: don't use a version greater than:
+#   $ brew search protobuf
+#   protobuf@21 ✔ (deprecated)
 
-# this uses uv instead of pipenv or poetry
-# see https://docs.astral.sh/uv/
-uv venv
 uv sync
 
-# make sqlite dev db
-python submit_ce/make_test_db.py bootstrap_db
-
+uv run python submit_ce/make_test_db.py bootstrap_db
 # this will give you an Authorization token, save that and use a browser extension
 # like modheader to add Authorization=eyJhb...
 
-flask --app submit_ce.ui.factory:create_web_app run
+uv run flask --app submit_ce.ui.factory:create_web_app run -p 8000
+open http://localhost:8000
+# mac notes: pick another port if 5000 used for music:
+#   uv run flask --app submit_ce.ui.factory:create_web_app run -p 5000
 
-google-chrome localhost:5000
 ```
 
+On the mac:
+```
+```bash
+# mac notes: don't use a version of protobuf greater than 21:
+brew search protobuf
+brew install protobuf@21
+uv sync
+uv run python submit_ce/make_test_db.py bootstrap_db
+# this will give you an Authorization token, save that and use a browser extension
+# like modheader to add Authorization=eyJhb...
+uv run flask --app submit_ce.ui.factory:create_web_app run -p 8000
+open http://localhost:8000
+```
 
 ## Build Docker Image
 
@@ -37,6 +49,15 @@ docker push gcr.io/arxiv-development/submit-ce/submit-ce-ui
 
 ## Tests
 
+Test setup:
+```
+sudo apt-get install google-cloud-cli-pubsub-emulator
+
+# mac notes: gcloud components install pubsub-emulator
+```
+
+See [emulator instructions](https://cloud.google.com/pubsub/docs/emulator) for other operating systems.
+
 To run the tests:
 
 ```bash

cicd/cloudbuild-submit-dev-arxiv.yaml

@@ -1,19 +1,16 @@
 steps:
+
   - name: gcr.io/cloud-builders/docker
     args:
       - build
       - '--no-cache'
-      - '--build-arg'
-      - 'GIT_COMMIT=${GIT_COMMIT}'
+      - '--build-arg=GIT_COMMIT=$COMMIT_SHA'
       - '-t'
       - >-
         $_AR_HOSTNAME/$_AR_PROJECT_ID/$_AR_REPOSITORY/$REPO_NAME/$_SERVICE_NAME:$COMMIT_SHA
       - .
       - '-f'
       - Dockerfile
-    env:
-      - 'GIT_COMMIT=${GIT_COMMIT}'
-    waitFor: ['get-git-sha']
     id: Build
 
   - name: gcr.io/cloud-builders/docker

clitools.py

@@ -1,15 +1,12 @@
 import os
-import tempfile
 import time
 import subprocess
 import random
-from typing import Optional
 
 import fire
 
 def gen_openapi_json(file:str = "openapi.json"):
     """Generate an openapi.yaml file for the current server code."""
-    import uvicorn
     import requests
     port = random.randint(9000, 12000)
     command = f"uvicorn submit_ce.api.app:app --host 127.0.0.1 --port {port}".split()
@@ -34,7 +31,7 @@ def gen_client(gen_spec:bool = True):
     """
 
     if gen_spec:
-        print(f"* Generating to openapi.json for current code")
+        print("* Generating to openapi.json for current code")
         gen_openapi_json()
 
     command = f"""

lint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+uv pip install ruff
+uv run ruff check --output-format=github submit_ce

pyproject.toml

@@ -3,7 +3,7 @@ name = "submit-ce"
 version = "0.1.0"
 description = "Paper submission system."
 readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.12"
 dependencies = [
     "annotated-types==0.7.0",
     "anyio==4.5.0",
@@ -27,8 +27,9 @@ dependencies = [
     "email-validator==2.2.0",
     "fastapi==0.114.2",
     "fastapi-cli==0.0.5",
-    "fire==0.5.0",
+    "fire>=0.5",
     "flask==3.0.3",
+    "gcld3>=3.0.13",
     "google-cloud-pubsub>=2.29.0",
     "greenlet==3.1.0",
     "gunicorn>=23.0.0",
@@ -59,10 +60,10 @@ dependencies = [
     "pluggy==1.5.0",
     "py==1.11.0",
     "pycparser==2.22",
-    "pydantic==2.9.2",
-    "pydantic-core==2.23.4",
-    "pydantic-extra-types==2.9.0",
-    "pydantic-settings==2.5.2",
+    "pydantic>=2.9",
+    "pydantic-core>=2.23.4",
+    "pydantic-extra-types>=2.9.0",
+    "pydantic-settings>=2.5.2",
     "pydocstyle==3.0.0",
     "pygments==2.18.0",
     "pyjwt==2.9.0",
@@ -102,37 +103,12 @@ dependencies = [
     "werkzeug==3.0.4",
     "wrapt==1.16.0",
     "wtforms==3.1.2",
+    "yarl>=1.23.0",
 ]
 
-[tool.black]
-line-length = 88
-exclude = '''
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-  )/
-)
-'''
-
-[tool.isort]
-profile = "black"
-skip = [
-    '.eggs', '.git', '.hg', '.mypy_cache', '.nox', '.pants.d', '.tox',
-    '.venv', '_build', 'buck-out', 'build', 'dist', 'node_modules', 'venv',
-]
-skip_gitignore = true
 
 [tool.uv.sources]
-arxiv-base = { git = "https://github.com/arXiv/arxiv-base.git", rev = "11e1c636f1db900a279d48f28f3a7a37ab6765a7" }
+arxiv-base = { git = "https://github.com/arXiv/arxiv-base.git", branch = "develop" }
 
 [dependency-groups]
 dev = [
@@ -188,12 +164,84 @@ dev = [
 
 [tool.coverage.run]
 branch = true
+relative_files = true
 source = ["submit_ce"]
-omit = [ "*/tests/*" ]
+omit = [
+    # Ignore tests
+    "**/conftest.py",
+    "**/tests/**",
+    "**/test_*.py",
+    "**/*_test.py",
+
+    # Ignore questionable code or code in the process of reimplementation
+    # Fastly code not currently used
+    "submit_ce/fastapi",
+    #
+    "submit_ce/implementations/compile/common.py",
+    "submit_ce/implementations/compile/compile_at_gcp.py", # migrate existing tests or eliminate script
+    "submit_ce/implementations/file_store/gs_file_store.py",
+    "submit_ce/implementations/file_store/legacy_file_store.py",
+    "submit_ce/ui/filters/**", # need rewrite
+    "submit_ce/ui/compile_sass.py",
+    "submit_ce/ui/controllers/new/create.py",
+    "submit_ce/ui/controllers/cross.py", # add later
+    "submit_ce/ui/controllers/delete.py", # add tests after reimplementation
+    "submit_ce/ui/controllers/new/process.py",
+    'submit_ce/ui/controllers/new/review.py', # placeholder
+    "submit_ce/ui/controllers/new/upload.py", # add tests after reimplementation
+    "submit_ce/ui/controllers/new/upload_delete.py"
+
+]
 
 [tool.coverage.report]
+fail_under = 80
+skip_covered = false
+omit = [
+    # Ignore tests
+    "**/tests/**",
+    "**/test_*.py",
+    "**/*_test.py",
+
+
+    # Ignore questionable code or code in the process of reimplementation
+    # Fastly code not currently used
+    "submit_ce/fastapi",
+    "submit_ce/implementations/compile/common.py", # migrate existing tests or eliminate script
+    "submit_ce/implementations/compile/compile_at_gcp.py", # migrate existing tests or eliminate script
+    "submit_ce/implementations/file_store/legacy_file_store.py", # add tests
+    "submit_ce/implementations/file_store/gs_file_store.py", # add tests
+    "submit_ce/implementations/legacy_implementation/interpolate.py",
+    "submit_ce/implementations/legacy_implementation/patch.py",
+    "submit_ce/implementations/legacy_implementation/util.py",
+    "submit_ce/ui/filters/**", # need rewrite
+    "submit_ce/ui/compile_sass.py",
+    "submit_ce/ui/controllers/new/create.py",
+    "submit_ce/ui/controllers/cross.py",
+    "submit_ce/ui/controllers/delete.py",  # add tests after reimplementation
+    "submit_ce/ui/controllers/new/process.py",
+    "submit_ce/ui/controllers/new/review.py", # placeholder
+    "submit_ce/ui/controllers/new/upload.py", # add tests after reimplementation
+    "submit_ce/ui/controllers/new/upload_delete.py"
+]
+
+exclude_lines = [
+  "pragma: no cover",
+  "if TYPE_CHECKING:",
+  "if __name__ == .__main__.:",
+  "raise NotImplementedError",
+  "except ImportError",
+]
+
 # Regexes for lines to exclude from consideration
 exclude_also = [
     # Don't complain about abstract methods, they aren't run:
     "@(abc\\.)?abstractmethod",
     ]
+
+
+[tool.ruff]
+target-version="py311"
+exclude = [
+    ".git",
+]
+

submit_ce/api/__init__.py

@@ -1,10 +1,12 @@
 """Core persistence methods for submissions and submission events."""
 
-from .domain import (Event,
-                     Submission,
-                     License,
-                     User, Client, PublicUser, StaffUser, System, ServiceAgent, agent_factory,
-                     HttpClient, InternalClient, user_from_session,
-                     Upload)
-from .file_store import SubmissionFileStore, SubmitFile
-from .submit import SubmitApi, SubmitFile
+
+from .file_store import SubmissionFileStore
+from .submit import SubmitApi
+from .compile_service import CompileService
+
+__all__ = [
+    SubmissionFileStore,
+    SubmitApi,
+    CompileService
+]