Merge pull request #245 from uw-it-aca/django-upgrade-new-2

Django upgrade new 2

Author: abztrakt

.github/workflows/cicd.yml

@@ -195,7 +195,7 @@ jobs:
           commit_hash: ${{ needs.context.outputs.commit_hash }}
           git_repo_branch: ${{ needs.context.outputs.git_repo_branch }}
           gh_auth_token: ${{ secrets.GH_AUTH_TOKEN }}
-          app_instance: eval
+          app_instance: dev
 
       - name: 'Surface context from executed build step'
         id: context

.gitignore

@@ -30,6 +30,9 @@ spotseeker_web
 # IDE files
 .idea/
 
+# vscode files
+.vscode/
+
 # pipenv
 Pipfile
 Pipfile.lock

Dockerfile

@@ -1,4 +1,4 @@
-ARG DJANGO_CONTAINER_VERSION=1.3.8
+ARG DJANGO_CONTAINER_VERSION=1.4.1
 
 FROM gcr.io/uwit-mci-axdd/django-container:${DJANGO_CONTAINER_VERSION} as app-container
 

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - "${PORT:-8000}:8000"
     environment:
       ENV: localdev
-      AUTH_MODULE: ${AUTH_MODULE:-all_ok}
+      SPOTSEEKER_OAUTH_ENABLED: ${SPOTSEEKER_OAUTH_ENABLED}
       SPOTSEEKER_AUTH_ADMINS: ${SPOTSEEKER_AUTH_ADMINS}
       SPOTSEEKER_WEB_SERVER_HOST: ${SPOTSEEKER_WEB_SERVER_HOST}
       SPOTSEEKER_WEB_OAUTH_KEY: ${SPOTSEEKER_WEB_OAUTH_KEY}

docker/app_start.sh

@@ -1,10 +1,8 @@
 #!/bin/bash
 
-if [ "$ENV" = "localdev" ]
-then
+if [ "$ENV" = "localdev" ]; then
 
-  python manage.py migrate
-  python manage.py loaddata uw.json
-  python manage.py loaddata dummy_oauth.json
+	python manage.py migrate
+	python manage.py loaddata uw.json
 
 fi

docker/dev-values.yml

@@ -0,0 +1,156 @@
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 1
+ingress:
+  enabled: true
+  tls:
+    spotseeker:
+      secretName: dev.api.scout.uw.edu-ingress-cert
+      hosts:
+        - dev.api.scout.uw.edu
+  hosts:
+    spotseeker:
+      host: dev.api.scout.uw.edu
+      paths:
+        - "/"
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    nginx.ingress.kubernetes.io/client-body-buffer-size: "16K"
+    nginx.ingress.kubernetes.io/proxy-body-size: "20m"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "500"
+    nginx.ingress.kubernetes.io/ssl-ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256"
+lifecycle:
+  enabled: true
+  preStop:
+    enabled: true
+affinity:
+   podsSpanNodes: true
+readiness:
+  enabled: true
+securityPolicy:
+  enabled: true
+externalService:
+  enabled: true
+  name: spotseeker-dev-db-service
+  type: ClusterIP
+  serviceAddress: 172.18.0.196
+  servicePort: 3306
+database:
+  engine: mysql
+  name: api_dev
+  hostname: spotseeker-dev-db-service
+  secretName: dev.api.scout.uw.edu-sql-secrets
+repo: spotseeker
+instance: dev
+image:
+  repository: gcr.io/uwit-mci-axdd/spotseeker
+  tag: IMAGE_TAG
+memcached:
+  enabled: true
+  replicaCount: 1
+  updateStrategy:
+    type: RollingUpdate
+gcsCredentials:
+  mounted: true
+  secretName: dev.api.scout.uw.edu-gc-service-credentials
+cronjob:
+  enabled: false
+  jobs:
+    - name: sync-techloan
+      schedule: "0,15,30,45 * * * *"
+      command: ["/scripts/management_command.sh"]
+      args: ["sync_techloan"]
+daemon:
+  enabled: false  # cronjob is probably more appropriate
+  daemons:
+    - name: sync-techloan
+      replicaCount: 1
+      command: ["/scripts/management_daemon.sh"]
+      args: ["--delay", "300", "sync_techloan"]
+environmentVariables:
+  - name: AUTH_MODULE
+    value: oauth
+  - name: CLUSTER_CNAME
+    value: dev.api.scout.uw.edu
+  - name: ENV
+    value: dev
+  - name: WEBSERVER
+    value: nginx
+  - name: CACHE_MAX_ENTRIES
+    value: "1000"
+  - name: CACHE_TIMEOUT
+    value: "86400"
+externalSecrets:
+  enabled: true
+  secrets:
+    - name: dev.api.scout.uw.edu-secrets
+      externalKey: axdd/kv/data/scout/dev-api/secrets
+      data:
+        - name: django-secret
+          property: django-secret
+        - name: spotseeker-auth-admins
+          property: spotseeker-auth-admins
+        - name: storage-bucket-name
+          property: storage-bucket-name
+        - name: storage-project-id
+          property: storage-project-id
+        - name: spotseeker-techloan-url
+          property: spotseeker-techloan-url
+        - name: spotseeker-web-server-host
+          property: spotseeker-web-server-host
+        - name: spotseeker-web-oauth-key
+          property: spotseeker-web-oauth-key
+        - name: spotseeker-web-oauth-secret
+          property: spotseeker-web-oauth-secret
+        - name: spotseeker-web-oauth-user
+          property: spotseeker-web-oauth-user
+    - name: dev.api.scout.uw.edu-sql-secrets
+      externalKey: axdd/kv/data/scout/common/sql-secrets
+      data:
+        - name: username
+          property: username
+        - name: password
+          property: password
+    - name: dev.api.scout.uw.edu-gc-service-credentials
+      externalKey: axdd/kv/data/scout/common/gc-service-credentials
+      data:
+        - name: credentials.json
+          property: credentials.json
+environmentVariablesSecrets:
+  djangoSecret:
+    name: DJANGO_SECRET
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: django-secret
+  spotseekerAuthAdmins:
+    name: SPOTSEEKER_AUTH_ADMINS
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-auth-admins
+  storageBucketName:
+    name: STORAGE_BUCKET_NAME
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: storage-bucket-name
+  storageProjectId:
+    name: STORAGE_PROJECT_ID
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: storage-project-id
+  spotseekerTechloanUrl:
+    name: SPOTSEEKER_TECHLOAN_URL
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-techloan-url
+  spotseekerWebServerHost:
+    name: SPOTSEEKER_WEB_SERVER_HOST
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-web-server-host
+  spotseekerWebOauthKey:
+    name: SPOTSEEKER_WEB_OAUTH_KEY
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-web-oauth-key
+  spotseekerWebOauthSecret:
+    name: SPOTSEEKER_WEB_OAUTH_SECRET
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-web-oauth-secret
+  spotseekerWebOauthUser:
+    name: SPOTSEEKER_WEB_OAUTH_USER
+    secretName: dev.api.scout.uw.edu-secrets
+    secretKey: spotseeker-web-oauth-user

docker/prod-values.yml

@@ -71,8 +71,8 @@ daemon:
       command: ["/scripts/management_daemon.sh"]
       args: ["--delay", "300", "sync_techloan"]
 environmentVariables:
-  - name: AUTH_MODULE
-    value: oauth
+  - name: SPOTSEEKER_OAUTH_ENABLED
+    value: true
   - name: CLUSTER_CNAME
     value: api.scout.uw.edu
   - name: ENV

docker/settings.py

@@ -8,15 +8,36 @@
     DEBUG = False
 
 INSTALLED_APPS += [
-    "oauth_provider",
     "spotseeker_server",
+    "oauth2_provider",
+    "corsheaders",
     "django_extensions",
 ]
 
 MIDDLEWARE += [
     "spotseeker_server.logger.oauth.LogMiddleware",
+    "corsheaders.middleware.CorsMiddleware",
 ]
 
+AUTH_USER_MODEL = "spotseeker_server.Client"
+
+AUTHENTICATION_BACKENDS = (
+    "oauth2_provider.backends.OAuth2Backend",
+    "django.contrib.auth.backends.ModelBackend",
+)
+
+CORS_ORIGIN_ALLOW_ALL = DEBUG
+if os.getenv("ENV", "localdev") == "prod":
+    CORS_ALLOWED_ORIGINS = [
+        "https://scout.uw.edu",
+        "https://manager.scout.uw.edu",
+    ]
+else:
+    CORS_ALLOWED_ORIGINS = [
+        "https://test.scout.uw.edu",
+        "https://test.manager.scout.uw.edu",
+    ]
+
 # django storages settings
 if not DEBUG:
     DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
@@ -28,9 +49,11 @@
         "/gcs/credentials.json"
     )
 
-SPOTSEEKER_AUTH_MODULE = "spotseeker_server.auth.{}".format(
-    os.getenv("AUTH_MODULE", "all_ok")
-)
+SPOTSEEKER_OAUTH_ENABLED = os.getenv("SPOTSEEKER_OAUTH_ENABLED", not DEBUG)
+# convert string to boolean if set in .env
+if type(SPOTSEEKER_OAUTH_ENABLED) == str:
+    SPOTSEEKER_OAUTH_ENABLED = SPOTSEEKER_OAUTH_ENABLED.lower() == "true"
+
 # turn string of auth admins into list
 SPOTSEEKER_AUTH_ADMINS = (
     os.getenv("SPOTSEEKER_AUTH_ADMINS", "").replace(" ", "").split(",")
@@ -60,10 +83,11 @@
         }
     }
 else:
-    # The various MEMCACHED variables are set in django-container's base_settings/common.py
+    # The various MEMCACHED variables are set in django-container's
+    # base_settings/common.py
     CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
-            'LOCATION': MEMCACHED_SERVERS,
+        "default": {
+            "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache",
+            "LOCATION": MEMCACHED_SERVERS,
         }
     }

docker/test-values.yml

@@ -65,8 +65,8 @@ daemon:
       command: ["/scripts/management_daemon.sh"]
       args: ["--delay", "300", "sync_techloan"]
 environmentVariables:
-  - name: AUTH_MODULE
-    value: oauth
+  - name: SPOTSEEKER_OAUTH_ENABLED
+    value: "true"
   - name: CLUSTER_CNAME
     value: test.api.scout.uw.edu
   - name: ENV

docker/urls.py

@@ -1,34 +1,24 @@
-"""docker URL Configuration
+# Copyright 2024 UW-IT, University of Washington
+# SPDX-License-Identifier: Apache-2.0
 
-The `urlpatterns` list routes URLs to views. For more information please see:
-    https://docs.djangoproject.com/en/1.11/topics/http/urls/
-Examples:
-Function views
-    1. Add an import:  from my_app import views
-    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
-Class-based views
-    1. Add an import:  from other_app.views import Home
-    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
-Including another URLconf
-    1. Import the include() function: from django.conf.urls import url, include
-    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
-"""
+from .base_urls import *
 from django.conf import settings
-from django.conf.urls import url, include
+from django.urls import path, include
+from django.contrib import admin
 
-urlpatterns = [
-    url(r'^auth/', include('oauth_provider.urls')),
-    url(r'^api/', include('spotseeker_server.urls')),
-    url(r'^', include('django_prometheus.urls')), # add here for django 1.11 compatibility
+urlpatterns += [
+    path("api/", include("spotseeker_server.urls")),
+    path(
+        "auth/", include("oauth2_provider.urls", namespace="oauth2_provider")
+    ),
 ]
 
 if settings.DEBUG:
-    from django.contrib import admin
     from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 
     urlpatterns += [
-        url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
-        url(r'^admin/', include(admin.site.urls)),
+        path("admin/doc/", include("django.contrib.admindocs.urls")),
+        path("admin/", admin.site.urls),
     ]
 
     urlpatterns += staticfiles_urlpatterns()

requirements.txt

@@ -1,4 +1,5 @@
 # these two can be upgraded when django gets an update
 mysqlclient~=1.3
-django-prometheus~=2.0
+# this is temporary for testing migrations in upgrade
+django-extensions==2.2.*
 -e .

spotseeker_server/management/commands/resources/building3.jpg resources/building3.jpg

@@ -3,11 +3,13 @@
 # Port to connect to with your browser
 #PORT=8001
 
-# Whether to use all_ok (default) or oauth auth module
-#AUTH_MODULE=oauth
-
 # Comma separated list of admins
 #SPOTSEEKER_AUTH_ADMINS=javerage,demo_user
 
+# enabled oauth (True or False)
+#SPOTSEEKER_OAUTH_ENABLED=True
+
 # URL to sync techloan from sync_techloan management command
-#SPOTSEEKER_TECHLOAN_URL=https://EXAMPLE_DOMAIN/techloan/api/v2/type/?embed=availability&embed=class
+#SPOTSEEKER_TECHLOAN_URL=https://EXAMPLE_DOMAIN/techloan/api/v2/type/?embed=availability&embed=class
+
+#ENV=localdev

spotseeker_server/management/commands/resources/building4.jpg resources/building4.jpg

@@ -7,18 +7,20 @@
     version="1.0",
     description="REST Backend for SpaceScout",
     install_requires=[
-        "Django==1.11.*",
+        "Django==2.2.*",
         "future",
         "mock<=1.0.1",
         "oauthlib==3.1.*",
+        "django-oauth-toolkit",
+        "django-cors-headers==3.10.*",
         "requests==2.26.*",
         "requests-oauthlib==1.3.*",
         "Pillow",
         "pylibmc",  # switch to what django-container uses upon upgrade
         "pyproj",
         "pytz",
         "simplejson>=2.1",
-        "django-oauth-plus@git+https://github.com/edx-unsupported/django-oauth-plus#egg=2.2.9.edx-4",
+        # "django-oauth-plus@git+https://github.com/abztrakt/django-oauth-plus@spotseeker-changes#egg=2.2.9.edx-4",
         "django-storages[google]",
         "schema",
         "six",

spotseeker_server/management/commands/resources/building5.jpg resources/building5.jpg

@@ -1,4 +1,4 @@
-# Copyright 2023 UW-IT, University of Washington
+# Copyright 2024 UW-IT, University of Washington
 # SPDX-License-Identifier: Apache-2.0
 
 __version__ = "1.0"