update backend

Author: maltaesousa

.env.sample

@@ -0,0 +1,60 @@
+# SITN specific stuff
+PGPOSTGRESPASSWORD=password
+DOCKER_HOST="ssh://[email protected]"
+DOCKER_BACK_PORT=5004
+DOCKER_FRONT_PORT=5005
+DOCKER_BASE=sitn/geoshop-local
+COMPOSE_PROJECT_NAME=geoshop-local
+API_BASE_URL=https://sitn.ne.ch
+
+# Do not set this to True if your server is exposed to internet!
+DEBUG=False
+
+PGDATABASE=geoshop
+PGUSER=geoshop
+PGHOST=host.docker.internal
+PGPORT=5432
+PGPASSWORD=geoshop
+PGSCHEMA=geoshop
+
+# The folder where the geodata will be uploaded
+MEDIA_ROOT=./files
+# A link to public images used in product and metadata
+MEDIA_URL=https://example.com/geoshop_media/
+
+ALLOWED_HOST=localhost,127.0.0.1,proxy.example.com
+# Base URL of the api
+ROOTURL=
+
+DEFAULT_FROM_EMAIL=[email protected]
+EMAIL_HOST=smtp.example.com
+ADMIN_USERNAME=admin
+ADMIN_EMAIL_LIST=[email protected]
+
+# This is used for CORS and to generate links in email templates
+FRONT_URL=localhost:4200
+FRONT_HREF=/geoshop
+FRONT_PROTOCOL=http
+CSRF_COOKIE_DOMAIN=localhost
+LOGGING_LEVEL=INFO
+
+# Generate this key with Django tools
+SECRET_KEY=GenerateOneSecretKey
+ADMIN_PASSWORD=Test1234
+EXTRACT_USER_PASSWORD=Test1234
+
+AUTO_LEGEND_URL=http://example.com/mapserv?FORMAT=image%2Fpng&TRANSPARENT=true&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetLegendGraphic&LAYER=
+# If you plan to manage secret metadata, provide your intranet legend service
+INTRA_LEGEND_URL=https://example.com/mapserv_proxy?ogcserver=source+for+image%2Fpng&FORMAT=image%2Fpng&TRANSPARENT=true&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetLegendGraphic&LAYER=
+
+# Only for testing purpose
+EMAIL_TEST_TO=[email protected]
+
+DEFAULT_LANGUAGE=en
+# For non-docker, uncomment this (only tested in Windows)
+#GDAL_PATH="C:\Applications\Mapserver72_x64\bin"
+#GDAL_LIBRARY_PATH="C:\Applications\Mapserver72_x64\bin\gdal300.dll"
+#GEOS_LIBRARY_PATH="C:\Applications\Mapserver72_x64\bin\geos_c.dll"
+
+# Geometric settings 
+DEFAULT_SRID=2056

.gitignore

@@ -7,3 +7,13 @@ logs/*
 *.pyc
 
 front/debug.log
+
+.env
+.env.local
+.env.dev
+.env.prod
+.env.prepub
+
+.venv
+Pipfile
+Pipfile.lock

README.md

@@ -7,94 +7,62 @@
 
 ## Getting started
 
-Fork and clone this repository. Make a copy of the `.env` file and adapt it to your environment settings:
+Fork and clone this repository. Make a copy of the `.env.sample` and `back/default_settings.py` files and adapt it to your environment settings:
 
 ```powershell
 git submodule init
 git submodule update
-cd back
 cp .env.sample .env
-cp default_settings.py settings.py
-cd ..
+cp back/default_settings.py back/settings.py
+cp scripts/custom.js back/api/templates/gis/admin
 ```
 
 ### Database
 
 The best is to backup and restore a production db:
 
-```sql
-.\scripts\sitn\1_fetch_prod_db.ps1
+```powershell
+.\scripts\1_fetch_prod_db.ps1
 ```
 
 Now that the database is ready, you can start backend either with Docker or not.
 
 ### Backend with docker
 
-To ease debug, you can add `DEBUG=True` to your `env.local` file. Never add this to a container exposed on internet.
-
-Build backend image:
-
-```powershell
-cd back
-docker build -t sitn/geoshop-dev-api --build-arg ENV_FILE=.env.local .
-```
-
-If you started with an empty database, run this once in order to create admin account:
-
-```powershell
-docker run --rm --name geoshop --env-file=.env.local sitn/geoshop-dev-api python manage.py fixturize
-```
+To ease debug, you can add `DEBUG=True` to your `.env` file. Never add this to a container exposed on internet.
 
-Now you can run it with:
+Run de docker composition with the following command and type `local` when asked:
 
 ```powershell
-docker run -d --rm --name geoshop --env-file=.env.local -p 8000:8000 -v ${PWD}:/app/geoshop_back sitn/geoshop-dev-api gunicorn --reload wsgi -b :8000
+python deploy.py
 ```
 
-You should be able to visit the API at http://localhost:8000.
+You should be able to visit the API at http://localhost:5004 or another port if you changed it.
 
 Run tests:
 
 ```powershell
-docker run --rm --env-file=.env.local -v ${PWD}:/app/geoshop_back sitn/geoshop-dev-api python manage.py test api
-```
-
-Make messages for translation:
-
-```powershell
-docker run --rm --env-file=.env.local -v ${PWD}:/app/geoshop_back:rw sitn/geoshop-dev-api python manage.py makemessages -l fr
-```
-
-Stop the server:
-```powershell
-docker stop geoshop
+docker compose exec api python manage.py test api
 ```
 
 ### Backend without docker on Windows
 
 If not using docker, additionnal packages are required:
 
 * pipenv (pip install pipenv)
-* GDAL 2.4 (see instructions below to install it in your venv)
+* GDAL (see instructions below to install it in your venv)
 
-Install the app. If you want your `venv` to be inside your project directory, you need to set `PIPENV_VENV_IN_PROJECT` environment variable, otherwise it'll go to your profile, if you want `DEBUG` to be enabled, change it in `settings.py` file but never commit it with debug enabled:
+Install the app. If you want your `venv` to be inside your project directory, you need to set `PIPENV_VENV_IN_PROJECT` environment variable, otherwise it'll go to your profile
 
 ```powershell
-cd back
 $env:PIPENV_VENV_IN_PROJECT="True"
-pipenv install --dev           # installs everything needed
-pipenv shell                   # activates venv and reads .env file
+pipenv install --dev -r back/requirements.txt
+pipenv shell # activates venv and reads .env file
 ```
 
 #### Installing GDAL on Windows, only once per machine
-Download the GDAL 2.4 wheel (3.X isn't supported yet by Django) on this page: https://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal. For example, if you have Python 3.6 64bit, choose `GDAL‑2.4.1‑cp36‑cp36m‑win_amd64.whl`.
-Then install it weather system wide or in your venv (the example below will show the venv variant and expects you have your venv activated):
 
-```powershell
-pip install path\to\your\GDAL-2.4XXXX.whl
-```
-
-You'll then need to add GDAL dll to your PATH if you installed it system wide. You can get the dll path with:
+You'll need to add GDAL dll to your PATH if you installed it system wide. You can get the dll path with:
 
 ```python
 python
@@ -122,21 +90,17 @@ Your database should be ready, now you can run the backend:
 python manage.py runserver
 ```
 
-Translations can be generated, static files collected with:
+Translations can be generated and static files collected with:
 
 ```powershell
 python manage.py compilemessages --locale=fr
-python .\manage.py collectstatic
+python manage.py collectstatic
 ```
 
-Then, set `DEBUG` to `True` in `back/settings.py` and you can finally run the server:
-
 ```powershell
 python manage.py runserver
 ```
 
-> :warning: **DO NOT COMMIT settings.py with `DEBUG` set to `True` !**
-
 ### Run tests
 
 ```powershell
@@ -185,17 +149,19 @@ Create an `env.prod` file base on `env.sample`.
 
 ### Application deployment
 
+Always:
+1. Run tests locally on docker
+2. Deploy prepub and test it with a fake order
+3. Deploy prod
+
 ```powershell
-python scripts\deploy.py
+python deploy.py
 ```
 
-And choose your instance to be deployed.
-
-Create a scheduled task that runs `scripts/geoshop_clean_orders.ps1` every month.
+And choose your instance to be deployed. You must have `.env.prepub` and `.env.prod` defined at the root of the project.
 
 More info on bookstack
 
-
 ## Upgrading front-end packages
 
 Go to https://update.angular.io/?v=16.0-17.0 and check there are not manual steps to be done.

back

@@ -1,13 +1,10 @@
 import os
-import pathlib
 import datetime
 import subprocess
 import re
 
 dest_config = input('Input "prod", "prepub", "dev" or "local": ')
 
-BASE_DIR = pathlib.Path(__file__).parent.absolute()
-
 def resub_callback(match):
     return os.environ.get(match.group(1), '')
 
@@ -19,36 +16,37 @@ def eval_templates(in_file, out_file):
     with open(out_file, "w", encoding="utf8") as f:
         f.write(file_content)
 
-# Do not deploy to internet with DEBUG set to True
-settings_path = f'{BASE_DIR}/../back/settings.py'
-with open(settings_path, 'r') as f:
-    settings = f.read()
-
-is_debug = re.search(r'^DEBUG = (True)$', settings, re.MULTILINE)
-if is_debug:
-    if dest_config != "local":
-        print("Cannot deploy if DEBUG=True in settings.py")
-        quit()
+env_file = f'.env.{dest_config}'
+if dest_config == "local":
+    env_file = '.env'
 
-env_file = f'{BASE_DIR}/../back/.env.{dest_config}'
 # Read .env
 with open(env_file, 'r') as f:
     for line in f:
         args = line.strip().split('=')
         if args[0] and not args[0].startswith('#'):
             os.environ[args[0].strip('"')] = args[1].strip('"')
 
-os.environ['ENV_FILE'] = f".env.{dest_config}"
+is_debug = os.environ['DEBUG'] == 'True'
+if is_debug:
+    if dest_config != "local":
+        print("Cannot deploy if DEBUG=True for this config")
+        quit()
+
+os.environ['ENV_FILE'] = env_file
 print(f"{str(datetime.datetime.now())} - COMPOSE_PROJECT_NAME IS {os.environ['COMPOSE_PROJECT_NAME']}")
+print(f"{str(datetime.datetime.now())} - ENV_FILE IS {os.environ['ENV_FILE']}")
 
-eval_templates(f'{BASE_DIR}/../front/src/assets/configs/config.json.tmpl', f'{BASE_DIR}/../front/src/assets/configs/config.json')
-eval_templates(f'{BASE_DIR}/../front/httpd.conf.tmpl', f'{BASE_DIR}/../front/httpd.conf')
+eval_templates(f'front/src/assets/configs/config.json.tmpl', f'front/src/assets/configs/config.json')
+eval_templates(f'front/httpd.conf.tmpl', f'front/httpd.conf')
 
 print(f"{str(datetime.datetime.now())} - DOCKER_HOST IS {os.environ['DOCKER_HOST']}")
 
+os.rename('.env', '.env.local')
 subprocess.run(['docker-compose', 'build', 'api'])
 subprocess.run(['docker-compose', 'build', 'front'])
 subprocess.run(['docker-compose', 'down'])
 subprocess.run(['docker-compose', 'up', '-d'])
+os.rename('.env.local', '.env')
 
 print(f"{str(datetime.datetime.now())} - END")

scripts/deploy.py renamed to deploy.py

@@ -5,15 +5,29 @@ services:
     image: ${DOCKER_BASE}-api
     build:
       context: ./back/
-      args:
-        ENV_FILE: ${ENV_FILE}
-    env_file: ./back/${ENV_FILE}
+    env_file: ${ENV_FILE:-.env}
     command: "gunicorn wsgi -b :8000 --timeout 90"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
     restart: unless-stopped
     ports:
       - "${DOCKER_BACK_PORT}:8000"
     volumes:
-      - "${GEOSHOP_DATA}:/mnt/geoshop_data"
+      - "static-files:/app/geoshop_back/static:ro"
+
+  migrate:
+    image: geoshop-api
+    build:
+      context: ./back/
+      dockerfile: Dockerfile
+    env_file: ${ENV_FILE:-.env}
+    command:
+      - bash
+      - -c
+      - "python manage.py migrate && python manage.py collectstatic --noinput && python manage.py compilemessages --locale=fr && python manage.py fixturize"
+    volumes:
+      - "static-files:/app/geoshop_back/static:rw"
 
   front:
     image: ${DOCKER_BASE}-front
@@ -24,3 +38,6 @@ services:
     restart: unless-stopped
     ports:
       - "${DOCKER_FRONT_PORT}:80"
+
+volumes:
+  static-files:

docker-compose.yml

@@ -1,5 +1,5 @@
 $pwd = pwd
-foreach ($line in Get-Content $PSScriptRoot\..\..\back\.env.prod) {
+foreach ($line in Get-Content $PSScriptRoot\..\.env.prod) {
     $args = $line -split "="
     If ($args[0] -And !$args[0].StartsWith("#")) {
         $cmd = '$env:' + $args[0].Trim('"') + '="' + $args[1].Trim('"') + '"'
@@ -22,7 +22,7 @@ Write-Host "$(Get-Date -Format g) PG dump done..."
 
 Write-Host "$(Get-Date -Format g) variables d'environnement remises sur .env.local"
 
-foreach ($line in Get-Content $PSScriptRoot\..\..\back\.env) {
+foreach ($line in Get-Content $PSScriptRoot\..\.env) {
     $args = $line -split "="
     If ($args[0] -And !$args[0].StartsWith("#")) {
         $cmd = '$env:' + $args[0].Trim('"') + '="' + $args[1].Trim('"') + '"'

scripts/sitn/1_fetch_prod_db.ps1 renamed to scripts/1_fetch_prod_db.ps1

@@ -0,0 +1,21 @@
+{% extends "gis/admin/openlayers.js" %}
+{% block map_options %}
+var options = {
+    projection: new OpenLayers.Projection('EPSG:2056'),
+    resolutions: [250, 100, 50, 20, 10, 5, 2.5, 2, 1.5, 1, 0.5, 0.25, 0.125, 0.0625],
+    units: 'm',
+    maxExtent: new OpenLayers.Bounds(2420000.0, 1030000.0, 2900000.0, 1360000.0),
+    tileSize: new OpenLayers.Size(256, 256)
+};
+{% endblock %}
+{% block base_layer %}new OpenLayers.Layer.WMTS({
+    name: "WMTS plan_cadastral",
+    url: "https://sitn.ne.ch/mapproxy95/wmts/1.0.0/plan_cadastral/default/{TileMatrixSet}/{TileMatrix}/{TileRow}/{TileCol}.png",
+    layer: 'plan_cadastral',
+    matrixSet: 'EPSG2056',
+    format: 'png',
+    isBaseLayer: true,
+    style: 'default',
+    requestEncoding: 'REST'
+});
+{% endblock %}