From 17efdc30554d33f8e7129e8610e35a5220992f88 Mon Sep 17 00:00:00 2001 From: Antoine D Date: Wed, 24 Oct 2018 17:28:46 +0200 Subject: [PATCH] working version --- .env | 1 - Pipfile | 17 +++++++++ Pipfile.lock | 42 +++++++++++++++++++++ docker-compose.build.yml | 18 +++++++++ docker-compose.run.yml | 18 +++++++++ docker-compose.yml | 33 ++++++++-------- explorer/Dockerfile | 2 - explorer/nginx.vh.default.conf | 11 +++++- explorer/package.json | 1 + explorer/src/vues/routes/data_dashboard.vue | 16 ++++++-- importer/Dockerfile | 2 - readme.md | 22 +++++++++-- run_explorer.sh | 23 ----------- tasks.py | 34 ++++++++--------- 14 files changed, 170 insertions(+), 70 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 docker-compose.build.yml create mode 100644 docker-compose.run.yml delete mode 100755 run_explorer.sh diff --git a/.env b/.env index ab6950d..b26ed1b 100644 --- a/.env +++ b/.env @@ -3,4 +3,3 @@ POSTGRES_USER=cosmogony POSTGRES_PASSWORD=cosmogony POSTGRES_HOST=postgres POSTGRES_PORT=5432 - diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..225be9c --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] + +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[requires] + +python_version = "3.6" + +[dev-packages] + + + +[packages] + +invoke = "*" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..d9f7aee --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,42 @@ +{ + "_meta": { + "hash": { + "sha256": "26e3b6ff3857692bf88328ae719005a1693aadf46a808a0d1a6cedb39d12532c" + }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "0", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "4.13.0-46-generic", + "platform_system": "Linux", + "platform_version": "#51-Ubuntu SMP Tue Jun 12 12:36:29 UTC 2018", + "python_full_version": "2.7.14", + "python_version": "2.7", + "sys_platform": "linux2" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "invoke": { + "hashes": [ + "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e", + "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", + "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7" + ], + "version": "==1.2.0" + } + }, + "develop": {} +} diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 0000000..de2333c --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,18 @@ +version: "3" + +services: + tiles: + build: ./tiles + image: osmwithoutborders/cosmogony-tiles + + api: + build: ./api + image: osmwithoutborders/cosmogony-api + + explorer: + build: ./explorer + image: osmwithoutborders/cosmogony-explorer + + importer: + build: ./importer + image: osmwithoutborders/cosmogony-importer diff --git a/docker-compose.run.yml b/docker-compose.run.yml new file mode 100644 index 0000000..a05fc42 --- /dev/null +++ b/docker-compose.run.yml @@ -0,0 +1,18 @@ +version: "3" + +volumes: + data-dashboard: + +services: + importer: + image: osmwithoutborders/cosmogony-importer + env_file: .env + volumes: + - ${PATH_TO_COSMOGONY_DIR:-./cosmogony_data}:/mnt/data + + data-dashboard: + image: cosmogony-data-dashboard + # image: osmwithoutborders/cosmogony-data-dashboard + volumes: + - ${PATH_TO_COSMOGONY_DIR:-./cosmogony_data}:/mnt/data + - data-dashboard:/mnt/data-dashboard diff --git a/docker-compose.yml b/docker-compose.yml index 97949cd..09acdfa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,36 +1,39 @@ version: "3" + volumes: pgdata: + data-dashboard: + services: postgres: image: "openmaptiles/postgis:2.9" volumes: - pgdata:/var/lib/postgresql/data ports: - - 127.0.0.1:5432:5432 + - 5432 env_file: .env - cosmogony-importer: - build: ./importer - env_file: .env - depends_on: - - postgres - cosmogony-tiles: - build: ./tiles + + tiles: + image: osmwithoutborders/cosmogony-tiles env_file: .env depends_on: - postgres ports: - - 127.0.0.1:6767:6767 - cosmogony-api: - build: ./api + - 6767 + + api: + image: osmwithoutborders/cosmogony-api env_file: .env depends_on: - postgres ports: - - 127.0.0.1:8000:8000 + - 8000 + explorer: - build: ./explorer + image: osmwithoutborders/cosmogony-explorer depends_on: - - cosmogony-api + - api ports: - - 8585:80 + - 8585:80 # TODO remove this + volumes: + - data-dashboard:/mnt/data_dashboard diff --git a/explorer/Dockerfile b/explorer/Dockerfile index b8ed25e..91693b7 100644 --- a/explorer/Dockerfile +++ b/explorer/Dockerfile @@ -7,8 +7,6 @@ RUN cd /tmp/cosmo_build &&\ node ./node_modules/webpack/bin/webpack --mode='production' && \ rm -rf ./node_modules - - FROM nginx:alpine COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf COPY --from=builder /tmp/cosmo_build/ /usr/share/nginx/html/ diff --git a/explorer/nginx.vh.default.conf b/explorer/nginx.vh.default.conf index 3b6172d..6d37585 100644 --- a/explorer/nginx.vh.default.conf +++ b/explorer/nginx.vh.default.conf @@ -16,14 +16,21 @@ server { } location /tiles/ { - proxy_pass http://cosmogony-tiles:6767/; + proxy_pass http://tiles:6767/; } location /api/ { - proxy_pass http://cosmogony-api:8000/; + proxy_pass http://api:8000/; add_header 'Access-Control-Allow-Origin' '*'; } + location /data { + # In the dockerfile we mounted the file generated by https://github.com/osm-without-borders/cosmogony-data-dashboard + # in /mnt/data_dashboard + alias /mnt/data_dashboard; + } + + #error_page 404 /404.html; # redirect server error pages to the static page /50x.html diff --git a/explorer/package.json b/explorer/package.json index b59c473..d780e04 100644 --- a/explorer/package.json +++ b/explorer/package.json @@ -30,6 +30,7 @@ "webpack-cli": "^2.0.4" }, "dependencies": { + "axios": "^0.18.0", "ejs": "^2.6.1", "expressjs": "^1.0.1" } diff --git a/explorer/src/vues/routes/data_dashboard.vue b/explorer/src/vues/routes/data_dashboard.vue index c5e0a77..67127e4 100644 --- a/explorer/src/vues/routes/data_dashboard.vue +++ b/explorer/src/vues/routes/data_dashboard.vue @@ -96,13 +96,21 @@ diff --git a/importer/Dockerfile b/importer/Dockerfile index c820659..80eda52 100644 --- a/importer/Dockerfile +++ b/importer/Dockerfile @@ -11,5 +11,3 @@ RUN pip install pipenv RUN pipenv install --system --deploy RUN chmod +x import.py - -CMD sleep infinity diff --git a/readme.md b/readme.md index 4c06239..68a548c 100644 --- a/readme.md +++ b/readme.md @@ -2,8 +2,6 @@ The main goal of this tool is to have some visual and geographical feedback on [Cosmogony](https://github.com/osm-without-borders/cosmogony) output. -:construction::warning: This is a work in progress, and deeply connected to the Cosmogony output format :construction::warning: - ![cosmogony demo](./demo.gif) @@ -12,8 +10,24 @@ The main goal of this tool is to have some visual and geographical feedback on [ This will run docker-compose containers, import the cosmogony zones to Postgis, pre-generate tiles, and launch the explorer in your browser. - You may need to run it with `sudo`, as it launches `docker-compose` commands. + +The easiest way to run it: + +```bash +pipenv install +``` + +```bash +pipenv run inv -e run-local +``` + +You can set a path to the cosmogony data by changing the environment variable `PATH_TO_COSMOGONY_DIR`. +You can do this either just with a classic environment variable, or by changing the file `.env`. +By default it will load the data in the `./cosmogony_data/` directory. + +By default it will load a file named `cosmogony.json` in this directory. You can change the file name with the cli parameter `--cosmogony-file-name=` + ```bash -./run_explorer.sh cosmogony.json +PATH_TO_COSMOGONY_DIR=./my_data_dir pipenv run inv -e run-local --cosmogony-file-name=cosmo_lux.json ``` diff --git a/run_explorer.sh b/run_explorer.sh deleted file mode 100755 index 83753f8..0000000 --- a/run_explorer.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ -z "$1" ]; then - echo "Usage: run_explorer.sh [COSMOGONY_FILE]" - exit 1 -fi - -docker-compose up -d --build - -echo "Import to Postgis will start..." -sleep 10 # Waiting for postgres -docker-compose run -v "`dirname $(realpath $1)`:/tmp/volume" cosmogony-importer ./import.py import_data /tmp/volume/`basename $1` - -echo "Generating tiles..." -docker-compose exec cosmogony-tiles /usr/bin/t_rex generate -c /config_generate.toml --minzoom 0 --maxzoom 6 --overwrite true --progress true - -# publish the data so they can be used (without any downtime) -docker-compose run cosmogony-importer ./import.py publish -docker-compose exec cosmogony-tiles /publish_tiles.sh - -sensible-browser "http://localhost:8585/#/2.5/32/0" diff --git a/tasks.py b/tasks.py index 32daa79..416b700 100644 --- a/tasks.py +++ b/tasks.py @@ -5,9 +5,9 @@ logging.basicConfig(level=logging.INFO) @task() -def run_local(ctx, cosmogony_file, build_dockers=False): +def run_local(ctx, cosmogony_file_name='cosmogony.json', build_dockers=False): pop_stack(ctx) - run_explorer(ctx, cosmogony_file, use_local_file=True) + run_explorer(ctx, cosmogony_file_name) ctx.run("sensible-browser 'http://localhost:8585/#/2.5/32/0'") @task() @@ -15,30 +15,30 @@ def pop_stack(ctx, build_dockers=False): if build_dockers: ctx.run("docker-compose build --pull") ctx.run("docker-compose up -d") - ctx.run("sleep 10") # TODO do better to wait for pg + ctx.run("sleep 5") # TODO do better to wait for pg -@task(default=True) -def run_explorer(ctx, cosmogony_file, use_local_file=False): +def _run_container(ctx, container_name, job): main_docker_files = _get_docker_compose() run_docker_files = {'COMPOSE_FILE': main_docker_files + ':docker-compose.run.yml'} - + ctx.run(f"docker-compose run --rm {container_name} {job}", env=run_docker_files) - if use_local_file: - # TODO com' - file_name = os.path.basename(cosmogony_file) - path_in_container = f"/mnt/data/{file_name}" - volume = f"-v `dirname $(realpath {cosmogony_file})`:/mnt/data" - else: - # TODO com' - volume = "" - path_in_container = cosmogony_file - ctx.run(f"docker-compose run --rm {volume} importer ./import.py import_data {path_in_container}", env=run_docker_files) +@task(default=True) +def run_explorer(ctx, cosmogony_file_name='cosmogony.json'): + + path_in_container = f'/mnt/data/{cosmogony_file_name}' + _run_container(ctx, "importer", f"./import.py import_data {path_in_container}") ctx.run("docker-compose exec tiles /usr/bin/t_rex generate -c /config_generate.toml --minzoom 0 --maxzoom 6 --overwrite true --progress true", pty=True) - ctx.run("docker-compose run --rm importer ./import.py publish", env=run_docker_files) + _run_container(ctx, "importer", f"./import.py publish") ctx.run("docker-compose exec tiles /publish_tiles.sh", pty=True) + generate_data_dashboard(ctx, cosmogony_file_name) + def _get_docker_compose(): return os.environ.get('COMPOSE_FILE', 'docker-compose.yml') + +@task() +def generate_data_dashboard(ctx, cosmogony_file_name='cosmogony.json'): + _run_container(ctx, "data-dashboard", f"--cosmogony=/mnt/data/{cosmogony_file_name} --output=/mnt/data-dashboard/test_results.json")