diff --git a/.circleci/config.yml b/.circleci/config.yml index 14870e4..5814d94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,7 @@ version: 2.1 requires: - lint - lint-shell + - build-prometheus-exporter-file: *build-http - build-fpm: &build-fpm requires: - lint @@ -21,6 +22,9 @@ version: 2.1 - build-fpm - build-cli - test-http-e2e: *test-http + - test-prometheus-exporter-file-e2e: &test-prometheus-exporter-file-e2e + requires: + - build-prometheus-exporter-file - test-fpm: &test-fpm requires: - build-http @@ -34,6 +38,7 @@ version: 2.1 - scan-vulnerability: &scan-vulnerability requires: - build-http + - build-prometheus-exporter-file - build-fpm - build-cli - push-http: &push-context @@ -47,7 +52,9 @@ version: 2.1 - test-fpm - test-http - test-http-e2e + - test-prometheus-exporter-file-e2e - scan-vulnerability + - push-prometheus-exporter-file: *push-context - push-fpm: *push-context - push-cli: *push-context @@ -55,10 +62,12 @@ version: 2.1 - lint - lint-shell - build-http: *build-http + - build-prometheus-exporter-file: *build-http - build-fpm: *build-fpm - build-cli: *build-cli - test-http: *test-http - test-http-e2e: *test-http + - test-prometheus-exporter-file-e2e: *test-prometheus-exporter-file-e2e - test-fpm: *test-fpm - test-cli: *test-cli - scan-vulnerability: *scan-vulnerability @@ -72,6 +81,7 @@ version: 2.1 - test-cli - test-fpm - test-http + - test-prometheus-exporter-file-e2e - push-http: &push-context-approval context: dockerhub filters: @@ -80,6 +90,7 @@ version: 2.1 - master requires: - push-approval + - push-prometheus-exporter-file: *push-context-approval - push-fpm: *push-context-approval - push-cli: *push-context-approval @@ -110,6 +121,8 @@ commands: steps: - docker_load: image: http + - docker_load: + image: prometheus-exporter-file - docker_load: image: fpm - docker_load: @@ -173,6 +186,17 @@ jobs: - run: make test-http-e2e - store_test_results: path: ./tmp/test-results + test-prometheus-exporter-file-e2e: + machine: true + steps: + - checkout + - attach_workspace: + at: ./tmp + - docker_load: + image: prometheus-exporter-file + - run: make test-prometheus-exporter-file-e2e + - store_test_results: + path: ./tmp/test-results scan-vulnerability: machine: true steps: @@ -205,6 +229,18 @@ jobs: paths: - usabillabv_php-http.tar - build-http.tags + build-prometheus-exporter-file: + machine: true + steps: + - checkout + - run: make build-prometheus-exporter-file + - run: cat ./tmp/build-prometheus-exporter-file.tags | xargs -I % docker inspect --format='%={{.Id}}:{{index .ContainerConfig.Env 1}}' % + - run: docker save usabillabv/php -o ./tmp/usabillabv_php-prometheus-exporter-file.tar + - persist_to_workspace: + root: ./tmp + paths: + - usabillabv_php-prometheus-exporter-file.tar + - build-prometheus-exporter-file.tags build-fpm: machine: true steps: @@ -238,6 +274,15 @@ jobs: - docker_load: image: http - run: make ci-push-http + push-prometheus-exporter-file: + machine: true + steps: + - checkout + - attach_workspace: + at: ./tmp + - docker_load: + image: prometheus-exporter-file + - run: make ci-push-prometheus-exporter-file push-cli: machine: true steps: diff --git a/Dockerfile-http b/Dockerfile-http index 63ffa9a..712bf67 100644 --- a/Dockerfile-http +++ b/Dockerfile-http @@ -5,6 +5,8 @@ RUN set -x \ && addgroup -g 1000 app \ && adduser -u 1000 -D -G app app +ARG NGINX_VHOST_TEMPLATE + # Env definition ENV NGINX_DOCUMENT_ROOT="/opt/project/public" ENV NGINX_SERVER_NAME=localhost @@ -16,14 +18,12 @@ ENV NGINX_CLIENT_BODY_BUFFER_SIZE=16k ENV NGINX_CLIENT_MAX_BODY_SIZE=1m ENV NGINX_LARGE_CLIENT_HEADER_BUFFERS="4 8k" -# Patch CVE-2018-1152 CVE-2018-11813 CVE-2017-15232 -RUN apk upgrade --no-cache libjpeg-turbo - # Nginx helper scripts COPY src/http/nginx/docker-nginx-* /usr/local/bin/ # Nginx configuration files -COPY src/http/nginx/conf/ /etc/nginx/ +COPY src/http/nginx/conf/main /etc/nginx/ +COPY src/http/nginx/conf/${NGINX_VHOST_TEMPLATE} /etc/nginx/ CMD ["docker-nginx-entrypoint"] diff --git a/Makefile b/Makefile index f09e0d6..a588873 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ qa: lint lint-shell build test scan-vulnerability -build: clean-tags build-cli build-fpm build-http +build: clean-tags build-cli build-fpm build-http build-prometheus-exporter-file push: build push-cli build-fpm push-http ci-push-cli: ci-docker-login push-cli ci-push-fpm: ci-docker-login push-fpm @@ -35,6 +35,12 @@ build-http: clean-tags ./build-nginx.sh 1.15 nginx ./build-nginx.sh 1.14 +# Docker Prometheus Exporter file images build matrix ./build-prometheus-exporter-file.sh (nginx version) (extra tag) +# Adding arbitrary version 1.0 in order to make sure if we break compatibility we have to up it +build-prometheus-exporter-file: BUILDINGIMAGE=prometheus-exporter-file +build-prometheus-exporter-file: clean-tags + ./build-prometheus-exporter-file.sh 1.15 prometheus-exporter-file1.0 prometheus-exporter-file1 + .NOTPARALLEL: clean-tags clean-tags: rm ${current_dir}/tmp/build-${BUILDINGIMAGE}.tags || true @@ -49,6 +55,9 @@ push-fpm: push-http: BUILDINGIMAGE=http push-http: cat ./tmp/build-${BUILDINGIMAGE}.tags | xargs -I % docker push % +push-prometheus-exporter-file: BUILDINGIMAGE=prometheus-exporter-file +push-prometheus-exporter-file: + cat ./tmp/build-${BUILDINGIMAGE}.tags | xargs -I % docker push % # CI dependencies ci-docker-login: @@ -60,7 +69,7 @@ lint: lint-shell: docker run --rm -v ${current_dir}:/mnt:ro koalaman/shellcheck src/http/nginx/docker* src/php/utils/install-* src/php/utils/docker/* build* test-* -test: test-cli test-fpm test-http +test: test-cli test-fpm test-http test-prometheus-exporter-file-e2e test-cli: ./tmp/build-cli.tags xargs -I % ./test-cli.sh % < ./tmp/build-cli.tags @@ -77,6 +86,9 @@ test-http: ./tmp/build-http.tags ./tmp/build-fpm.tags test-http-e2e: ./tmp/build-http.tags xargs -I % ./test-http-e2e.sh % < ./tmp/build-http.tags +test-prometheus-exporter-file-e2e: ./tmp/build-prometheus-exporter-file.tags + xargs -I % ./test-prometheus-exporter-file-e2e.sh % < ./tmp/build-prometheus-exporter-file.tags + scan-vulnerability: docker-compose -f test/security/docker-compose.yml -p clair-ci up -d RETRIES=0 && while ! wget -T 10 -q -O /dev/null http://localhost:6060/v1/namespaces ; do sleep 1 ; echo -n "." ; if [ $${RETRIES} -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; RETRIES=$$(($${RETRIES}+1)) ; done diff --git a/README.md b/README.md index a28ba1e..7b9da6d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ A series of Docker images to run PHP Applications on Usabilla Style - [Alpine Linux situation](#alpine-linux-situation) - [The available tags](#the-available-tags) - [Adding more supported versions](#adding-more-supported-versions) +- [Prometheus Exporter](#prometheus-exporter) - [Dockerfile example with Buildkit](#dockerfile-example) - [PHP FPM functional example](docs/examples/hello-world-fpm) - [Contributing](.github/CONTRIBUTING.md) @@ -417,6 +418,72 @@ Both are enabled via the helper script, by running $ docker-php-dev-mode config ``` +## Prometheus Exporter + +In order to monitor applications many systems implement Prometheus to expose metrics, one challenge specially in PHP is how to expose those to Prometheus without having to, either implement an endpoint in our application, or add HTTP and an endpoint for non-interactive containers. + +This prove has the aim to provide support for the sidecar pattern for monitoring. + +More about ["Make your application easy to monitor" by Google](https://cloud.google.com/solutions/best-practices-for-operating-containers#make_your_application_easy_to_monitor) + +### Static File + +The easiest way to solve this problem in the PHP ecosystem is to make your application write down the metrics to a text file, which then is shared via a volume to a sidecar container which can expose it to Prometheus. + +The container we offer is a simple Nginx based on the same configuration as [the one for PHP-FPM](#for-nginx-customization), with the difference it only serves static content. + +#### Docker image + +The image named `prometheus-exporter-file` is available via our docker registry under with the tags (from less to more specific versions): + +- `usabillabv/php:prometheus-exporter-file` - This has the behavior of latest +- `usabillabv/php:prometheus-exporter-file1` +- `usabillabv/php:prometheus-exporter-file1.0` + +#### Kubernetes Deployment Example + +```yaml +# Pod v1 core Spec - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#pod-v1-core + +spec: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "80" + prometheus.io/scrape: "true" + spec: + containers: + - image: usabillabv/php:7.3-cli-alpine3.9 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /prometheus + name: prometheus-metrics + - image: usabillabv/php:prometheus-exporter-file1 + imagePullPolicy: IfNotPresent + name: prometheus-exporter + ports: + - containerPort: 80 + name: http + protocol: TCP + volumeMounts: + - mountPath: /opt/project/public + name: prometheus-metrics + volumes: + - emptyDir: {} + name: prometheus-metrics + +``` + +In this example the PHP container *must* write down the metrics in the file `/prometheus/metrics`, the exporter container will have the same file mount at `/opt/project/public/metrics`. +Which will then be available via http as `http://pod:80/metrics`, observe that the filename becomes the url which we configured the prometheus scrape to look for. + +### Open Census + +_To be created and/or documented_ + +For now please refer to: https://github.com/basvanbeek/opencensus-php-docker and https://github.com/census-instrumentation/opencensus-php + ## Dockerfile example The Dockerfile in the example below is meant to centralize the production and development images in a single Dockerfile, diff --git a/build-nginx.sh b/build-nginx.sh index c1c4805..410cea3 100755 --- a/build-nginx.sh +++ b/build-nginx.sh @@ -22,10 +22,12 @@ declare -r USABILLA_EXTRA_TAG_DEV="${USABILLA_EXTRA_TAG}-dev" TAG_FILE="./tmp/build-${IMAGE}.tags" -sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${IMAGE}" | docker build --pull -t "${USABILLA_TAG}" --target="${IMAGE}" -f - . \ +sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${IMAGE}" | docker build --pull -t "${USABILLA_TAG}" \ + --build-arg=NGINX_VHOST_TEMPLATE=php-fpm --target="${IMAGE}" -f - . \ && echo "${USABILLA_TAG}" >> "${TAG_FILE}" -sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${IMAGE}" | docker build --pull -t "${USABILLA_TAG_DEV}" --target="${IMAGE}-dev" -f - . \ +sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${IMAGE}" | docker build --pull -t "${USABILLA_TAG_DEV}" \ + --build-arg=NGINX_VHOST_TEMPLATE=php-fpm --target="${IMAGE}-dev" -f - . \ && echo "$USABILLA_TAG_DEV" >> "${TAG_FILE}" if [[ -n ${IMAGE_EXTRA_TAG} ]]; then diff --git a/build-prometheus-exporter-file.sh b/build-prometheus-exporter-file.sh new file mode 100755 index 0000000..820f1ab --- /dev/null +++ b/build-prometheus-exporter-file.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -eEuo pipefail + +declare -r IMAGE="prometheus-exporter-file" + +declare -r DOCKER_FILE="http" + +declare -r VERSION_NGINX=$1 + +# I could create a placeholder like nginx:x.y-alpine in the Dockerfile itself, +# but I think it wouldn't be a good experience if you try to build the image yourself +# thus that's the way I opted to have dynamic base images +declare -r IMAGE_ORIGINAL_TAG="nginx:1.[0-9][0-9]?-alpine" + +declare -r IMAGE_TAG="nginx:${VERSION_NGINX}-alpine" +declare -r USABILLA_TAG_PREFIX="usabillabv/php" +declare -r USABILLA_TAG="${USABILLA_TAG_PREFIX}:${IMAGE}" + +TAG_FILE="./tmp/build-${IMAGE}.tags" + +sed -E "s/${IMAGE_ORIGINAL_TAG}/${IMAGE_TAG}/g" "Dockerfile-${DOCKER_FILE}" | docker build --pull -t "${USABILLA_TAG}" \ + --build-arg=NGINX_VHOST_TEMPLATE=prometheus-exporter-file --target="http" -f - . \ + && echo "${USABILLA_TAG}" >> "${TAG_FILE}" + +for USABILLA_TAG_EXTRA in "${@:2}" +do + docker tag "${USABILLA_TAG}" "${USABILLA_TAG_PREFIX}:${USABILLA_TAG_EXTRA}" \ + && echo "${USABILLA_TAG_PREFIX}:${USABILLA_TAG_EXTRA}" >> "${TAG_FILE}" +done diff --git a/src/http/nginx/conf/location.d-available/cors.conf b/src/http/nginx/conf/main/location.d-available/cors.conf similarity index 100% rename from src/http/nginx/conf/location.d-available/cors.conf rename to src/http/nginx/conf/main/location.d-available/cors.conf diff --git a/src/http/nginx/conf/location.d-enabled/.gitkeep b/src/http/nginx/conf/main/location.d-enabled/.gitkeep similarity index 100% rename from src/http/nginx/conf/location.d-enabled/.gitkeep rename to src/http/nginx/conf/main/location.d-enabled/.gitkeep diff --git a/src/http/nginx/conf/nginx.conf.template b/src/http/nginx/conf/main/nginx.conf.template similarity index 96% rename from src/http/nginx/conf/nginx.conf.template rename to src/http/nginx/conf/main/nginx.conf.template index deb3c76..fc5f598 100644 --- a/src/http/nginx/conf/nginx.conf.template +++ b/src/http/nginx/conf/main/nginx.conf.template @@ -11,7 +11,7 @@ events { http { include /etc/nginx/mime.types; - default_type application/octet-stream; + default_type text/plain; log_format gzip '${ESCAPE}remote_addr - ${ESCAPE}remote_user [${ESCAPE}time_local] "${ESCAPE}request" ' '${ESCAPE}status ${ESCAPE}body_bytes_sent "${ESCAPE}http_referer" ' diff --git a/src/http/nginx/conf/vhost.conf.template b/src/http/nginx/conf/php-fpm/vhost.conf.template similarity index 100% rename from src/http/nginx/conf/vhost.conf.template rename to src/http/nginx/conf/php-fpm/vhost.conf.template diff --git a/src/http/nginx/conf/prometheus-exporter-file/vhost.conf.template b/src/http/nginx/conf/prometheus-exporter-file/vhost.conf.template new file mode 100644 index 0000000..a6368de --- /dev/null +++ b/src/http/nginx/conf/prometheus-exporter-file/vhost.conf.template @@ -0,0 +1,18 @@ +server { + listen 80; + listen [::]:80; + server_name ${NGINX_SERVER_NAME}; + + root ${NGINX_DOCUMENT_ROOT}; + charset UTF-8; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location / { + sendfile on; + sendfile_max_chunk 1m; + } +} diff --git a/test-prometheus-exporter-file-e2e.sh b/test-prometheus-exporter-file-e2e.sh new file mode 100755 index 0000000..4e60f0d --- /dev/null +++ b/test-prometheus-exporter-file-e2e.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# A simple script to start a Docker container +# and run Testinfra in it +# Original script: https://gist.github.com/renatomefi/bbf44d4e8a2614b1390416c6189fbb8e +# Author: @renatomefi https://github.com/renatomefi +# + +set -eEuo pipefail + +# The first parameter is a Docker tag or image id +declare -r DOCKER_TAG="$1" + +declare -r TEST_SUITE="prometheus_exporter_file_e2e" + +# Finally, run the tests! +docker run --net="host" --rm -t \ + -v "$(pwd)/test/e2e:/tests" \ + -v "$(pwd)/tmp/test-results:/results" \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + renatomefi/docker-testinfra:2 \ + -m "$TEST_SUITE" --junitxml="/results/http-e2e-$DOCKER_TAG.xml" \ + --verbose --tag="$1" diff --git a/test/e2e/test_prometheus_exporter_file.py b/test/e2e/test_prometheus_exporter_file.py new file mode 100644 index 0000000..9f08559 --- /dev/null +++ b/test/e2e/test_prometheus_exporter_file.py @@ -0,0 +1,39 @@ +import pytest +import requests + + +@pytest.mark.prometheus_exporter_file_e2e +def test_prometheus_exporter_file_propagates_content_type_text(host, container): + nginx_port = host.check_output("docker inspect " + container + " --format '{{ (index (index .NetworkSettings.Ports \"80/tcp\") 0).HostPort }}'") + + req_root = requests.get("http://localhost:{}/".format(nginx_port)) + assert req_root.status_code == 404 + + add_file = host.run('docker exec -t {} sh -c "mkdir -p /opt/project/public && echo -n \'Hey there!\' > /opt/project/public/hi"'.format(container)) + assert add_file.rc is 0 + + req_file = requests.get("http://localhost:{}/hi".format(nginx_port)) + assert req_file.status_code == 200 + assert req_file.text == u'Hey there!' + + assert 'content-type' in req_file.headers + assert req_file.headers['content-type'] == 'text/plain; charset=UTF-8' + +@pytest.mark.prometheus_exporter_file_e2e +def test_prometheus_exporter_file_propagates_content_type_json(host, container): + nginx_port = host.check_output("docker inspect " + container + " --format '{{ (index (index .NetworkSettings.Ports \"80/tcp\") 0).HostPort }}'") + + req_root = requests.get("http://localhost:{}/".format(nginx_port)) + assert req_root.status_code == 404 + + add_file = host.run('docker exec -t {} sh -c "mkdir -p /opt/project/public && echo -n \'{}\' > /opt/project/public/hi.json"' + .format(container, '{\\"text\\": \\"Hi thére!\\"}') + ) + assert add_file.rc is 0 + + req_file = requests.get("http://localhost:{}/hi.json".format(nginx_port)) + assert req_file.status_code == 200 + assert req_file.text == u'{"text": "Hi thére!"}' + + assert 'content-type' in req_file.headers + assert req_file.headers['content-type'] == 'application/json'