Integration tests and v1.0.0-beta5 features (#13)

Author: raffis

.github/workflows/action.yml

@@ -8,18 +8,22 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v2.1.3
         with:
           go-version: '1.15'
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
-      - name: Unit tests & coverage
-        run: |
-          make test build
-      - name: Full build & docker image
+      - name: Unit tests
+        run: make deps vet format unittest
+      - name: Build binary
+        run: make build
+      - name: Integration tests
+        run: make integrationtest
+      - name: Build docker image
         run: make docker DOCKER_IMAGE_TAG=${{ steps.get_version.outputs.VERSION }}
-      - uses: shogo82148/actions-goveralls@v1
+      - name: Send go coverage report
+        uses: shogo82148/actions-goveralls@v1
         with:
           path-to-profile: coverage.out
       - name: Create beta Release
@@ -66,24 +70,9 @@ jobs:
           asset_path: ./mongodb_query_exporter
           asset_name: mongodb_query_exporter
           asset_content_type: application/octet-stream
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        if: startsWith(github.ref, 'refs/tags/v')
+      - name: Publish to DockerHub
+        uses: elgohr/Publish-Docker-Github-Action@master
         with:
+          name: raffis/mongodb-query-exporter:${{ steps.get_version.outputs.VERSION }}
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Push to Docker Hub
-        uses: docker/build-push-action@v2
-        if: startsWith(github.ref, 'refs/tags/v')
-        with:
-          push: true
-          tags: raffis/mongodb-query-exporter:${{ steps.get_version.outputs.VERSION }}
-      - name: Run helm lint
-        id: lint
-        uses: WyriHaximus/github-action-helm3@v2
-        with:
-          exec: helm lint chart/

.github/workflows/chart.yml

@@ -0,0 +1,15 @@
+name: Chart
+on: push
+
+jobs:
+  chart:
+    name: Chart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run helm lint
+        id: lint
+        uses: WyriHaximus/github-action-helm3@v2
+        with:
+          exec: helm lint chart/prometheus-mongodb-query-exporter

CHANGELOG.md

@@ -4,11 +4,16 @@
 
 ## Features
 * Support for multiple MongoDB servers
-* Support to configure custom tls certificate
 * Support for versioned configurations
-* GO public API
+* Redesigned GO public API
+* New log implementation, the default is now json log format
+* Added new counter metric for internal stats about quries `mongodb_query_exporter_query_total`
+* Added root hint to go to /metrics
 
 ## Changes
+* Dropped interval pulling mechanism, pull is now synchronous while getting called by an http request
+* Drop support for counters (use gauge instead)
+* No internal tracking of upstreams anymore, using prometheus ConstMetric
 * go 1.15 update
 
 ## Packaging

Dockerfile

@@ -4,7 +4,7 @@ ADD .   /go/src/github.com/raffis/mongodb-query-exporter
 WORKDIR /go/src/github.com/raffis/mongodb-query-exporter
 
 
-RUN make all
+RUN make deps vet format build unittest
 
 FROM        gcr.io/distroless/base
 COPY        --from=builder /go/src/github.com/raffis/mongodb-query-exporter/mongodb_query_exporter /bin/mongodb_query_exporter

Makefile

@@ -14,7 +14,9 @@
 GO     := go
 GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
 PROMU  := $(GOPATH)/bin/promu
-pkgs    = $(shell $(GO) list ./... | grep -v /vendor/)
+pkgs    := $(shell $(GO) list ./... | grep -v /vendor/)
+units    := $(shell $(GO) list ./... | grep -v /vendor/ | grep -v cmd)
+integrations    := $(shell $(GO) list ./... | grep cmd)
 
 PREFIX              ?= $(shell pwd)
 BIN_DIR             ?= $(shell pwd)
@@ -28,9 +30,15 @@ style:
 	@echo ">> checking code style"
 	@! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^'
 
-test:
-	@echo ">> running tests"
-	@$(GO) test -short -race -v -coverprofile=coverage.out $(pkgs)
+test: unittest integrationtest
+
+unittest:
+	@echo ">> running unit tests"
+	@$(GO) test -short -race -v -coverprofile=coverage.out $(units)
+
+integrationtest:
+	@echo ">> running integration tests"
+	@$(GO) test -short -race -v  $(integrations)
 
 deps:
 	@echo ">> install dependencies"

README.md

@@ -1,5 +1,5 @@
 # Prometheus MongoDB query exporter
-![.github/workflows/action.yml](https://github.com/raffis/mongodb-query-exporter/workflows/.github/workflows/action.yml/badge.svg)
+[![.github/workflows/action.yml](https://github.com/raffis/mongodb-query-exporter/workflows/.github/workflows/action.yml/badge.svg)](https://github.com/raffis/mongodb-query-exporter/actions)
 [![Go Report Card](https://goreportcard.com/badge/github.com/raffis/mongodb-query-exporter)](https://goreportcard.com/report/github.com/raffis/mongodb-query-exporter)
 [![PkgGoDev](https://pkg.go.dev/badge/github.com/raffis/mongodb-query-exporter?tab=subdirectories)](https://pkg.go.dev/github.com/raffis/mongodb-query-exporter?tab=subdirectories)
 [![Coverage Status](https://coveralls.io/repos/github/raffis/mongodb-query-exporter/badge.svg?branch=master)](https://coveralls.io/github/raffis/mongodb-query-exporter?branch=master)
@@ -9,11 +9,12 @@ MongoDB aggregation query exporter for [Prometheus](https://prometheus.io).
 
 ## Features
 
-* Support for gauge and counter metrics
+* Support for gauge metrics
 * Multiple metrics for different db/collections
 * Pull and Push (Push is only supported for MongoDB >= 3.6)
 * Supports multiple MongoDB servers
 * Public API for Golang
+* Metric caching support
 
 Note that this is not designed to be a replacement for the [MongoDB exporter](https://github.com/percona/mongodb_exporter) to instrument MongoDB internals. This application exports custom MongoDB metrics in the prometheus format based on the queries (aggregations) you want.
 
@@ -28,6 +29,12 @@ Get Prometheus MongoDB aggregation query exporter, either as a [binary](https://
 ### Helm Chart
 For kubernetes users there is an official helm chart for the MongoDB query exporter.
 
+Install the chart (Note only helm 3 is supported):
+```
+helm repo add mongodb-query-exporter
+helm install mongodb-query-exporter mongodb-query-exporter/mongodb-query-exporter
+```
+
 ## Usage
 
 ```
@@ -86,8 +93,8 @@ You may also use env variables to configure the exporter:
 
 Note if you have multiple collectors you can inject an env variable for the MongoDB connection URI like:
 
-1. `MDBEXPORTER_COLLECTOR_0_MONGODB_URI=mongodb://srv1:27017`
-2. `MDBEXPORTER_COLLECTOR_1_MONGODB_URI=mongodb://srv2:27017`
+1. `MDBEXPORTER_SERVER_0_MONGODB_URI=mongodb://srv1:27017`
+2. `MDBEXPORTER_SERVER_1_MONGODB_URI=mongodb://srv2:27017`
 3. ...
 
 ### Format v2.0
@@ -105,63 +112,67 @@ log:
   level: info
   development: false
   disableCaller: false
-collectors:
-- mongodb:
-    uri: mongodb://localhost:27017
-    queryTimeout: 10
-    maxConnection: 3
-    defaultInterval: 5
-  metrics:
-  - name: myapp_example_simplevalue_total
-    type: gauge
-    help: 'Simple gauge metric'
-    value: total
-    labels: []
-    mode: pull
-    interval: 10
-    constLabels: []
-    database: mydb
-    collection: objects  
-    pipeline: |
-      [
-        {"$count":"total"}
-      ]  
-  - name: myapp_example_processes_total
-    type: gauge
-    help: 'The total number of processes in a job queue'
-    value: total
-    mode: push
-    labels: [type,status]
-    constLabels: []
-    database: mydb
-    collection: queue
-    pipeline: |
-      [
-        {"$group": {
-          "_id":{"status":"$status","name":"$class"},
-          "total":{"$sum":1}
-        }},
-        {"$project":{
-          "_id":0,
-          "type":"$_id.name",
-          "total":"$total",
-          "status": {
-            "$switch": {
-                "branches": [
-                   { "case": { "$eq": ["$_id.status", 0] }, "then": "waiting" },
-                   { "case": { "$eq": ["$_id.status", 1] }, "then": "postponed" },
-                   { "case": { "$eq": ["$_id.status", 2] }, "then": "processing" },
-                   { "case": { "$eq": ["$_id.status", 3] }, "then": "done" },
-                   { "case": { "$eq": ["$_id.status", 4] }, "then": "failed" },
-                   { "case": { "$eq": ["$_id.status", 5] }, "then": "canceled" },
-                   { "case": { "$eq": ["$_id.status", 6] }, "then": "timeout" }
-                ],
-                "default": "unknown"
-            }}
-        }}
-      ]
+global:
+  queryTimeout: 10
+  maxConnection: 3
+  defaultCache: 0
+servers:
+- name: main
+  uri: mongodb://localhost:27017
+metrics:
+- name: myapp_example_simplevalue_total
+  type: gauge #Can also be empty, the default is gauge
+  servers: [main] #Can also be empty, if empty the metric will be used for every server defined
+  help: 'Simple gauge metric'
+  value: total
+  labels: []
+  mode: pull
+  cache: 0
+  database: mydb
+  collection: objects
+  pipeline: |
+    [
+      {"$count":"total"}
+    ]
+- name: myapp_example_processes_total
+  type: gauge
+  help: 'The total number of processes in a job queue'
+  value: total
+  mode: push
+  labels: [type,status]
+  constLabels:
+    app: foo
+  database: mydb
+  collection: queue
+  pipeline: |
+    [
+      {"$group": {
+        "_id":{"status":"$status","name":"$class"},
+        "total":{"$sum":1}
+      }},
+      {"$project":{
+        "_id":0,
+        "type":"$_id.name",
+        "total":"$total",
+        "status": {
+          "$switch": {
+              "branches": [
+                 { "case": { "$eq": ["$_id.status", 0] }, "then": "waiting" },
+                 { "case": { "$eq": ["$_id.status", 1] }, "then": "postponed" },
+                 { "case": { "$eq": ["$_id.status", 2] }, "then": "processing" },
+                 { "case": { "$eq": ["$_id.status", 3] }, "then": "done" },
+                 { "case": { "$eq": ["$_id.status", 4] }, "then": "failed" },
+                 { "case": { "$eq": ["$_id.status", 5] }, "then": "canceled" },
+                 { "case": { "$eq": ["$_id.status", 6] }, "then": "timeout" }
+              ],
+              "default": "unknown"
+          }}
+      }}
+    ]
 ```
 
+See more examples in the `/examples` folder.
+
 ### Format v1.0
 
 The config version v1.0 is the predescer of v2.0 and does not have support for multiple MongoDB servers
@@ -188,7 +199,6 @@ metrics:
   labels: []
   mode: pull
   interval: 10
-  constLabels: []
   database: mydb
   collection: objects  
   pipeline: |
@@ -201,7 +211,8 @@ metrics:
   value: total
   mode: push
   labels: [type,status]
-  constLabels: []
+  constLabels:
+    app: foo
   database: mydb
   collection: queue
   pipeline: |
@@ -231,6 +242,55 @@ metrics:
     ]
 ```
 
+## Cache & Push
+Prometheus is designed to scrape metrics (meaning pull). During each scrape the mongodb-query-exporter will evaluate all configured metrics.
+If you have expensive queries there is an option to cache the aggregation result by setting a cache ttl in secconds.
+However it is more effective to **avoid cache** and design good aggregation pipelines or use a different scrape interval or use the **push** mode.
+For individual metrics and/or MongoDB servers older than 3.6 it might still be a good option though.
+
+Example:
+```yaml
+metrics:
+- name: myapp_example_simplevalue_total
+  servers: [main]
+  help: 'Simple gauge metric which is cached for 5min'
+  value: total
+  mode: pull
+  cache: 300
+  database: mydb
+  collection: objects
+  pipeline: |
+    [
+      {"$count":"total"}
+    ]
+```
+
+A better approach of reducing load on the MongoDB server is the supported push mode. The push automatically caches the metric at scrape time. However the cache for a metric with mode push
+will be invalidated automatically if anything changes on the configured MongoDB collection. Meaning the aggregation will only be executed if there have been changes during scrape intervals.
+
+>**Note**: This requires at least MongoDB 3.6.
+
+Example:
+```yaml
+metrics:
+# With the mode push the pipeline is only executed if a change occured on the collection called objects
+- name: myapp_example_simplevalue_total
+  servers: [main]
+  help: 'Simple gauge metric'
+  value: total
+  mode: push
+  database: mydb
+  collection: objects
+  pipeline: |
+    [
+      {"$count":"total"}
+    ]
+```
+
+## Debug
+The mongodb-query-exporters also publishes a counter metric called `mongodb_query_exporter_query_total` which publishes query results for each configured metric.
+Furthermore you might increase the log level to get more insight.
+
 ## Go API
 Instead using the mongodb-query-exporter you may use the API to integrate the exporter within your go project.
 Please check out the [go package reference](https://pkg.go.dev/badge/github.com/raffis/mongodb-query-exporter?tab=subdirectories).

chart/prometheus-mongodb-query-exporter/Chart.yaml

@@ -6,10 +6,10 @@ keywords:
 - exporter
 - metrics
 - mongodb
-- query-
 - aggregation
+- query
 - prometheus
 name: prometheus-mongodb-query-exporter
 sources:
 - https://github.com/raffis/mongodb-query-exporter
-version: 1.0.0
+version: 1.0.1

chart/prometheus-mongodb-query-exporter/README.md

@@ -7,7 +7,7 @@ Installs the [MongoDB Query Exporter](https://github.com/raffis/mongodb-query-ex
 To install the chart with the release name `my-release`:
 
 ```console
-$ helm upgrade --install my-release mongodb-query-exporter/prometheus-mongodb-exporter
+$ helm upgrade --install my-release mongodb-query-exporter/prometheus-mongodb-exporter --set mongodb.0 mongodb://mymongodb:27017
 ```
 
 This command deploys the MongoDB Exporter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.