Feature/deploy fluentd

ansible/deploy_swarm_monitoring.yml

@@ -0,0 +1,18 @@
+#
+# Copyright StackHPC, 2018
+#
+---
+- name: Deploy Swarm cluster monitoring
+  hosts: cluster
+  become: yes
+  roles:
+  - role: monasca_agent_docker
+  - role: monasca_fluentd
+  - role: prometheus-docker-node
+
+- name: Deploy Prometheus server
+  hosts: master
+  become: yes
+  roles:
+  - role: prometheus-server
+

ansible/group_vars/all/all

@@ -15,7 +15,7 @@ alaska_cloud: alaska
 alaska_homedir: /alaska
 alaska_softiron: 10.4.99.101
 
-# OpenStack fully qualified project name
+# OpenStack fully qualified project name (used for Grafana with domain support)
 project_name: p3@default
 
 # Virtual IP address of the controller node
@@ -28,6 +28,24 @@ alaska_monitoring_server: 10.60.253.3
 monasca_agent_p3_username: p3-monasca-agent
 monasca_agent_p3_password: "{{ vault_monasca_agent_password }}"
 
+# Monasca Fluentd config
+monasca_fluentd_log_api_uri: http://{{ controller_vip }}:5607
+monasca_fluentd_keystone_uri: http://{{ controller_vip }}:5000
+monasca_fluentd_username: "{{ monasca_agent_p3_username }}"
+monasca_fluentd_password: "{{ monasca_agent_p3_password }}"
+monasca_fluentd_project_domain_id: default
+monasca_fluentd_project_name: p3
+
+# Monasca Agent (Docker) config
+monasca_agent_docker_forwarder_port: 17120
+monasca_agent_docker_log_level: INFO
+monasca_agent_docker_api_uri: http://{{ controller_vip }}:8082/v2.0
+monasca_agent_docker_keystone_uri: http://{{ controller_vip }}:5000/v3
+monasca_agent_docker_username: "{{ monasca_agent_p3_username }}"
+monasca_agent_docker_password: "{{ monasca_agent_p3_password }}"
+monasca_agent_docker_project_name: p3
+
+
 # Local Grafana admin account for configuring Grafana
 grafana_admin_username: grafana-admin
 grafana_admin_password: "{{ vault_grafana_admin_password }}"

ansible/roles/monasca_agent_docker/tasks/main.yml

@@ -0,0 +1,116 @@
+#
+# Copyright StackHPC, 2018
+#
+---
+- name: Start monasca-agent-forwarder container
+  docker_container:
+    name: monasca-agent-forwarder
+    pull: true
+    image: stackhpc/agent-forwarder:latest
+    state: started
+    volumes:
+      - plugins:/etc/monasca/agent/conf.d:ro
+    ports:
+      - "{{ monasca_agent_docker_forwarder_port }}:{{ monasca_agent_docker_forwarder_port }}"
+    env:
+      LOG_LEVEL: "{{ monasca_agent_docker_log_level }}"
+      OS_AUTH_URL: "{{ monasca_agent_docker_keystone_uri }}"
+      OS_USERNAME: "{{ monasca_agent_docker_username }}"
+      OS_PASSWORD: "{{ monasca_agent_docker_password }}"
+      OS_USER_DOMAIN_NAME: Default
+      OS_PROJECT_NAME: "{{ monasca_agent_docker_project_name }}"
+      OS_PROJECT_DOMAIN_NAME: Default
+      MONASCA_URL: "{{ monasca_agent_docker_api_uri }}"
+      SERVICE_TYPE: monitoring
+      ENDPOINT_TYPE: public
+      REGION_NAME: RegionOne
+      AGENT_HOSTNAME: "{{ ansible_hostname }}"
+      FORWARDER_URL: "http://monasca_agent-forwarder:{{ monasca_agent_docker_forwarder_port }}"
+      FORWARDER_PORT: "{{ monasca_agent_docker_forwarder_port }}"
+
+- name: Create Monasca collector plugin directory
+  file:
+    path: /etc/monasca/agent/conf.d/
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Template Monasca collector plugins
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/monasca/agent/conf.d/{{ item }}"
+    owner: root
+    mode: 0644
+  with_items:
+    - cpu.yaml
+    - docker.yaml
+    - disk.yaml
+    - ib_network.yaml
+    - load.yaml
+    - memory.yaml
+    - network.yaml
+
+- name: Wait for forwarder service
+  wait_for:
+    port: "{{ monasca_agent_docker_forwarder_port }}"
+    delay: 1
+
+- name: Start monasca-agent-collector container
+  docker_container:
+    name: monasca-agent-collector
+    pull: true
+    links:
+      - monasca-agent-forwarder
+    image: stackhpc/agent-collector:latest
+    state: started
+    env:
+      DOCKER: True
+      LOG_LEVEL: "{{ monasca_agent_docker_log_level }}"
+      OS_AUTH_URL: "{{ monasca_agent_docker_keystone_uri }}"
+      OS_USERNAME: "{{ monasca_agent_docker_username }}"
+      OS_PASSWORD: "{{ monasca_agent_docker_password }}"
+      OS_USER_DOMAIN_NAME: Default
+      OS_PROJECT_NAME: "{{ monasca_agent_docker_project_name }}"
+      OS_PROJECT_DOMAIN_NAME: Default
+      MONASCA_URL: "{{ monasca_agent_docker_api_uri }}"
+      SERVICE_TYPE: monitoring
+      ENDPOINT_TYPE: public
+      REGION_NAME: RegionOne
+      AGENT_HOSTNAME: "{{ ansible_hostname }}"
+      FORWARDER_URL: "http://monasca-agent-forwarder:{{ monasca_agent_docker_forwarder_port }}"
+      FORWARDER_PORT: "{{ monasca_agent_docker_forwarder_port }}"
+    volumes:
+      - "/:/rootfs"
+      - "/var/run:/var/run:rw"
+      - "/sys:/sys:ro"
+      - "/var/lib/docker/:/var/lib/docker:ro"
+      - "/dev/disk/:/dev/disk:ro"
+      - "/etc/monasca/agent/conf.d:/etc/monasca/agent/conf.d:ro"
+
+- name: Start monasca-agent-statsd container
+  docker_container:
+    name: monasca-agent-statsd
+    pull: true
+    links:
+      - monasca-agent-forwarder
+    image: stackhpc/agent-statsd:latest
+    state: started
+    ports:
+      - "8125:8125/udp"
+    env:
+      DOCKER: True
+      LOG_LEVEL: "{{ monasca_agent_docker_log_level }}"
+      OS_AUTH_URL: "{{ monasca_agent_docker_keystone_uri }}"
+      OS_USERNAME: "{{ monasca_agent_docker_username }}"
+      OS_PASSWORD: "{{ monasca_agent_docker_password }}"
+      OS_USER_DOMAIN_NAME: Default
+      OS_PROJECT_NAME: "{{ monasca_agent_docker_project_name }}"
+      OS_PROJECT_DOMAIN_NAME: Default
+      MONASCA_URL: "{{ monasca_agent_docker_api_uri }}"
+      SERVICE_TYPE: monitoring
+      ENDPOINT_TYPE: public
+      REGION_NAME: RegionOne
+      AGENT_HOSTNAME: "{{ ansible_hostname }}"
+      FORWARDER_URL: "http://monasca-agent-forwarder:{{ monasca_agent_docker_forwarder_port }}"
+      FORWARDER_PORT: "{{ monasca_agent_docker_forwarder_port }}"

ansible/roles/monasca_agent_docker/templates/cpu.yaml.j2

@@ -0,0 +1,4 @@
+init_config: null
+instances:
+- built_by: System
+  name: cpu_stats

ansible/roles/monasca_agent_docker/templates/disk.yaml.j2

@@ -0,0 +1,6 @@
+init_config: null
+instances:
+- built_by: System
+  device_blacklist_re: .*freezer_backup_snap.*
+  ignore_filesystem_types: iso9660,tmpfs
+  name: disk_stats

ansible/roles/monasca_agent_docker/templates/docker.yaml.j2

@@ -0,0 +1,5 @@
+init_config:
+  docker_root: '/'
+
+instances:
+  - url: 'unix://var/run/docker.sock'

ansible/roles/monasca_agent_docker/templates/ib_network.yaml.j2

@@ -0,0 +1,4 @@
+init_config: null
+instances:
+- built_by: IBNetworkDetect
+  name: ib_network_stats

ansible/roles/monasca_agent_docker/templates/load.yaml.j2

@@ -0,0 +1,4 @@
+init_config: null
+instances:
+- built_by: System
+  name: load_stats

ansible/roles/monasca_agent_docker/templates/memory.yaml.j2

@@ -0,0 +1,4 @@
+init_config: null
+instances:
+- built_by: System
+  name: memory_stats

ansible/roles/monasca_agent_docker/templates/network.yaml.j2

@@ -0,0 +1,5 @@
+init_config: null
+instances:
+- built_by: System
+  excluded_interface_re: lo.*|vnet.*|tun.*|ovs.*|br.*|tap.*|qbr.*|qvb.*|qvo.*
+  name: network_stats

ansible/roles/monasca_fluentd/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart Fluentd
+  docker_container:
+    name: fluentd
+    restart: True
+  when: fluentd_config.changed

ansible/roles/monasca_fluentd/tasks/main.yml

@@ -0,0 +1,39 @@
+#
+# Copyright StackHPC, 2018
+#
+---
+- name: Create Fluentd user
+  user:
+    name: fluent
+    state: present
+
+- name: Create Fluentd config directory
+  file:
+    path: /etc/fluentd
+    state: directory
+    owner: fluent
+    group: fluent
+    mode: 0755
+
+- name: Generate Fluentd config
+  template:
+    src: fluentd.conf.j2
+    dest: /etc/fluentd/fluentd.conf
+    owner: fluent
+    group: fluent
+    mode: 0644
+  register: fluentd_config
+  notify: Restart Fluentd
+
+- name: Start Fluentd container
+  docker_container:
+    name: fluentd
+    image: stackhpc/monasca-fluentd:latest
+    state: started
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    volumes:
+      - /etc/fluentd/:/fluentd/etc:ro
+    env:
+      FLUENTD_CONF: "fluentd.conf"

ansible/roles/monasca_fluentd/templates/fluentd.conf.j2

@@ -0,0 +1,45 @@
+# Accept logs from Docker Fluentd log driver
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+# Add a timestamp dimension to all logs to record the event time. The
+# event time is the time extracted from the log message in all cases
+# where the time_key is set, and the time the record entered fluentd
+# if no time_key is set.
+# logs.
+<filter *.**>
+  @type record_transformer
+  <record>
+    timestamp ${time}
+  </record>
+</filter>
+
+# Docker saves all logs under the 'log' field. The fluentd-monasca
+# plugin assumes that they are saved under the 'message' field. Here
+# we map the 'log' field to the 'message' field for all logs.
+<filter *.**>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    message ${record["log"]}
+  </record>
+  remove_keys log
+</filter>
+
+<match *.**>
+    type copy
+    <store>
+       @type monasca
+       keystone_url {{ monasca_fluentd_keystone_uri }}
+       monasca_log_api {{ monasca_fluentd_log_api_uri }}
+       monasca_log_api_version v3.0
+       username {{ monasca_fluentd_username }}
+       password {{ monasca_fluentd_password }}
+       domain_id {{ monasca_fluentd_project_domain_id }}
+       project_name {{ monasca_fluentd_project_name }}
+    </store>
+</match>
+

ansible/roles/prometheus-docker-node/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- name: Add Prometheus node-exporter
+  docker_container:
+    name: prom-node-exporter
+    image: prom/node-exporter
+    network_mode: host  # TODO should access host fs too
+    ports:
+      - 9100:9100
+
+- name: Add cAdvisor
+  docker_container:
+    name: cAdvisor
+    image: google/cadvisor:latest
+    privileged: yes
+    detach: yes
+    ports:
+      - 8080:8080
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro

ansible/roles/prometheus-server/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+prometheus_home: /home/centos/prometheus  # TODO - better default?