From 767a6faca7e5ec931b5d31df17e0f47a98e77ad0 Mon Sep 17 00:00:00 2001
From: Eduardo Silva <eduardo@treasure-data.com>
Date: Fri, 3 Nov 2017 14:33:24 -0600
Subject: [PATCH] New repo structure and Manifess

Signed-off-by: Eduardo Silva <eduardo@treasure-data.com>
---
 README.md                               | 35 ++++-------
 fluent-bit-daemonset-configmap.yaml     | 83 -------------------------
 fluent-bit-daemonset-elasticsearch.yaml | 59 ------------------
 fluent-bit-rbac.yaml                    | 44 -------------
 fluent-bit-role-binding.yaml            | 12 ++++
 fluent-bit-role.yaml                    | 10 +++
 fluent-bit-service-account.yaml         |  5 ++
 output/elasticsearch/fluent-bit-ds.yaml | 49 +++++++++++++++
 8 files changed, 89 insertions(+), 208 deletions(-)
 delete mode 100644 fluent-bit-daemonset-configmap.yaml
 delete mode 100644 fluent-bit-daemonset-elasticsearch.yaml
 delete mode 100644 fluent-bit-rbac.yaml
 create mode 100644 fluent-bit-role-binding.yaml
 create mode 100644 fluent-bit-role.yaml
 create mode 100644 fluent-bit-service-account.yaml
 create mode 100644 output/elasticsearch/fluent-bit-ds.yaml

diff --git a/README.md b/README.md
index c6e1d90..b5835c3 100644
--- a/README.md
+++ b/README.md
@@ -1,35 +1,26 @@
-# Help to Test!
+# Kubernetes Logging with Fluent Bit
 
-[Fluent Bit](http://fluentbit.io) v0.11 was just released at the end of March 2017 and it comes with support for Kubernetes logging needs: it can consume logs and enrich them with proper Kubernetes metadata, retrieved from the API server of course.
+[Fluent Bit](http://fluentbit.io) is a lightweight and extensible __Log Processor__ that comes with full support for Kubernetes:
 
-Our goal is to make logging cheaper in terms of memory consumption. In order to accomplish this we need to deal with different factors, not only processing incoming logs but also _formatting_ the data for our backends like Elasticsearch, which can be a little expensive as it requires a JSON representation.
+- Read Kubernetes/Docker log files from the file system or through Systemd Journal
+- Enrich logs with Kubernetes metadata
+- Deliver logs to third party storage services like Elasticsearch, InfluxDB, HTTP, etc.
 
-> Note: Fluent Bit uses a binary representation for logs, converting this to JSON requires enough memory for the process.
+This repository contains a set of Yaml files to deploy Fluent Bit which consider namespace, RBAC, Service Account, etc.
 
 ## Getting started
 
-Fluent Bit must be deployed as a DaemonSet, on that way it will be available on every node of your Kubernetes cluster.
+[Fluent Bit](http://fluentbit.io) must be deployed as a DaemonSet, so on that way it will be available on every node of your Kubernetes cluster. To get started:
 
-This repository contains two Yaml DaemonSet files:
+__FIXME__
 
-| Yaml file | Description |
-|-----------|-------------|
-| [fluent-bit-daemonset-elasticsearch](fluent-bit-daemonset-elasticsearch.yaml) | deploys a stable release of Fluent Bit. |
-
-The current DaemonSet points to this specific Docker Hub image:
-
-[0.11](https://hub.docker.com/r/fluent/fluent-bit-kubernetes-daemonset/tags/) fluent/fluent-bit-kubernetes-daemonset:0.11
-
-### Steps
-
-1. Make sure your Elasticsearch backend is running and can be reach through the hostname _elasticsearch-logging_. This value can be changed in the Yaml file
-
-2. Deploy the daemonset file from this repository:
-
-```bash
-$ kubectl apply -f fluent-bit-daemonset-elasticsearch.yaml
 ```
+$ kubectl create namespace logging
+$ kubectl create -f ./fluent-bit-service-account.yaml
+$ kubectl create -f ./fluent-bit-role.yaml
+$ kubectl create -f ./fluent-bit-role-binding.yaml
 
+```
 ## Details
 
 The default configuration of Fluent Bit makes sure of the following:
diff --git a/fluent-bit-daemonset-configmap.yaml b/fluent-bit-daemonset-configmap.yaml
deleted file mode 100644
index 0b85a2c..0000000
--- a/fluent-bit-daemonset-configmap.yaml
+++ /dev/null
@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fluent-bit-config-v1
-  namespace: kube-system
-data:
-  fluent-bit.conf: |
-    [SERVICE]
-    Flush          1
-    Daemon         Off
-    Log_Level      info
-    Parsers_File   parsers.conf
-
-    [INPUT]
-        Name           tail
-        Tag            kubernetes.*
-        Path           /var/log/containers/*.log
-        Parser         docker
-        DB             /var/log/flb_kube.db
-        Mem_Buf_Limit  5MB
-
-    [FILTER]
-        Name           kubernetes
-        Match          kubernetes.*
-        Kube_URL       https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
-        tls.verify      False
-        Merge_JSON_Log On
-
-    [OUTPUT]
-        Name   es
-        Match  *
-        Host   ${FLUENT_ELASTICSEARCH_HOST}
-        Port   ${FLUENT_ELASTICSEARCH_PORT}
-        Logstash_Format On
-        Retry_Limit False
-
-  parsers.conf: |
-
-    [PARSER]
-    Name   apache
-    Format regex
-    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-    Time_Key time
-    Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   apache2
-        Format regex
-        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   apache_error
-        Format regex
-        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
-
-    [PARSER]
-        Name   nginx
-        Format regex
-        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name   json-test
-        Format json
-        Time_Key time
-        Time_Format %d/%b/%Y:%H:%M:%S %z
-
-    [PARSER]
-        Name        docker
-        Format      json
-        Time_Key    time
-        Time_Format %Y-%m-%dT%H:%M:%S.%L
-        Time_Keep   On
-
-    [PARSER]
-        Name        syslog
-        Format      regex
-        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
-        Time_Key    time
-        Time_Format %b %d %H:%M:%S
diff --git a/fluent-bit-daemonset-elasticsearch.yaml b/fluent-bit-daemonset-elasticsearch.yaml
deleted file mode 100644
index 4e166a4..0000000
--- a/fluent-bit-daemonset-elasticsearch.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: fluent-bit
-  namespace: kube-system
-  labels:
-    k8s-app: fluent-bit-logging
-    version: v1
-    kubernetes.io/cluster-service: "true"
-spec:
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        k8s-app: fluent-bit-logging
-        version: v1
-        kubernetes.io/cluster-service: "true"
-      annotations:
-        # This annotation ensures that fluentd does not get evicted if the node
-        # supports critical pod annotation based priority scheme.
-        # Note that this does not guarantee admission on the nodes (#40573).
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-    spec:
-      serviceAccountName: fluent-bit-logging
-      containers:
-      - name: fluent-bit
-        image: globegitter/fluent-bit-kubernetes-daemonset:0.12.1
-        env:
-          - name:  FLUENT_ELASTICSEARCH_HOST
-            value: "elasticsearch-logging"
-          - name:  FLUENT_ELASTICSEARCH_PORT
-            value: "9200"
-        resources:
-          limits:
-            memory: 100Mi
-          requests:
-            cpu: 100m
-            memory: 100Mi
-        volumeMounts:
-        - name: varlog
-          mountPath: /var/log
-        - name: varlibdockercontainers
-          mountPath: /var/lib/docker/containers
-          readOnly: true
-      terminationGracePeriodSeconds: 10
-      # Do not schedule on master nodes
-      tolerations:
-        - key: "kubernetes.io/role"
-          operator: "Equal"
-          value: "master"
-          effect: "NoSchedule"
-      volumes:
-      - name: varlog
-        hostPath:
-          path: /var/log
-      - name: varlibdockercontainers
-        hostPath:
-          path: /var/lib/docker/containers
diff --git a/fluent-bit-rbac.yaml b/fluent-bit-rbac.yaml
deleted file mode 100644
index 1ce2cdd..0000000
--- a/fluent-bit-rbac.yaml
+++ /dev/null
@@ -1,44 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: fluentd-bit-logging
-  namespace: kube-system
-  labels:
-    run: fluentd-bit-logging
-
----
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: fluentd-bit-logging
-  labels:
-    run: fluentd-bit-logging
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - "namespaces"
-  - "pods"
-  verbs:
-  - "get"
-  - "watch"
-  - "list"
-
----
-
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: fluentd-bit-logging
-  labels:
-    run: fluentd-bit-logging
-subjects:
-- kind: ServiceAccount
-  name: fluentd-bit-logging
-  namespace: kube-system
-  apiGroup: ""
-roleRef:
-  kind: ClusterRole
-  name: fluentd-bit-logging
-  apiGroup: ""
diff --git a/fluent-bit-role-binding.yaml b/fluent-bit-role-binding.yaml
new file mode 100644
index 0000000..ff6f258
--- /dev/null
+++ b/fluent-bit-role-binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: fluent-bit-read
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fluent-bit-read
+subjects:
+- kind: ServiceAccount
+  name: fluent-bit
+  namespace: logging
diff --git a/fluent-bit-role.yaml b/fluent-bit-role.yaml
new file mode 100644
index 0000000..e55463a
--- /dev/null
+++ b/fluent-bit-role.yaml
@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: fluent-bit-read
+rules:
+- apiGroups: [""]
+  resources:
+  - namespaces
+  - pods
+  verbs: ["get", "list", "watch"]
diff --git a/fluent-bit-service-account.yaml b/fluent-bit-service-account.yaml
new file mode 100644
index 0000000..c541d81
--- /dev/null
+++ b/fluent-bit-service-account.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluent-bit
+  namespace: logging
diff --git a/output/elasticsearch/fluent-bit-ds.yaml b/output/elasticsearch/fluent-bit-ds.yaml
new file mode 100644
index 0000000..bc5e37c
--- /dev/null
+++ b/output/elasticsearch/fluent-bit-ds.yaml
@@ -0,0 +1,49 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: logging
+  labels:
+    k8s-app: fluent-bit-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluent-bit-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: fluent-bit
+        image: fluent/fluent-bit:0.12.7
+        env:
+        - name: FLUENT_ELASTICSEARCH_HOST
+          value: "10.0.2.2"
+        - name: FLUENT_ELASTICSEARCH_PORT
+          value: "9200"
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: fluent-bit-config
+          mountPath: /fluent-bit/etc/
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: fluent-bit-config
+        configMap:
+          name: fluent-bit-config
+      serviceAccountName: fluent-bit
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule