diff --git a/kubernetes/manifest/02_config/fluentbit.yml b/kubernetes/manifest/02_config/fluentbit.yml new file mode 100644 index 0000000..c94c4d1 --- /dev/null +++ b/kubernetes/manifest/02_config/fluentbit.yml @@ -0,0 +1,60 @@ +apiVersion: v1 +data: + custom_parsers.conf: | + [PARSER] + Name docker_no_time + Format json + Time_Keep Off + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + fluent-bit.conf: | + [SERVICE] + Daemon Off + Flush 1 + Log_Level info + Parsers_File /fluent-bit/etc/parsers.conf + Parsers_File /fluent-bit/etc/conf/custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Health_Check On + + [INPUT] + Name tail + Path /var/log/containers/*.log + multiline.parser docker, cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + [INPUT] + Name systemd + Tag host.* + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Read_From_Tail On + + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + [OUTPUT] + Name http + Match * + URI /api/default/default/_json + Host 10.43.204.152 + Port 5080 + tls Off + Format json + Json_date_key _timestamp + Json_date_format iso8601 + HTTP_User romain@ihr.org + HTTP_Passwd rootroot + compress gzip + +kind: ConfigMap +metadata: + name: fluentbit-config diff --git a/kubernetes/manifest/02_config/openobserve.yml b/kubernetes/manifest/02_config/openobserve.yml new file mode 100644 index 0000000..ff48cd7 --- /dev/null +++ b/kubernetes/manifest/02_config/openobserve.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: openobserve-config +data: + email: romain@ihr.org + password: rootroot diff --git a/kubernetes/manifest/03_openobserve/openobserve.yml b/kubernetes/manifest/03_openobserve/openobserve.yml new file mode 100644 index 0000000..96e39a7 --- /dev/null +++ b/kubernetes/manifest/03_openobserve/openobserve.yml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + name: openobserve +spec: + clusterIP: None + selector: + app: openobserve + ports: + - name: http + port: 5080 + targetPort: 5080 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openobserve + labels: + name: openobserve +spec: + serviceName: openobserve + replicas: 1 + selector: + matchLabels: + name: openobserve + app: openobserve + template: + metadata: + labels: + name: openobserve + app: openobserve + spec: + securityContext: + fsGroup: 2000 + runAsUser: 10000 + runAsGroup: 3000 + runAsNonRoot: true + containers: + - name: openobserve + image: public.ecr.aws/zinclabs/openobserve:latest + env: + - name: ZO_ROOT_USER_EMAIL + valueFrom: + configMapKeyRef: + name: openobserve-config + key: email + - name: ZO_ROOT_USER_PASSWORD + valueFrom: + configMapKeyRef: + name: openobserve-config + key: password + - name: ZO_DATA_DIR + value: /data + resources: + limits: + cpu: 4096m + memory: 2048Mi + requests: + cpu: 256m + memory: 50Mi + ports: + - containerPort: 5080 + name: http + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: openobserve-0 +spec: + selector: + statefulset.kubernetes.io/pod-name: openobserve-0 + type: NodePort + ports: + - port: 5080 + nodePort: 31000 diff --git a/kubernetes/manifest/03_producers/bgp.yml b/kubernetes/manifest/03_producers/bgp.yml deleted file mode 100644 index d78a8c0..0000000 --- a/kubernetes/manifest/03_producers/bgp.yml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: rrc12-collector - labels: - type: producer - type_name: bgp -spec: - schedule: "0 0 * * *" - jobTemplate: - spec: - template: - spec: - containers: - - name: rrc00-collector-container - image: internethealthreport/bgp - args: ["-t", "updates", "--collector", "rrc12"] - env: - - name: KAFKA_HOST - valueFrom: - configMapKeyRef: - name: kafka-config - key: bootstrap_servers - # resources: - # requests: - # cpu: "500m" - # memory: "100Mi" - # limits: - # cpu: "1" - # memory: "200Mi" - restartPolicy: Never diff --git a/kubernetes/manifest/04_fluentbit/fluentbit.yml b/kubernetes/manifest/04_fluentbit/fluentbit.yml new file mode 100644 index 0000000..7875831 --- /dev/null +++ b/kubernetes/manifest/04_fluentbit/fluentbit.yml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentbit + labels: + name: fluentbit +spec: + selector: + matchLabels: + name: fluentbit + template: + metadata: + labels: + name: fluentbit + spec: + containers: + - name: fluentbit + image: fluent/fluent-bit:latest + volumeMounts: + - name: varlog + mountPath: /var/log + - name: fluentbit-config + mountPath: /fluent-bit/etc/ + volumes: + - name: varlog + hostPath: + path: /var/log + - name: fluentbit-config + configMap: + name: fluentbit-config diff --git a/kubernetes/manifest/05_producers/bgp.yml b/kubernetes/manifest/05_producers/bgp.yml new file mode 100644 index 0000000..9d881f1 --- /dev/null +++ b/kubernetes/manifest/05_producers/bgp.yml @@ -0,0 +1,725 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc00-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc00-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc00"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc01-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc01-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc01"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc04-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc04-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc04"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc06-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc06-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc06"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc10-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc10-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc10"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc11-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc11-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc11"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc13-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc13-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc13"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc14-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc14-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc14"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc15-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc15-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc15"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc16-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc16-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc16"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc20-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc20-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc20"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc23-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc23-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc23"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc24-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc24-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc24"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc25-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc25-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc25"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc26-updates + labels: + type: producer + type_name: bgp +spec: + schedule: "*/15 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc26-updates-container + image: internethealthreport/bgp + args: ["-t", "updates", "--collector", "rrc26"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc00-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "29 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc00-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc00"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc01-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "23 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc01-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc01"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc04-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "24 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc04-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc04"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc06-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "21 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc06-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc06"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc10-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "25 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc10-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc10"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc11-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "26 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc11-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc11"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc13-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "27 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc13-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc13"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc15-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "28 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc15-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc15"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc16-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "25 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc16-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc16"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc20-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "26 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc20-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc20"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc23-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "23 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc23-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc23"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc24-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "27 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc24-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc24"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc25-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "27 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc25-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc25"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rrc26-rib + labels: + type: producer + type_name: bgp +spec: + schedule: "28 0 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: rrc26-rib-container + image: internethealthreport/bgp + args: ["-t", "ribs", "--collector", "rrc26"] + env: + - name: KAFKA_HOST + valueFrom: + configMapKeyRef: + name: kafka-config + key: bootstrap_servers + restartPolicy: Never +--- diff --git a/kubernetes/manifest/03_producers/peeringdb.yml b/kubernetes/manifest/05_producers/peeringdb.yml similarity index 100% rename from kubernetes/manifest/03_producers/peeringdb.yml rename to kubernetes/manifest/05_producers/peeringdb.yml