From e4feb667917b8b13ee44a932ec1da27f4cc9b321 Mon Sep 17 00:00:00 2001
From: Igor Artamonov <igor@artamonov.ru>
Date: Wed, 10 Jun 2015 23:32:13 +0600
Subject: [PATCH] first commit

---
 .gitignore      |  1 +
 Dockerfile      | 42 ++++++++++++++++++++++++++++++++++++++++
 README.md       | 24 +++++++++++++++++++++++
 logstash.conf   | 51 +++++++++++++++++++++++++++++++++++++++++++++++++
 server-start.sh | 16 ++++++++++++++++
 5 files changed, 134 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Dockerfile
 create mode 100644 README.md
 create mode 100644 logstash.conf
 create mode 100644 server-start.sh

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..074546f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+volumes
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..8dbc616
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,42 @@
+FROM java:8-jre
+
+MAINTAINER Igor Artamonov <igor@artamonov.ru>
+
+### ---- Install Elastic Search
+
+RUN \
+    mkdir /etc/service && \
+    mkdir /etc/service/elasticsearch /opt/elasticsearch && \
+    wget --progress=dot:mega -O - https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.5.2.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/elasticsearch";
+
+EXPOSE 9200 9300
+
+### ---- Install Logstash
+
+RUN \
+    mkdir /etc/service/logstash /opt/logstash && \
+    wget --progress=dot:mega -O - http://download.elastic.co/logstash/logstash/logstash-1.5.0.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/logstash";
+
+### ---- Install Kibana
+
+RUN \
+    mkdir /etc/service/kibana /opt/kibana && \
+    wget --progress=dot:mega -O - https://download.elastic.co/kibana/kibana/kibana-4.0.2-linux-x64.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/kibana";
+
+EXPOSE 5601
+
+ADD server-start.sh /usr/local/bin/
+ADD logstash.conf /opt/logstash/
+
+VOLUME /var/inputlogs
+VOLUME /var/sincedb
+VOLUME /opt/elasticsearch/data/elasticsearch
+
+RUN \
+    chmod +x /usr/local/bin/server-start.sh
+
+CMD ["server-start.sh"]
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d20f4cd
--- /dev/null
+++ b/README.md
@@ -0,0 +1,24 @@
+Kibana for Appengine Logs
+=========================
+
+
+Basic configuration of ELK (Elasticsearch,Logstash,Kibana) for Google Appengine Logs packed as Docker container
+
+Run:
+```bash
+docker run -d -t \
+  -p 9200:9200 -p 5601:5601 \
+  -v $(pwd)/volumes/inputlogs:/var/inputlogs \
+  -v $(pwd)/volumes/sincedb:/var/sincedb \
+  -v $(pwd)/volumes/elastic:/opt/elasticsearch/data/elasticsearch \
+  splix/appengine-logs-kibana
+```
+
+Download logs:
+```
+gsutil -m cp -R -n "gs://__MY_BUCKET_WITH_LOGS__/appengine.googleapis.com/request_log/" volumes/inputlogs/
+```
+
+Open Kibana at port 5601 (something like http://192.168.59.103:5601/) and setup `metadata.timestamp` as timestamp field. 
+
+Enjoy
diff --git a/logstash.conf b/logstash.conf
new file mode 100644
index 0000000..6335bb2
--- /dev/null
+++ b/logstash.conf
@@ -0,0 +1,51 @@
+input {
+  file {
+    path => "/var/inputlogs/**/*.json"
+    codec => json {}
+    sincedb_path => "/var/sincedb/inputlogs.db"
+
+    #debug => true
+    #start_position => "beginning"
+  }
+}
+
+filter {
+
+  date {
+    match => [ "[metadata][timestamp]", "ISO8601" ]
+  }
+
+  date {
+    match => [ "[protoPayload][startTime]", "ISO8601" ]
+  }
+
+  date {
+    match => [ "[protoPayload][endTime]", "ISO8601" ]
+  }
+
+  geoip {
+    source => "[protoPayload][ip]"
+  }
+
+  mutate {
+    gsub => ["[protoPayload][latency]", "s", ""]
+    remove_field => ["[protoPayload][@type]", "[protoPayload][combined]", "insertId", "log", "[metadata][serviceName]"]
+  }
+
+  mutate {
+    convert => ["[protoPayload][latency]", "float"]
+    convert => ["[protoPayload][cost]", "float"]
+    convert => ["[protoPayload][megaCycles]", "integer"]
+    convert => ["[protoPayload][responseSize]", "integer"]
+    convert => ["[protoPayload][status]", "integer"]
+  }
+}
+
+output {
+  elasticsearch {
+      host => localhost
+      embedded => false
+      port => 9200
+      protocol => http
+  }
+}
diff --git a/server-start.sh b/server-start.sh
new file mode 100644
index 0000000..278faf1
--- /dev/null
+++ b/server-start.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo Start ElasticSearch...
+cd /opt/elasticsearch
+bin/elasticsearch -d
+
+sleep 10
+
+echo Start LogStash...
+cd /opt/logstash
+bin/logstash agent -f logstash.conf -l /var/log/logstash.log --debug &
+#bin/logstash agent -f logstash.conf --debug -e
+
+echo Start Kibana...
+cd /opt/kibana
+bin/kibana