first commit

.gitignore

@@ -0,0 +1 @@
+volumes

Dockerfile

@@ -0,0 +1,42 @@
+FROM java:8-jre
+
+MAINTAINER Igor Artamonov <[email protected]>
+
+### ---- Install Elastic Search
+
+RUN \
+    mkdir /etc/service && \
+    mkdir /etc/service/elasticsearch /opt/elasticsearch && \
+    wget --progress=dot:mega -O - https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.5.2.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/elasticsearch";
+
+EXPOSE 9200 9300
+
+### ---- Install Logstash
+
+RUN \
+    mkdir /etc/service/logstash /opt/logstash && \
+    wget --progress=dot:mega -O - http://download.elastic.co/logstash/logstash/logstash-1.5.0.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/logstash";
+
+### ---- Install Kibana
+
+RUN \
+    mkdir /etc/service/kibana /opt/kibana && \
+    wget --progress=dot:mega -O - https://download.elastic.co/kibana/kibana/kibana-4.0.2-linux-x64.tar.gz \
+        | tar xzf - --strip-components=1 -C "/opt/kibana";
+
+EXPOSE 5601
+
+ADD server-start.sh /usr/local/bin/
+ADD logstash.conf /opt/logstash/
+
+VOLUME /var/inputlogs
+VOLUME /var/sincedb
+VOLUME /opt/elasticsearch/data/elasticsearch
+
+RUN \
+    chmod +x /usr/local/bin/server-start.sh
+
+CMD ["server-start.sh"]
+

README.md

@@ -0,0 +1,24 @@
+Kibana for Appengine Logs
+=========================
+
+
+Basic configuration of ELK (Elasticsearch,Logstash,Kibana) for Google Appengine Logs packed as Docker container
+
+Run:
+```bash
+docker run -d -t \
+  -p 9200:9200 -p 5601:5601 \
+  -v $(pwd)/volumes/inputlogs:/var/inputlogs \
+  -v $(pwd)/volumes/sincedb:/var/sincedb \
+  -v $(pwd)/volumes/elastic:/opt/elasticsearch/data/elasticsearch \
+  splix/appengine-logs-kibana
+```
+
+Download logs:
+```
+gsutil -m cp -R -n "gs://__MY_BUCKET_WITH_LOGS__/appengine.googleapis.com/request_log/" volumes/inputlogs/
+```
+
+Open Kibana at port 5601 (something like http://192.168.59.103:5601/) and setup `metadata.timestamp` as timestamp field. 
+
+Enjoy

logstash.conf

@@ -0,0 +1,51 @@
+input {
+  file {
+    path => "/var/inputlogs/**/*.json"
+    codec => json {}
+    sincedb_path => "/var/sincedb/inputlogs.db"
+
+    #debug => true
+    #start_position => "beginning"
+  }
+}
+
+filter {
+
+  date {
+    match => [ "[metadata][timestamp]", "ISO8601" ]
+  }
+
+  date {
+    match => [ "[protoPayload][startTime]", "ISO8601" ]
+  }
+
+  date {
+    match => [ "[protoPayload][endTime]", "ISO8601" ]
+  }
+
+  geoip {
+    source => "[protoPayload][ip]"
+  }
+
+  mutate {
+    gsub => ["[protoPayload][latency]", "s", ""]
+    remove_field => ["[protoPayload][@type]", "[protoPayload][combined]", "insertId", "log", "[metadata][serviceName]"]
+  }
+
+  mutate {
+    convert => ["[protoPayload][latency]", "float"]
+    convert => ["[protoPayload][cost]", "float"]
+    convert => ["[protoPayload][megaCycles]", "integer"]
+    convert => ["[protoPayload][responseSize]", "integer"]
+    convert => ["[protoPayload][status]", "integer"]
+  }
+}
+
+output {
+  elasticsearch {
+      host => localhost
+      embedded => false
+      port => 9200
+      protocol => http
+  }
+}

server-start.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo Start ElasticSearch...
+cd /opt/elasticsearch
+bin/elasticsearch -d
+
+sleep 10
+
+echo Start LogStash...
+cd /opt/logstash
+bin/logstash agent -f logstash.conf -l /var/log/logstash.log --debug &
+#bin/logstash agent -f logstash.conf --debug -e
+
+echo Start Kibana...
+cd /opt/kibana
+bin/kibana