Add VM for cache and /vagrant/repos as well.

Fix virtualenv creation for sentry (python)

Author: lighta

Vagrantfile

@@ -5,12 +5,21 @@ Vagrant.configure VAGRANTFILE_API_VERSION do |config|
   config.vm.box = "trusty64"
   config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box"
 
+  #find . -type f -print0 | xargs -0 dos2unix
+  
   config.vm.provider :virtualbox do |v|
     v.customize ["modifyvm", :id, "--cpus", "2"]
     v.customize ["modifyvm", :id, "--ioapic", "on"]
     v.customize ["modifyvm", :id, "--natdnsproxy1", "on", "--natdnshostresolver1", "on"]
   end
 
+   config.vm.define "cacher" do |box|
+    box.vm.hostname = "VmCache"
+    box.vm.network :private_network, ip: "192.168.12.13"
+#   box.vm.network "forwarded_port", guest: 3142, host: 3142, auto_correct: true
+    box.vm.provision "shell", inline: "cd /vagrant && ./install_apt-cache.sh Server"
+  end
+  
   config.vm.define "monitoring" do |box|
     box.vm.hostname = "monitoring"
     box.vm.network :private_network, ip: "192.168.12.10"
@@ -19,13 +28,15 @@ Vagrant.configure VAGRANTFILE_API_VERSION do |config|
       v.customize ["modifyvm", :id, "--memory", "2048"]
     end
 
+    box.vm.provision "shell", inline: "cd /vagrant && ./install_apt-cache.sh client forcedIP 192.168.12.13"
     box.vm.provision :shell, inline: "cd /vagrant && ./install-all-server-with-kale.sh"
   end
 
   config.vm.define "app1" do |box|
     box.vm.hostname = "app1"
     box.vm.network :private_network, ip: "192.168.12.11"
 
+    box.vm.provision "shell", inline: "cd /vagrant && ./install_apt-cache.sh client forcedIP 192.168.12.13"
     box.vm.provision :shell, inline: "cd /vagrant && ./install-all-client.sh"
   end
 end

env.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 BASE_PATH=$(pwd)
+REPOS_PATH=$BASE_PATH/repos
+USE_CACHE=true #should we keep file in repos ?
 
 export DEBIAN_FRONTEND=noninteractive
 

etc/apt-cacher-ng/acng.conf

@@ -6,12 +6,20 @@ set -eux
 
 source env.sh
 BUILD_PATH=$BASE_PATH/build
-
+BASE_URL_TAR=http://sensuapp.org/docs/0.12/tools
+SSL_TAR=ssl_certs.tar
+	
 mkdir -p $BUILD_PATH
 cd $BUILD_PATH
 
-curl -O http://sensuapp.org/docs/0.12/tools/ssl_certs.tar
-tar xf ssl_certs.tar
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$SSL_TAR ]; then wget -P $REPOS_PATH $BASE_URL_TAR/$SSL_TAR; fi
+	tar xf $REPOS_PATH/$SSL_TAR &> rabbitssl.log
+else
+	curl -O $BASE_URL_TAR
+	tar xf $SSL_TAR &> rabbitssl.log
+fi
+
 
 cd ssl_certs
 

generate-sensu-ssl.sh

@@ -12,9 +12,12 @@
 set -eux
 
 source env.sh
-
+FLUME_BASE_URL=http://apache.osuosl.org/flume
+FLUME_TAR=apache-flume-${FLUME_VERSION}-bin.tar.gz
+FLUME_INST_PATH=/opt/flume
+	
 # Prepare user and directories
-useradd -d /opt/flume -M flume
+useradd -d $FLUME_INST_PATH -M flume
 
 mkdir -p /etc/flume/conf.d /var/lib/flume /var/log/flume
 chown flume:flume /var/lib/flume /var/log/flume
@@ -25,14 +28,17 @@ apt-get install -yq openjdk-7-jre-headless
 
 # Install Flume
 cd /tmp
-
-curl -sOL http://apache.osuosl.org/flume/${FLUME_VERSION}/apache-flume-${FLUME_VERSION}-bin.tar.gz
-tar xf apache-flume-${FLUME_VERSION}-bin.tar.gz
-mv apache-flume-${FLUME_VERSION}-bin /opt/flume
+sudo mkdir -p $FLUME_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$FLUME_TAR ]; then wget -P $REPOS_PATH $FLUME_BASE_URL/$FLUME_TAR; fi
+	sudo tar --strip-components=1 -xzvf $REPOS_PATH/$FLUME_TAR -C $FLUME_INST_PATH &> flume.log
+else
+	curl $FLUME_BASE_URL/$FLUME_TAR | sudo tar --strip-components=1 -xzv -C $FLUME_INST_PATH &> flume.log
+fi
 
 # Copy libraries necessary for the Elasticsearch sink
-cp /usr/share/elasticsearch/lib/lucene-* /opt/flume/lib
-cp /usr/share/elasticsearch/lib/elasticsearch-* /opt/flume/lib
+cp /usr/share/elasticsearch/lib/lucene-* $FLUME_INST_PATH/lib
+cp /usr/share/elasticsearch/lib/elasticsearch-* $FLUME_INST_PATH/lib
 
 # Configure Flume
 cp $BASE_PATH/etc/flume/* /etc/flume

install-flume.sh

@@ -14,14 +14,21 @@
 set -eux
 
 source env.sh
-
+GRAF_BASE_URL=http://grafanarel.s3.amazonaws.com
+GRAF_TAR=grafana-${GRAFANA_VERSION}.tar.gz
+GRAF_INST_PATH=/usr/share/grafana
+	
 cd /tmp
 
-curl -sOL http://grafanarel.s3.amazonaws.com/grafana-${GRAFANA_VERSION}.tar.gz
-tar xf grafana-${GRAFANA_VERSION}.tar.gz
-cp -R grafana-${GRAFANA_VERSION} /usr/share/grafana
+sudo mkdir -p $GRAF_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$GRAF_TAR ]; then wget -P $REPOS_PATH $GRAF_BASE_URL/$GRAF_TAR; fi
+	sudo tar --strip-components=1 -xzvf $REPOS_PATH/$GRAF_TAR -C $GRAF_INST_PATH &> grafana.log
+else
+	curl $GRAF_BASE_URL/$GRAF_TAR | sudo tar --strip-components=1 -xzv -C $GRAF_INST_PATH &> grafana.log
+fi
 
-cp $BASE_PATH/usr/share/grafana/config.js /usr/share/grafana
+cp $BASE_PATH/usr/share/grafana/config.js $GRAF_INST_PATH
 
 apt-get install -yq apache2
 

install-grafana.sh

@@ -16,7 +16,10 @@
 set -eux
 
 source env.sh
-
+GRAY_BASE_URL=https://github.com/Graylog2/graylog2-server/releases/download/${GRAYLOG_VERSION}/
+GRAY_TAR=graylog2-server-${GRAYLOG_VERSION}.tgz
+GRAY_INST_PATH=/usr/share/graylog2-server
+	
 apt-get install -yq openjdk-7-jre-headless
 
 cd /tmp
@@ -26,9 +29,13 @@ useradd -s /bin/false -d /var/lib/graylog2 -m graylog2
 mkdir -p /var/log/graylog2
 chown graylog2:graylog2 /var/log/graylog2
 
-curl -sOL https://github.com/Graylog2/graylog2-server/releases/download/${GRAYLOG_VERSION}/graylog2-server-${GRAYLOG_VERSION}.tgz
-tar xf graylog2-server-${GRAYLOG_VERSION}.tgz
-cp -R graylog2-server-${GRAYLOG_VERSION} /usr/share/graylog2-server
+sudo mkdir -p $GRAY_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$GRAY_TAR ]; then wget -P $REPOS_PATH $GRAY_BASE_URL/$GRAY_TAR; fi
+	sudo tar --strip-components=1 -xzvf $REPOS_PATH/$GRAY_TAR -C $GRAY_INST_PATH &> gray.log
+else
+	curl $GRAY_BASE_URL/$GRAY_TAR | sudo tar --strip-components=1 -xzv -C $GRAY_INST_PATH &> gray.log
+fi
 
 mkdir -p /etc/graylog2
 

install-graylog2-server.sh

@@ -13,12 +13,18 @@
 set -eux
 
 source env.sh
-
+GRAYDASH_BASE_URL=https://github.com/Graylog2/graylog2-stream-dashboard/releases/download/0.90/
+GRAYDASH_TAR=graylog2-stream-dashboard-${GRAYLOG_STREAM_DASHBOARD_VERSION}.tgz
+GRAYDASH_INST_PATH=/usr/share/graylog2-stream-dashboard
+	
 cd /tmp
-
-curl -sOL https://github.com/Graylog2/graylog2-stream-dashboard/releases/download/0.90/graylog2-stream-dashboard-${GRAYLOG_STREAM_DASHBOARD_VERSION}.tgz
-tar xf graylog2-stream-dashboard-${GRAYLOG_STREAM_DASHBOARD_VERSION}.tgz
-cp -R graylog2-stream-dashboard-${GRAYLOG_STREAM_DASHBOARD_VERSION} /usr/share/graylog2-stream-dashboard
+sudo mkdir -p $GRAYDASH_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$GRAYDASH_TAR ]; then wget -P $REPOS_PATH $GRAYDASH_BASE_URL/$GRAYDASH_TAR; fi
+	sudo tar --strip-components=1 -xzvf $REPOS_PATH/$GRAYDASH_TAR -C $GRAYDASH_INST_PATH &> graydash.log
+else
+	curl $GRAYDASH_BASE_URL/$GRAYDASH_TAR | sudo tar --strip-components=1 -xzv -C $GRAYDASH_INST_PATH &> graydash.log
+fi
 
 apt-get install -yq apache2
 

install-graylog2-stream-dashboard.sh

@@ -11,7 +11,10 @@
 set -eux
 
 source env.sh
-
+GRAY2_BASE_URL=https://github.com/Graylog2/graylog2-web-interface/releases/download/${GRAYLOG_VERSION}
+GRAY2_TAR=graylog2-web-interface-${GRAYLOG_VERSION}.tgz
+GRAY2_INST_PATH=/usr/share/graylog2-web-interface
+	
 apt-get install -yq openjdk-7-jre-headless
 
 cd /tmp
@@ -21,9 +24,13 @@ useradd -s /bin/false -d /var/lib/graylog2 -m graylog2
 mkdir -p /var/log/graylog2
 chown graylog2:graylog2 /var/log/graylog2
 
-curl -sOL https://github.com/Graylog2/graylog2-web-interface/releases/download/${GRAYLOG_VERSION}/graylog2-web-interface-${GRAYLOG_VERSION}.tgz
-tar xf graylog2-web-interface-${GRAYLOG_VERSION}.tgz
-cp -R graylog2-web-interface-${GRAYLOG_VERSION} /usr/share/graylog2-web-interface
+mkdir -p $GRAY2_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$GRAY2_TAR ]; then wget -P $REPOS_PATH $GRAY2_BASE_URL/$GRAY2_TAR; fi
+	tar --strip-components=1 -xzvf $REPOS_PATH/$GRAY2_TAR -C $GRAY2_INST_PATH &> gray2.log
+else
+	curl $GRAY2_BASE_URL/$GRAY2_TAR | tar --strip-components=1 -xzv -C $GRAY2_INST_PATH &> gray2.log
+fi
 
 mkdir -p /etc/graylog2
 

install-graylog2-web.sh

@@ -13,7 +13,9 @@
 set -eux
 
 source env.sh
-HBASE_PREFIX=/opt/hbase
+HBASE_BASE_URL=http://apache.osuosl.org/hbase/hbase-${HBASE_VERSION}
+HBASE_TAR=hbase-${HBASE_VERSION}-hadoop2-bin.tar.gz
+HBASE_INST_PATH=/opt/hbase
 
 useradd -d /var/lib/hbase -s /bin/bash -m hbase
 
@@ -22,13 +24,16 @@ apt-get install -yq openjdk-7-jre-headless supervisor
 
 # Install HBase
 cd /tmp
-
-curl -sOL http://apache.osuosl.org/hbase/hbase-${HBASE_VERSION}/hbase-${HBASE_VERSION}-hadoop2-bin.tar.gz
-tar xf hbase-${HBASE_VERSION}-hadoop2-bin.tar.gz
-mv hbase-${HBASE_VERSION}-hadoop2 /opt/hbase
+mkdir -p $HBASE_INST_PATH 
+if [ "x$USE_CACHE" == "xtrue" ]; then
+	if [ ! -e $REPOS_PATH/$HBASE_TAR ]; then wget -P $REPOS_PATH $HBASE_BASE_URL/$HBASE_TAR; fi
+	tar --strip-components=1 -xzvf $REPOS_PATH/$HBASE_TAR -C $HBASE_INST_PATH &> hbase.log
+else
+	curl $HBASE_BASE_URL/$HBASE_TAR | tar --strip-components=1 -xzv -C $HBASE_INST_PATH &> hbase.log
+fi
 
 # Configure HBase
-cp $BASE_PATH/opt/hbase/conf/* $HBASE_PREFIX/conf
+cp $BASE_PATH/opt/hbase/conf/* $HBASE_INST_PATH/conf
 
 # Start with supervisord
 cp $BASE_PATH/etc/supervisor/conf.d/hbase.conf /etc/supervisor/conf.d