devbox-photon/Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

# Copyright 2015 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, without warranties or
# conditions of any kind, EITHER EXPRESS OR IMPLIED.  See the License for the
# specific language governing permissions and limitations under the License.

Vagrant.configure("2") do |config|
  # ================ VARIABLES ================
  ESXCLOUD_DIR="/var/esxcloud"
  DATASTORE_DIR="/tmp/devbox_datastore"
  API_SHARED_SECRET="#{SecureRandom.uuid}"
  DEPLOYMENT_ID=ENV["RANDOM_GENERATED_DEPLOYMENT_ID"] || "test-deployment"

  DOCKER_PORT=ENV["DOCKER_PORT"] || 2375

  # Output directory for scripts generated by vagrantfile
  SCRIPT_DIR="/home/vagrant/bin"

  # Extensions for config files that should be processed by mustache
  CONFIG_EXTS="{yml,config,js,sh,sql,cfg,json}"

  # Filter for selecting dynamicParamters from test JSON
  # Remove the deployment ID so it can be overridden by env var
  CONFIG_JQ_FILTER='.dynamicParameters | with_entries(select (.key != "DEPLOYMENT_ID"))'

  SERVICE_NAME_MAP = {
    :photon_controller_core => "photon-controller-core",
  }

  SERVICE_CONFIG_MAP = {
    :photon_controller_core => "photon-controller-core",
  }

  # Root of the git repo and the current git hash, used by agent VIB build
  GIT_ROOT=%x(git rev-parse --show-toplevel).strip
  GIT_REVISION=%x(git rev-parse HEAD).strip

  # ================ VM DEFINITIONS ================

  #
  # photon devbox supports virtualbox only for now
  #
  config.vm.provider "virtualbox"

  restart_always = "--restart=always"
  if ENV["NO_RESTART_ALWAYS"]
    restart_always = ""
  end

  # Where to save dynamic parameters
  dynamic_params_file = "/home/vagrant/dynamic-params.json"

  # Prepare build directory
  config.vm.provision :shell, :privileged => false, :inline => <<-EOS
    set -ex
    sudo mkdir -p #{DATASTORE_DIR}
    sudo chown -R vagrant:vagrant $HOME
  EOS

  # ================ PARAMETERS AND SYNC DIRECTORIES ================
  #
  # Configure sync-ed directories
  #
  config.vm.synced_folder "..", "/devbox_data"
  config.vm.synced_folder ".", "/vagrant", mount_options: ["dmode=777", "fmode=775"]
  config.vm.synced_folder "#{Dir.home}/.gradle/#{ENV["JOB_NAME"] || "devbox-local"}", "/gradle", create: true
  config.vm.synced_folder "stats_data", "/stats_data", mount_options: ["dmode=777,fmode=777"]

  #
  # Provided for convenience only; don't take a strong dependency on it
  #
  config.vm.synced_folder Dir.home, "/home_data"

  # Clean utility scripts (file provision fails to do this)
  config.vm.provision :shell, :inline => "rm -rf #{SCRIPT_DIR}/"
  # Copy utility scripts
  config.vm.provision "file", source: "scripts/bin/", destination: "#{SCRIPT_DIR}/"
  # Copy profile scripts
  config.vm.provision :shell, :inline => "cp -fa /vagrant/scripts/profile/* /etc/profile.d/"

  # ================ PROXY CONFIGURATION ================

  #
  # Our environments need proxies to be set explicitly, we do it by
  # adding /etc/profile.d/proxy.sh script. Vagrant VMs don't pick up
  # those automatically, so we need to copy it to Vagrant VM.
  #
  if ENV["PROXY_PROFILE"]
    proxy_script = "/etc/profile.d/proxy.sh"
    if File.exists?(proxy_script)
      FileUtils.cp(proxy_script, ".")
      config.vm.provision :shell, :inline => <<-EOS
          set -ex
          cp /vagrant/proxy.sh /etc/profile.d
          echo "\nexport no_proxy=#{network_ip},.sock,\\$no_proxy" >> /etc/profile.d/proxy.sh
      EOS
    else
      abort("PROXY_PROFILE is set but '#{proxy_script}' doesn't exist")
    end
  end

  # Install jsonlint, useful for catching errors in dynamic_params early
  config.vm.provision :shell, :inline  => <<-EOM
    tdnf makecache
    tdnf install -y gmp-devel
    gem install jsonlint --no-rdoc --no-ri
  EOM

  if ENV['ENABLE_AUTH'] == 'true'
  #
  # VM definition for lightwave
  #
  config.vm.define "lightwave" do |lightwave|
    lightwave.vm.guest = :photon
    lightwave.vm.box = ENV['DEVBOX_NAME'] || "photon-devbox-60"
    lightwave.vm.box_url = ENV['DEVBOX_URL'] ||
        "https://s3.amazonaws.com/photon-platform/artifacts/devbox/60/photon-devbox.box"

    #
    # Configure memory and nat dns
    #
    config.vm.provider "virtualbox" do |v|
      v.customize ["modifyvm", :id, "--memory", "1024"]
      v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
      v.customize ["modifyvm", :id, "--cpus", "1"]
      v.customize ['modifyvm', :id, '--acpi', 'off'] # needed to run PhotonOS RC1 in vagrant under stress
    end

    unless ENV["NO_PORT_FORWARDING"]
      #Lightwave
      lightwave.vm.network :forwarded_port, guest: 22, host: 9022,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 53, host: 9053,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 88, host: 9188,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 389, host: 9389,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 636, host: 9636,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 2012, host: 2012,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 2014, host: 2014,
          auto_correct: true
      lightwave.vm.network :forwarded_port, guest: 2020, host: 2020,
          auto_correct: true
    end

    # Configure network
    if ENV["PUBLIC_NETWORK_IP"]
      public_network_ip = ENV["PUBLIC_LW_NETWORK_IP"]
      network_ip = public_network_ip
      bridge_network = ENV["BRIDGE_NETWORK"]
      public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"] || "255.255.255.128"
      public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"]
      lightwave.vm.network :public_network, bridge: bridge_network, ip: public_network_ip, netmask: public_network_netmask

      public_network_gateway = ENV["PUBLIC_NETWORK_GATEWAY"]
      # Assign correct default gateway to public network
      lightwave.vm.provision :shell, :inline => <<-EOS
        set -ex
        ip route del default
        ip route add default via #{public_network_gateway} dev enp0s8
      EOS
      ENV["DOCKER_URL"] = ENV["DOCKER_URL"] || "http://#{public_network_ip}:#{DOCKER_PORT}"
    else
      private_network_lw_ip = ENV["PRIVATE_NETWORK_LW_IP"] || "172.31.253.67"
      lightwave.vm.network :private_network, ip: private_network_lw_ip
    end

    #
    # Configure hostname
    #
    lightwave.vm.hostname = "lightwave"

    lightwave.vm.provision :shell, :privileged => false, :inline => <<-EOS
      set -xe
      sleep 15 # docker takes longer to come up in RC1
      docker tag devbox/compileservices photon/compileservices
    EOS

    dynamic_params = %Q(
    {)


    if ENV['LW_DOMAIN_NAME']
      dynamic_params += %Q(\n  "LIGHTWAVE_DOMAIN" : "#{ENV['LW_DOMAIN_NAME']}")
    else
      abort("auth is enabled for devbox lightwave, but domain is not set")
    end

    if ENV['LW_PASSWORD']
      dynamic_params += %Q(,\n  "LIGHTWAVE_PASSWORD" : "#{ENV['LW_PASSWORD']}")
    else
      abort("auth is enabled for devbox lightwave, but password is not set")
    end

    if ENV["PUBLIC_NETWORK_IP"]
      if ENV["PUBLIC_LW_NETWORK_IP"]
        # The hostname is set to the IP since Photon Controller does not have DNS set
        # to Lightwave yet, but Photon Controller during auth enabled needs to
        # verify the hostname matches the Lightwave server certificate.
        dynamic_params += %Q(,\n  "LIGHTWAVE_HOSTNAME" : "#{ENV['PUBLIC_LW_NETWORK_IP']}")
      else
        abort("auth is enabled for devbox lightwave, but hostname is not set")
      end
    else
      private_network_lw_ip = ENV["PRIVATE_NETWORK_LW_IP"] || "172.31.253.67"
      dynamic_params += %Q(,\n  "LIGHTWAVE_HOSTNAME" : "#{private_network_lw_ip}")
    end

    dynamic_params += "\n}"

    # Write dynamic_params file
    lightwave.vm.provision :shell, :privileged => false, :inline => <<-EOS
cat > #{dynamic_params_file} <<'EOM'
  #{dynamic_params}
EOM
  EOS

    # Validate dynamic_params file, catch errors early
    lightwave.vm.provision :shell, :inline => <<-EOS
      printf "Validating dynamic params JSON file:"
      jsonlint #{dynamic_params_file}
      if [[ $? != 0 ]]; then
        echo "Validation failed for file:"
        cat #{dynamic_params_file}
        exit 1
      fi
    EOS

    #
    # Generate Lightwave container start script
    #
    lightwave_config_file="/devbox_data/java/containers/lightwave/config/lightwave_test.json"
    lightwave.vm.provision :shell, :privileged => true, :inline => <<-EOS
    set -ex
out="#{SCRIPT_DIR}/start-lightwave"
cat > $out <<-'EOM'
  #!/bin/bash

  # Setup hostname for this host, which is required to run the Lightwave server.
  sed -i s/photon-.*/"lightwave.#{ENV['LW_DOMAIN_NAME']} lightwave"/ /etc/hosts
  hostnamectl set-hostname lightwave
  hostnamectl set-hostname --static lightwave

  docker build -t vmware/lightwave-sts /devbox_data/java/containers/lightwave

  config_build_dir="/devbox_data/java/containers/lightwave/build/config"
  mkdir -p $config_build_dir
  combined_params=$(jq -s '.[0] * (.[1]#{CONFIG_JQ_FILTER})' #{dynamic_params_file} #{lightwave_config_file})

  cd $(dirname #{lightwave_config_file})
  for file in ./*.#{CONFIG_EXTS}
  do
    if [ -f $file ]; then
      mustache - $file <<< "$combined_params" > "$config_build_dir/$file"
    fi
  done
EOM
cat >> $out <<-EOM
  $(gen-docker-stop #{lightwave_config_file}) &> /dev/null || true
  $(gen-docker-rm #{lightwave_config_file}) &> /dev/null || true
  $(gen-docker-run #{lightwave_config_file} /usr/sbin/init)
  echo "Start lightwave"

EOM
EOS
    if ENV['PUBLIC_LW_NETWORK_IP']
      lightwave.vm.provision :shell, :privileged => true, :inline => <<-EOS
    set -ex
out="#{SCRIPT_DIR}/start-lightwave"
cat >> $out <<-EOM
  # Set a static (public) IP for Lightwave
  # Note: Gateway needs to be set to only get a static IP, otherwise a secondary IP will
  # be made by networkd, confusing the Lightwave setup.
  echo "DNS=#{ENV['PUBLIC_LW_NETWORK_IP']}" >> /etc/systemd/network/10-dhcp-enp0s8.network
  echo "Domains=#{ENV['LW_DOMAIN_NAME']}" >> /etc/systemd/network/10-dhcp-enp0s8.network
  echo "Gateway=#{ENV["PUBLIC_NETWORK_GATEWAY"]}" >> /etc/systemd/network/10-dhcp-enp0s8.network

EOM
EOS
    else
    private_network_lw_ip = ENV["PRIVATE_NETWORK_LW_IP"] || "172.31.253.67"
    lightwave.vm.provision :shell, :privileged => true, :inline => <<-EOS
    set -ex
out="#{SCRIPT_DIR}/start-lightwave"
cat >> $out <<-EOM
  # Set a static (private) IP for Lightwave
  echo "DNS=#{private_network_lw_ip}" >> /etc/systemd/network/10-dhcp-enp0s8.network
  echo "Domains=#{ENV['LW_DOMAIN_NAME']}" >> /etc/systemd/network/10-dhcp-enp0s8.network

EOM
EOS
    end
    lightwave.vm.provision :shell, :privileged => true, :inline => <<-EOS
    set -ex
out="#{SCRIPT_DIR}/start-lightwave"
cat >> $out <<-'EOM'
  # Restart to pick up the networking changes.
  systemctl restart systemd-networkd systemd-resolved

  chmod +x /devbox_data/java/containers/lightwave/config/add_lightwave_users.sh
  chmod +x /devbox_data/java/containers/lightwave/config/update_token_expiration.sh

  echo "Checking to connect to Lightwave REST client to start"
  attempts=1
  reachable="false"
  total_attempts=50
  while [ $attempts -lt $total_attempts ] && [ $reachable != "true" ]; do
    http_code=$(curl -w "%{http_code}" -s -X GET --insecure https://lightwave.#{ENV['LW_DOMAIN_NAME']})
    # The curl returns 000 when it fails to connect to the lightwave server
    if [ $http_code -eq 000 ]; then
      echo "Lightwave REST server not reachable (attempt $attempts/$total_attempts), will try again."
      attempts=$[$attempts+1]
      sleep 5
    else
      reachable="true"
      echo "Adding Lightwave users."
      /devbox_data/java/containers/lightwave/config/add_lightwave_users.sh #{ENV['LW_PASSWORD']}
      echo "Updating tenant token expiration time."
      /devbox_data/java/containers/lightwave/config/update_token_expiration.sh #{ENV['LW_PASSWORD']} #{ENV['LW_DOMAIN_NAME']}
      break
    fi
  done
  if [ $attempts -eq $total_attempts ]; then
    echo "Could not connect to Lightwave REST client after $total_attempts attempts, users not added and token expiration not updated."
    exit 1
  fi
EOM
chmod +x $out
$out
    EOS
  end
  end

  #
  # VM definition for photon
  #
  config.vm.define "photon", primary: true do |photon|
    photon.vm.guest = :photon
    photon.vm.box = ENV['DEVBOX_NAME'] || "photon-devbox-60"
    photon.vm.box_url = ENV['DEVBOX_URL'] ||
      "https://s3.amazonaws.com/photon-platform/artifacts/devbox/60/photon-devbox.box"

    #
    # Configure memory and nat dns
    #
    config.vm.provider "virtualbox" do |v|
      v.customize ["modifyvm", :id, "--memory", ENV["DEVBOX_PHOTON_MEMORY"] || "2048"]
      v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
      v.customize ["modifyvm", :id, "--cpus", ENV["DEVBOX_PHOTON_CPUS"] || "3"]
      v.customize ['modifyvm', :id, '--acpi', 'off'] # needed to run PhotonOS RC1 in vagrant under stress
    end

    unless ENV["NO_PORT_FORWARDING"]
      # Load Balancer
      photon.vm.network :forwarded_port, guest: 8080, host: 9180,
        auto_correct: true
      # APIFE
      photon.vm.network :forwarded_port, guest: 9000, host: 9080,
        auto_correct: true
      # Graphite
      photon.vm.network :forwarded_port, guest: 8082, host: 9082,
        auto_correct: true
      # Carbon (graphite db backend)
      photon.vm.network :forwarded_port, guest: 2003, host: 9083,
        auto_correct: true
      photon.vm.network :forwarded_port, guest: 2004, host: 9084,
        auto_correct: true
      # Zookeeper
      photon.vm.network :forwarded_port, guest: 2181, host: 9085,
        auto_correct: true
      # Deployer
      photon.vm.network :forwarded_port, guest: 18000, host: 9088,
        auto_correct: true
      photon.vm.network :forwarded_port, guest: 18001, host: 9089,
        auto_correct: true
      # Load Balancer
      photon.vm.network :forwarded_port, guest: 443, host: 9089,
        auto_correct: true
      photon.vm.network :forwarded_port, guest: 28080, host: 9090,
          auto_correct: true

      # Graphite web interface port
      photon.vm.network :forwarded_port, guest: 8000, host: 8000
      # Graphite carbon-cache pickle receiver (stats endpoint for agent)
      photon.vm.network :forwarded_port, guest: 2004, host: 2004

      # Expose Docker for the Gradle build on the host
      photon.vm.network :forwarded_port, guest: 2375, host: DOCKER_PORT
    end

    photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
      set -xe
      sleep 15 # docker takes longer to come up in RC1
      docker tag devbox/compileservices photon/compileservices
    EOS


    # ================ NETWORK CONFIGURATION ================

    #
    # Configure network
    #
    if ENV["PUBLIC_NETWORK_IP"]
      public_network_ip = ENV["PUBLIC_NETWORK_IP"]
      network_ip = public_network_ip
      bridge_network = ENV["BRIDGE_NETWORK"]
      public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"] || "255.255.255.128"
      public_network_netmask = ENV["PUBLIC_NETWORK_NETMASK"]
      photon.vm.network :public_network, bridge: bridge_network, ip: public_network_ip, netmask: public_network_netmask

      public_network_gateway = ENV["PUBLIC_NETWORK_GATEWAY"]
      # Assign correct default gateway to public network
      photon.vm.provision :shell, :inline => <<-EOS
        set -ex
        ip route del default
        ip route add default via #{public_network_gateway} dev enp0s8
      EOS
      ENV["DOCKER_URL"] = ENV["DOCKER_URL"] || "http://#{public_network_ip}:#{DOCKER_PORT}"
    else
      private_network_ip = ENV["PRIVATE_NETWORK_IP"] || "172.31.253.66"
      network_ip = private_network_ip
      photon.vm.network :private_network, ip: private_network_ip
    end

    #
    # Configure hostname
    #
    photon.vm.hostname = "devbox"


    dynamic_params = %Q(
    {
      "REGISTRATION_ADDRESS" : "#{network_ip}",
      "TASK_EXPIRATION_THRESHOLD": "3 minutes",
      "TASK_EXPIRATION_SCAN_INTERVAL": "3 minutes",
      "SHARED_SECRET": "#{API_SHARED_SECRET}",
      "DEPLOYMENT_ID": "#{DEPLOYMENT_ID}",
      "LOAD_BALANCER_SERVERS": [ { "serverName": "devbox-photon-controller-core", "serverAddress": "#{network_ip}:9000" } ],
      "ZOOKEEPER_INSTANCES": [ { "zookeeperInstance": "server.1=#{network_ip}:2888:3888" } ],
      "APIFE_IP": "#{network_ip}",
      "ZOOKEEPER_QUORUM": "#{network_ip}:2181",
      "MGMT_API_HTTP_SERVERS": [ { "serverName": "devbox-photon-controller-core", "serverAddress": "#{network_ip}:9000" } ],
      "MGMT_UI_HTTP_SERVERS": [ { "serverName": "devbox-management-ui-http", "serverAddress": "#{network_ip}:80" } ],
      "MGMT_UI_HTTPS_SERVERS": [ { "serverName": "devbox-management-ui-https", "serverAddress": "#{network_ip}:443" } ])

    # Auth parameters
    if ENV['ENABLE_AUTH'] == 'true'
        dynamic_params += %Q(,\n  "ENABLE_AUTH" : "true")
        dynamic_params += %Q(,\n  "APIFE_PORT" : "443")
        dynamic_params += %Q(,\n  "MGMT_API_PORT_SELECTOR" : "true")

        if ENV['PHOTON_AUTH_LS_ENDPOINT']
          dynamic_params += %Q(,\n  "AUTH_SERVER_ADDRESS" : "#{ENV['PHOTON_AUTH_LS_ENDPOINT']}")
        else
          abort("auth is enabled, but server address is not set")
        end

        if ENV['PHOTON_AUTH_SERVER_PORT']
          dynamic_params += %Q(,\n  "AUTH_SERVER_PORT" : "#{ENV['PHOTON_AUTH_SERVER_PORT']}")
        else
          abort("auth is enabled, but auth server port is not set")
        end

        if ENV['PHOTON_AUTH_SERVER_TENANT']
          dynamic_params += %Q(,\n  "AUTH_SERVER_TENANT" : "#{ENV['PHOTON_AUTH_SERVER_TENANT']}")
        else
          abort("auth is enabled, but auth tenant is not set")
        end

        if ENV['PHOTON_SWAGGER_LOGIN_URL']
          dynamic_params += %Q(,\n  "SWAGGER_LOGIN_URL" : "#{ENV['PHOTON_SWAGGER_LOGIN_URL']}")
        else
          abort("auth is enabled, but swagger login url is not set")
        end

      if ENV['PHOTON_SWAGGER_LOGOUT_URL']
        dynamic_params += %Q(,\n  "SWAGGER_LOGOUT_URL" : "#{ENV['PHOTON_SWAGGER_LOGOUT_URL']}")
      else
        abort("auth is enabled, but swagger logout url is not set")
      end

    else
      dynamic_params += %Q(,\n  "ENABLE_AUTH" : "false")
      dynamic_params += %Q(,\n  "APIFE_PORT" : "28080")
    end

    # Virtual network parameters
    if ENV.has_key?('USE_VIRTUAL_NETWORK')
      dynamic_params += %Q(,\n  "USE_VIRTUAL_NETWORK" : "#{ENV['USE_VIRTUAL_NETWORK']}")
    else
      dynamic_params += %Q(,\n  "USE_VIRTUAL_NETWORK" : "false")
    end

    # Real agent parameters
    if ENV['REAL_AGENT']
      dynamic_params += %Q(,\n  "USE_ESX_STORE" : "true")

      if ENV['ESX_IP'] && ENV['ESX_DATASTORE']
        dynamic_params += %Q(,\n  "ESX_HOST" : "#{ENV['ESX_IP']}")
        dynamic_params += %Q(,\n  "DATASTORE" : "#{ENV['ESX_DATASTORE']}")
      else
        abort("ESX_IP or ESX_DATASTORE not set")
      end
    else
      dynamic_params += %Q(,\n  "USE_ESX_STORE" : "false")
      dynamic_params += %Q(,\n  "DATASTORE" : "devbox_datastore")
    end

    # Syslog parameters
    if ENV['ENABLE_SYSLOG'] == 'true'
      if ENV['SYSLOG_ENDPOINT']
        dynamic_params += %Q(,\n  "SYSLOG_ENDPOINT" : "#{ENV['SYSLOG_ENDPOINT']}")
        dynamic_params += %Q(,\n  "ENABLE_SYSLOG" : "true")
      else
        abort("ENABLE_SYSLOG is set, but SYSLOG_ENDPOINT is not")
      end
    else
      dynamic_params += %Q(,\n  "ENABLE_SYSLOG" : "false")
    end
    dynamic_params += "\n}"

    # Write dynamic_params file
    config.vm.provision :shell, :privileged => false, :inline => <<-EOS
cat > #{dynamic_params_file} <<'EOM'
  #{dynamic_params}
EOM
  EOS

  # Validate dynamic_params file, catch errors early
  config.vm.provision :shell, :inline => <<-EOS
    printf "Validating dynamic params JSON file:"
    jsonlint #{dynamic_params_file}
    if [[ $? != 0 ]]; then
      echo "Validation failed for file:"
      cat #{dynamic_params_file}
      exit 1
    fi
  EOS

  # Set hostname for devbox. This is needed for the lightwave client.
  config.vm.provision :shell, :inline => <<-EOS
    sed -i s/photon-.*/"devbox.#{ENV['LW_DOMAIN_NAME'] || "photon.vmware.com"} devbox"/ /etc/hosts
    hostnamectl set-hostname devbox
    hostnamectl set-hostname --static devbox
  EOS

  # ================ PROVISIONING ================

  # renaming esxcloud containers to zookeeper Containers
  photon.vm.provision :shell, :inline => <<-EOS
    docker tag devbox/zookeeper photon/zookeeper
    docker tag devbox/haproxy photon/haproxy
  EOS

  #
  # Give access to zookeeper user inside container to write to log directory in host
  #
  photon.vm.provision :shell, :inline => <<-EOS
    set -ex
    chmod -R 777 /var/log/
  EOS

  #
  # Script for building agent VIB
  #
  agent_config_build_dir="/mirror/devbox_data/java/devbox/agent/build/configuration"

  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/build-agent"
cat > $out <<-'EOM'
#!/bin/bash -xe

# Keep an updated mirror of /devbox_data/ dirs needed for mounting into the container.
# This avoids vagrant shared folder issues with hard links.
# Only copy files needed for agent build, exclude all else.
sudo mkdir -p /mirror/devbox_data
time sudo rsync -rLgotp --exclude="develop" /devbox_data/python /mirror/devbox_data
time sudo rsync -rLgotp /devbox_data/thrift /mirror/devbox_data


container=$(docker run -d \
  -v /mirror/devbox_data:/devbox_data \
  -v /devbox_data/java/devbox/compileservices:/var/compileservices \
  -e "GIT_ROOT=#{GIT_ROOT}" \
  -e "GIT_REVISION=#{GIT_REVISION}" \
  photon/compileservices /var/compileservices/agent_compile_install.sh)
docker attach $container
docker commit $container devbox/agent

# Copy for use by integration test
sudo mkdir -p /var/esxcloud/packages
sudo rm -f /var/esxcloud/packages/*.vib
sudo cp /mirror/devbox_data/python/dist/*.vib /var/esxcloud/packages/
envoy_vib_url=#{ENV["ENVOY_VIB_URL"]}
if [ ! -z "$envoy_vib_url" ]; then
  sudo wget $envoy_vib_url -P /var/esxcloud/packages/
fi
EOM
chmod +x $out
  EOS

  agent_parameters_file = "/home/vagrant/agent-parameters.json"
  # Make sure agent params file exists so that mustache doesn't fail
  photon.vm.provision :shell, :privileged => false, :inline => "touch #{agent_parameters_file}"

  if !(ENV["PUBLIC_NETWORK_IP"] || ENV["REAL_AGENT"])
    agent_parameters = %Q(
    {
      "AGENT_BIND_ADDRESS": "#{network_ip}",
      "AGENT_LOG_PATH": "/var/log",
      "AGENT_ZOOKEEPER": "#{network_ip}:2181")
    if(ENV['ESX_AGENT_COUNT'])
     agent_parameters += %Q(,\n  "AGENT_COUNT": "#{ENV['ESX_AGENT_COUNT']}")
    end
    agent_parameters += "\n}"

    # Validate agent parameters JSON
    config.vm.provision :shell, :privileged => false, :inline => <<-EOS
cat > #{agent_parameters_file} <<-'EOM'
#{agent_parameters}
EOM
    EOS

    config.vm.provision :shell, :privileged => false, :inline => <<-EOS
    printf "Validating dynamic params JSON file:"
    jsonlint #{agent_parameters_file}
    if [[ $? != 0 ]]; then
      echo "Validation failed for file:"
      cat #{agent_parameters_file}
      exit 1
    fi
    EOS
  end

  #
  # Start agent script
  #
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
set -ex
out=#{SCRIPT_DIR}/start-agent
cat > $out <<-'EOM'
#!/bin/bash -xe

agent_config_dir=/devbox_data/java/devbox/agent/configuration

mkdir -p #{agent_config_build_dir}
for file in $agent_config_dir/*.#{CONFIG_EXTS}
do
  if [ -f $file ]; then
    output="#{agent_config_build_dir}/$(basename $file)"
    mustache "#{agent_parameters_file}" $file > "$output"
    if [[ "${file#*.}" == "sh" ]]; then
      chmod +x "$output"
    fi
  fi
done

mkdir -p /vagrant/log/agent

# Stop any old running agent container
docker stop devbox/agent &> /dev/null || true
docker rm -f devbox/agent &> /dev/null || true

# Start fake agent
docker run -d -p 8835:8835 \
  --net=host #{restart_always} \
  --name=agent \
  -v /vagrant/log/agent:/var/log \
  -v /devbox_data/tmp:/devbox_data/tmp \
  -v #{DATASTORE_DIR}:/tmp \
  -v #{ESXCLOUD_DIR}:#{ESXCLOUD_DIR} \
  -v #{agent_config_build_dir}:/etc/esxcloud/agent \
  devbox/agent /etc/esxcloud/agent/run.sh
EOM
chmod +x $out
  EOS

  #
  # Stop agent script
  #
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
set -ex
out=#{SCRIPT_DIR}/stop-agent
cat > $out <<-'EOM'
#!/bin/bash -x
docker stop agent
EOM
chmod +x $out
  EOS

  #
  # Create log directories for mounting
  #
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
    set -ex
    mkdir -p /vagrant/log/zookeeper
    mkdir -p /vagrant/log/haproxy
  EOS

  #
  # Prepare services configurations
  #
  SERVICE_CONFIG_MAP.each do | key, servicePath |
    service = SERVICE_NAME_MAP[key]
    config_build_dir = "/devbox_data/java/#{servicePath}/build/configuration"
    config_dir = "/devbox_data/java/#{servicePath}/src/dist/configuration"
    config_file="#{config_dir}/#{service}_test.json"

    # Write service start script
    photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/start-#{service}"
cat > $out <<-'EOM'
  #!/bin/bash -ex

  # Combine Vagrant dynamic params with test config
  mkdir -p #{config_build_dir}
  combined_params=$(jq -s '.[0] * (.[1]#{CONFIG_JQ_FILTER})' #{dynamic_params_file} #{config_file})

  for file in #{config_dir}/*.#{CONFIG_EXTS}
  do
    if [ -f $file ]; then
      output="#{config_build_dir}/$(basename $file)"
      mustache - $file <<< "$combined_params" > "$output"
      if [[ "${file#*.}" == "sh" ]]; then
        chmod +x "$output"
      fi
    fi
  done

  mkdir -p /vagrant/log/#{service}

EOM

cat >> $out <<-EOM
  #{service == "photon-controller-core" ? "copy-deployer-config" : ""}

  $(gen-docker-stop #{config_file}) &> /dev/null || true
  $(gen-docker-rm #{config_file}) &> /dev/null || true
  $(gen-docker-run #{config_file} /usr/sbin/init "" "-v #{config_build_dir}:/etc/esxcloud")
EOM
chmod +x $out
    EOS

    photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/copy-deployer-config"
cat >> $out <<-'EOM'
  dir="/etc/esxcloud-deployer/configurations/#{service}/"
  sudo mkdir -p $dir
  sudo cp -a #{config_dir}/* $dir
EOM
chmod +x $out
    EOS
  end

  #
  # Generate Zookeeper container start script
  #
  zk_config_file="/devbox_data/java/containers/zookeeper/config/zookeeper_test.json"
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/start-zookeeper"
cat > $out <<-'EOM'
  #!/bin/bash
  config_build_dir="/devbox_data/java/containers/zookeeper/build/config"
  mkdir -p $config_build_dir
  combined_params=$(jq -s '.[0] * (.[1]#{CONFIG_JQ_FILTER})' #{dynamic_params_file} #{zk_config_file})

  cd $(dirname #{zk_config_file})
  for file in ./*.#{CONFIG_EXTS}
  do
    if [ -f $file ]; then
      mustache - $file <<< "$combined_params" > "$config_build_dir/$file"
    fi
  done
EOM
cat >> $out <<-EOM
  $(gen-docker-stop #{zk_config_file}) &> /dev/null || true
  $(gen-docker-rm #{zk_config_file}) &> /dev/null || true
  $(gen-docker-run #{zk_config_file} /bin/bash /usr/lib/zookeeper/conf/run.sh)
  echo "Sleeping for ZK start"
  sleep 30
EOM
chmod +x $out
  EOS

  #
  # Generate HAProxy container start script
  #
  haproxy_config_file="/devbox_data/java/containers/haproxy/config/haproxy_test.json"
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/start-haproxy"
cat > $out <<-'EOM'
  #!/bin/bash
  config_build_dir="/devbox_data/java/containers/haproxy/build/config"
  mkdir -p $config_build_dir
  combined_params=$(jq -s '.[0] * (.[1]#{CONFIG_JQ_FILTER})' #{dynamic_params_file} #{haproxy_config_file})

  cd $(dirname #{haproxy_config_file})
  for file in ./*.#{CONFIG_EXTS}
  do
    if [ -f $file ]; then
      mustache - $file <<< "$combined_params" > "$config_build_dir/$file"
    fi
  done
EOM
cat >> $out <<-EOM
  $(gen-docker-stop #{haproxy_config_file}) &> /dev/null || true
  $(gen-docker-rm #{haproxy_config_file}) &> /dev/null || true
  $(gen-docker-run #{haproxy_config_file} /bin/bash /etc/haproxy/run.sh "-e HAPROXY_IP=#{network_ip}")
EOM
chmod +x $out
  EOS

  #
  # Set up Graphite container for integration tests
  #
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
    set -ex
    mkdir -p $HOME/graphite/conf

    # Storage aggregation config for Graphite. Without xFilesFactor set here,
    # Graphite will drop stats data that is too infrequent for the default setting.
    cat <<'EOM' > $HOME/graphite/conf/storage-aggregation.conf
[photon]
pattern = ^photon\..*
xFilesFactor = 0
EOM

    out=$HOME/graphite/start-graphite
    cat <<'EOM' > $out
# See for details: https://github.com/SamSaffron/graphite_docker
#
# 80: the graphite web interface
# 3000: the grafana web interface
# 2003: the carbon-cache line receiver (the standard graphite protocol)
# 2004: the carbon-cache pickle receiver
# 7002: the carbon-cache query port (used by the web interface)
# 8125: the statsd UDP port
# 8126: the statsd management port

docker run \
  -v /stats_data/graphite:/data \
  -v $HOME/graphite/conf/storage-aggregation.conf:/var/lib/graphite/conf/storage-aggregation.conf \
  -p 8000:80 \
  -p 3000:3000 \
  -p 2003:2003 \
  -p 2004:2004 \
  -p 7002:7002 \
  -p 8125:8125/udp \
  -p 8126:8126 \
  -d --name graphite \
  samsaffron/graphite
EOM
    chmod +x $out
    $out
  EOS

  #
  # Generate service check script, gets called by start-all script
  #
  photon.vm.provision :shell, :privileged => false, :inline => <<-EOS
out="#{SCRIPT_DIR}/check-services"
cat > $out <<-'EOM'
  #!/bin/bash -xe

  function get_logs() {
    echo "Dumping container logs to /vagrant/log/docker"
    mkdir -p /vagrant/log/docker
    local container=""
    for container in $(docker ps --format "{{.Names}}"); do
      docker logs $container > /vagrant/log/docker/${container}.log
    done
  }

  trap 'get_logs' EXIT

  echo "Containers:"
  docker ps -a

  MAX_RETRIES=300
  SLEEP_BETWEEN_RETRIES_SEC=1
  echo "Checking status of services"
  i=0
  while (($i < $MAX_RETRIES))
  do
    status=$(curl -s --connect-timeout 60 --max-time 60 --header "Authorization: Bearer #{API_SHARED_SECRET}" http://127.0.0.1:9000/status/)
    if [ "READY" == "$(echo $status | jq -r '.status')" ]; then
      echo "Services started successfully"
      exit 0
    fi
    # Print current statuses
    echo "Service status:"
    echo $status | jq -r '.components[] | .component + ": " + .status'
    echo "Retry count: $i"
    sleep $SLEEP_BETWEEN_RETRIES_SEC
    ((i++))
  done
  exit -1
EOM
chmod +x $out
  EOS
end


#
  # Clean log directory if selected
  #
  if ENV["DEVBOX_CLEAN_LOGS"]
    config.vm.provision :shell, :inline => <<-EOS
      set -ex
      echo "Cleaning logs"
      rm -rf /vagrant/log/*
    EOS
  end
end