add chef vault functionality

Author: Jörg Herzinger

.gitignore

@@ -1 +1,2 @@
 Gemfile.lock
+.bundle

Gemfile

@@ -1,3 +1,4 @@
-source 'https://rcrs.rbinternational.com/artifactory/api/gems/gems/'
+source 'https://rubygems.org'
 
 gem 'mixlib-shellout', '~> 2.3.2'
+gem 'deep_merge', '~> 1.2.1'

README.md

@@ -10,7 +10,9 @@ To run this you need at least:
 
   * Git
   * Docker
-  * Ruby (>= 2)
+  * Ruby (>= 2) and Bundler
+
+You also have to initialize the Chef Data Bag used for the certificates. By default that is called `letsencrypt` via `knife data bag create letsencrypt`.
 
 ### Config
 
@@ -33,9 +35,10 @@ Create a file called `config.json` alongside the `config.default.json` and fill
   "chef": {
     "docker_image": "docker.local/chef/chefdk:v2",
     "data_bag": "letsencrypt",
+    "config_dir": "~/.chef",
     "admins": ["admin1", "admin2"]
   },
-  "backends": [
+  "steps": [
     "chef_vault"
   ],
   "certificates": {
@@ -70,7 +73,14 @@ rake init
 rake certbot:register
 ```
 
-once and commit the changes.
+once and commit the changes. With the account registered and the Chef Data Bag create the usual workflow is to update the `config.json` with the certificates that should be created and then to run:
+
+```
+rake init
+rake certbot:renew
+rake chef_vault:upload
+rake cleanup
+```
 
 ### Running in Jenkins
 
@@ -98,3 +108,5 @@ Pretty much everything I guess...
   * How to upload only changed certificates
     * Read state from git?
     * Keep state file with changes?
+  * Change Rake tasks to be able to take a list of certificates as argument
+  * Add Chef Vault refresh rake task

Rakefile

@@ -12,8 +12,8 @@ Rake.application.options.suppress_backtrace_pattern = %r{/} # Suppress trace whe
 desc 'Initialize project'
 task :init do
   Helpers.log Certbot.init
-  # ChefVault.init if Helpers.config['backends'].key?('chef_vault')
-  # HashicorpVault.init if Helpers.config['backends'].key?('hashicorp_vault')
+  Helpers.log ChefVault.init if Helpers.config['steps'].include?('chef_vault')
+  # HashicorpVault.init if Helpers.config['steps'].key?('hashicorp_vault')
 end
 
 desc 'Displays Help'
@@ -56,11 +56,14 @@ end
 namespace :chef_vault do
   desc 'Upload all changed certificates as chef vaults'
   task :upload do
-    ChefVault.upload
+    Helpers.config['certificates'].each do |name, props|
+      Helpers.log ChefVault.upload(name, props['chef_vault'] || {})
+    end
   end
-end
+end if Helpers.config['steps'].include?('chef_vault')
 
 desc 'Cleanup all temporary files, docker containers etc'
 task :cleanup do
   Helpers.log Certbot.cleanup
+  Helpers.log ChefVault.cleanup if Helpers.config['steps'].include?('chef_vault')
 end

config.default.json

@@ -5,16 +5,20 @@
     "docker_image": "certbot/dns-route53:v0.22.0",
     "docker_container": "certbot",
     "aws_credentials": "~/.aws/credentials",
-    "email": "[email protected]",
     "server": "https://acme-staging-v02.api.letsencrypt.org/directory"
   },
-  "chef": {
+  "chef_vault": {
     "docker_image": "chef/chefdk:2",
+    "docker_container": "chefdk",
     "data_bag": "letsencrypt",
-    "admins": ["admin1", "admin2"]
+    "config_dir": "~/.chef"
   },
-  "backends": [
-    "chef_vault"
+  "steps": [
+    "init",
+    "certbot",
+    "chef_vault",
+    "git",
+    "cleanup"
   ],
   "certificates": {
   }

lib/certbot.rb

@@ -1,3 +1,4 @@
+require 'etc'
 require_relative 'helpers'
 
 # Cerbot module
@@ -25,6 +26,10 @@ def register
   def renew(name, domains)
     Helpers.info_log("Registering/Renewing certificate #{name} for domains #{domains.join(',')}")
     run("certonly --non-interactive --dns-route53 --cert-name #{name} --domains #{domains.join(',')}")
+
+    # Fix permissions for etc_letsencrypt so we can actually commit it to GIT
+    user = Etc.getpwuid
+    Helpers.run_command("docker exec -i #{Helpers.config['certbot']['docker_container']} chown -R #{user.uid}:#{user.gid} /etc/letsencrypt")
   end
 
   def run(cmd)

lib/chef_vault.rb

@@ -1,8 +1,53 @@
+require 'json'
+
 # Chef Vault module
 module ChefVault
   extend self
 
   def init
+    options = [ "-itd #{Helpers.environment}",
+                "--name #{Helpers.config['chef_vault']['docker_container']}",
+                "-v `pwd`/etc_letsencrypt:/etc/letsencrypt",
+                "-v #{Helpers.config['chef_vault']['config_dir']}:/root/.chef"
+    ]
+
+    Helpers.run_command("docker run #{options.join(' ')}  #{Helpers.config['chef_vault']['docker_image']}")
+  end
+
+  def cleanup
+    Helpers.run_command("docker rm -f #{Helpers.config['chef_vault']['docker_container']}")
+  end
+
+  # Upload all certificates to the chef server
+  def upload(cert, props = {})
+    Helpers.info_log("Uploading Chef Vault for #{cert}")
+    options = [ '--mode client' ]
+    options << "--admins #{Helpers.config['chef_vault']['admins'].join(',')}" if Helpers.config['chef_vault'].key?('admins')
+    options << "--clients #{props['clients'].join(',')}" if props.key?('clients')
+    options << "--search #{props['search']}" if props.key?('search')
+
+    current_item = run("vault itemtype #{Helpers.config['chef_vault']['data_bag']} #{cert}")
+    if current_item[:return] == 0 && current_item[:stdout].include?('vault')
+      run("vault update --clean #{options.join(' ')} #{Helpers.config['chef_vault']['data_bag']} #{cert} '#{vault_json(cert)}'")
+    elsif current_item[:return] == 0 && !current_item[:stdout].include?('vault')
+      { return: 1, stdout: "The DataBag item #{Helpers.config['chef_vault']['data_bag']}/#{cert} is not of type vault. Please delete the item first.", stderr: ''}
+    elsif current_item[:return] == 100 && current_item[:stderr].include?('ERROR: The object you are looking for could not be found')
+      run("vault create #{options.join(' ')} #{Helpers.config['chef_vault']['data_bag']} #{cert} '#{vault_json(cert)}'")
+    else
+      current_item
+    end
+  end
+
+  def run(cmd)
+    Helpers.run_command("docker exec -i #{Helpers.config['chef_vault']['docker_container']} knife #{cmd}")
+  end
+
+  def vault_json(cert)
+    vault = {}
+    %w(cert chain fullchain privkey).each do |file|
+      vault[file] = File.read("etc_letsencrypt/live/#{cert}/#{file}.pem")
+    end
 
+    vault.to_json
   end
 end

lib/helpers.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'fileutils'
 require 'mixlib/shellout'
+require 'deep_merge'
 
 # Helpers module for stuff that is always needed
 module Helpers
@@ -27,7 +28,7 @@ def config
                     {}
                   end
 
-    @@config = default_config.merge(user_config)
+    @@config = default_config.deep_merge!(user_config, {:overwrite_arrays => true})
   end
 
   # Simple info logger using the log helper
@@ -49,7 +50,7 @@ def log(out, error_msg = '')
       puts out[:stdout] unless out[:stdout].nil? || out[:stdout].empty?
     else
       puts "--- Failed with exit code #{out[:return]}"
-      puts "Command was: #{out[:cmd]}"
+      puts "Command was: #{out[:cmd]}" if out.key?('cmd')
       unless out[:stdout].empty?
         puts "--- STDOUT:"
         puts out[:stdout].strip

lib/init.sh

@@ -1,7 +1,7 @@
 if [[ -e Gemfile.lock  ]]
 then
-  bundle23 update
+  bundle update
   touch Gemfile.lock # Update time on file so rake won't call it again
 else
-  bundle23 install
+  bundle install --path .bundle
 fi