Abort gracefully when some servers cannot be created

vitobotta · Nov 13, 2021 · b355398 · b355398
1 parent 4e5aaf0
commit b355398
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 41 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hetzner-k3s (0.4.3)
+    hetzner-k3s (0.4.5)
       bcrypt_pbkdf
       ed25519
       http
@@ -20,12 +20,12 @@ GEM
     diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    dry-configurable (0.12.1)
+    dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.8.0)
+      dry-core (~> 0.6)
+    dry-container (0.9.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
+      dry-configurable (~> 0.13, >= 0.13.0)
     dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.3.0)
@@ -99,7 +99,7 @@ GEM
     to_regexp (0.2.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
     yajl-ruby (1.4.1)
     yaml-safe_load_stream (0.1.1)
 

diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ This will install the `hetzner-k3s` executable in your PATH.
 Alternatively, if you don't want to set up a Ruby runtime but have Docker installed, you can use a container. Run the following from inside the directory where you have the config file for the cluster (described in the next section):
 
 ```bash
-docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s:v0.4.5 create-cluster --config-file /cluster/test.yaml
+docker run --rm -it -v ${PWD}:/cluster -v ${HOME}/.ssh:/tmp/.ssh vitobotta/hetzner-k3s:v0.4.6 create-cluster --config-file /cluster/test.yaml
 ```
 
 Replace `test.yaml` with the name of your config file.
@@ -242,6 +242,9 @@ I recommend that you create a separate Hetzner project for each cluster, because
 
 ## changelog
 
+- 0.4.5
+  - Added a check to abort gracefully when for some reason one or more servers are not created, for example due to temporary problems with the Hetzner API.
+
 - 0.4.5
   - Fix network creation (bug introduced in the previous version)
 

diff --git a/bin/build.sh b/bin/build.sh
@@ -6,9 +6,9 @@ set -e
 
 IMAGE="vitobotta/hetzner-k3s"
 
-docker build -t ${IMAGE}:v0.4.5 \
+docker build -t ${IMAGE}:v0.4.6 \
   --platform=linux/amd64 \
-  --cache-from ${IMAGE}:v0.4.4 \
+  --cache-from ${IMAGE}:v0.4.5 \
   --build-arg BUILDKIT_INLINE_CACHE=1 .
 
-docker push vitobotta/hetzner-k3s:v0.4.5
+docker push vitobotta/hetzner-k3s:v0.4.6
diff --git a/lib/hetzner/infra/server.rb b/lib/hetzner/infra/server.rb
@@ -40,12 +40,22 @@ def create(location:, instance_type:, instance_id:, firewall_id:, network_id:, s
         placement_group: placement_group_id
       }
 
-      response = hetzner_client.post("/servers", server_config).body
+      response = hetzner_client.post("/servers", server_config)
+      response_body = response.body
+
+      server = JSON.parse(response_body)["server"]
+
+      unless server
+        puts "Error creating server #{server_name}. Response details below:"
+        puts
+        p response
+        return
+      end
 
       puts "...server #{server_name} created."
       puts
 
-      JSON.parse(response)["server"]
+      server
     end
 
     def delete(server_name:)

diff --git a/lib/hetzner/k3s/cluster.rb b/lib/hetzner/k3s/cluster.rb
@@ -157,6 +157,11 @@ def create_resources
 
       threads.each(&:join) unless threads.empty?
 
+      if server_configs.size != servers.size
+        puts "Something went wrong while creating some servers, please try again."
+        exit 1
+      end
+
       puts
       threads = servers.map do |server|
         Thread.new { wait_for_ssh server }
@@ -230,41 +235,41 @@ def master_script(master)
       taint = schedule_workloads_on_masters? ? " " : " --node-taint CriticalAddonsOnly=true:NoExecute "
 
       <<~EOF
-      curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
-        --disable-cloud-controller \
-        --disable servicelb \
-        --disable traefik \
-        --disable local-storage \
-        --disable metrics-server \
-        --write-kubeconfig-mode=644 \
-        --node-name="$(hostname -f)" \
-        --cluster-cidr=10.244.0.0/16 \
-        --etcd-expose-metrics=true \
-        --kube-controller-manager-arg="address=0.0.0.0" \
-        --kube-controller-manager-arg="bind-address=0.0.0.0" \
-        --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
-        --kube-scheduler-arg="address=0.0.0.0" \
-        --kube-scheduler-arg="bind-address=0.0.0.0" \
-        #{taint} \
-        --kubelet-arg="cloud-provider=external" \
-        --advertise-address=$(hostname -I | awk '{print $2}') \
-        --node-ip=$(hostname -I | awk '{print $2}') \
-        --node-external-ip=$(hostname -I | awk '{print $1}') \
-        --flannel-iface=#{flannel_interface} \
-        #{server} #{tls_sans}" sh -
+        curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="#{k3s_version}" K3S_TOKEN="#{k3s_token}" INSTALL_K3S_EXEC="server \
+          --disable-cloud-controller \
+          --disable servicelb \
+          --disable traefik \
+          --disable local-storage \
+          --disable metrics-server \
+          --write-kubeconfig-mode=644 \
+          --node-name="$(hostname -f)" \
+          --cluster-cidr=10.244.0.0/16 \
+          --etcd-expose-metrics=true \
+          --kube-controller-manager-arg="address=0.0.0.0" \
+          --kube-controller-manager-arg="bind-address=0.0.0.0" \
+          --kube-proxy-arg="metrics-bind-address=0.0.0.0" \
+          --kube-scheduler-arg="address=0.0.0.0" \
+          --kube-scheduler-arg="bind-address=0.0.0.0" \
+          #{taint} \
+          --kubelet-arg="cloud-provider=external" \
+          --advertise-address=$(hostname -I | awk '{print $2}') \
+          --node-ip=$(hostname -I | awk '{print $2}') \
+          --node-external-ip=$(hostname -I | awk '{print $1}') \
+          --flannel-iface=#{flannel_interface} \
+          #{server} #{tls_sans}" sh -
       EOF
     end
 
     def worker_script(worker)
       flannel_interface = find_flannel_interface(worker)
 
       <<~EOF
-      curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
-        --node-name="$(hostname -f)" \
-        --kubelet-arg="cloud-provider=external" \
-        --node-ip=$(hostname -I | awk '{print $2}') \
-        --node-external-ip=$(hostname -I | awk '{print $1}') \
-        --flannel-iface=#{flannel_interface}" sh -
+        curl -sfL https://get.k3s.io | K3S_TOKEN="#{k3s_token}" INSTALL_K3S_VERSION="#{k3s_version}" K3S_URL=https://#{first_master_private_ip}:6443 INSTALL_K3S_EXEC="agent \
+          --node-name="$(hostname -f)" \
+          --kubelet-arg="cloud-provider=external" \
+          --node-ip=$(hostname -I | awk '{print $2}') \
+          --node-external-ip=$(hostname -I | awk '{print $1}') \
+          --flannel-iface=#{flannel_interface}" sh -
       EOF
     end
 

diff --git a/lib/hetzner/k3s/version.rb b/lib/hetzner/k3s/version.rb
@@ -1,5 +1,5 @@
 module Hetzner
   module K3s
-    VERSION = "0.4.5"
+    VERSION = "0.4.6"
   end
 end