Add win-builder

build-image-win-builder.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+packer init -upgrade -var-file=packer-vars-win-builder.hcl packer/gha-win-builder.pkr.hcl
+packer validate -var-file=packer-vars-win-builder.hcl packer/gha-win-builder.pkr.hcl
+packer build -var-file=packer-vars-win-builder.hcl packer/gha-win-builder.pkr.hcl

packer-vars-win-builder.hcl

@@ -0,0 +1,18 @@
+custom_shell_commands = [
+"mkdir C:/tmp/infra",
+"git clone https://github.com/compiler-explorer/infra C:/tmp/infra",
+"C:/tmp/infra/packer/InstallPwsh.ps1",
+"C:/tmp/infra/packer/InstallBuilderTools.ps1"
+]
+arch = "amd64"
+instance_type = "c5.large"
+runner_version = "2.321.0"
+region = "us-east-1"
+security_group_id = "sg-f53f9f80" # AdminNode (so we can ssh to it) just for builds
+subnet_id = "subnet-690ed81e"
+associate_public_ip_address = "true"
+global_tags = {
+    Site = "CompilerExplorer"
+    Subsystem = "CI"
+}
+iam_instance_profile = "XaniaBlog"

packer/gha-win-builder.pkr.hcl

@@ -0,0 +1,177 @@
+packer {
+  required_plugins {
+    amazon = {
+      version = ">= 0.0.2"
+      source  = "github.com/hashicorp/amazon"
+    }
+  }
+}
+
+variable "arch" {
+  description = "Architecture"
+  type        = string
+}
+
+locals {
+  runner_version = coalesce(var.runner_version, trimprefix(jsondecode(data.http.github_runner_release_json.body).tag_name, "v"))
+  aws_arch = {"amd64": "x86_64", "arm64": "aarch64"}[var.arch]
+  runner_arch = {"amd64": "x64", "arm64": "arm64" }[var.arch]
+}
+
+variable "runner_version" {
+  description = "The version (no v prefix) of the runner software to install https://github.com/actions/runner/releases. The latest release will be fetched from GitHub if not provided."
+  default     = null
+}
+
+variable "region" {
+  description = "The region to build the image in"
+  type        = string
+  default     = "eu-west-1"
+}
+
+variable "instance_type" {
+  description = "The instance type Packer will use for the builder"
+  type        = string
+  default     = "m4.xlarge"
+}
+
+variable "iam_instance_profile" {
+  description = "IAM instance profile Packer will use for the builder. An empty string (default) means no profile will be assigned."
+  type        = string
+  default     = ""
+}
+
+variable "security_group_id" {
+  description = "The ID of the security group Packer will associate with the builder to enable access"
+  type        = string
+  default     = null
+}
+
+variable "subnet_id" {
+  description = "If using VPC, the ID of the subnet, such as subnet-12345def, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC"
+  type        = string
+  default     = null
+}
+
+variable "root_volume_size_gb" {
+  type    = number
+  default = 30
+}
+
+variable "ebs_delete_on_termination" {
+  description = "Indicates whether the EBS volume is deleted on instance termination."
+  type        = bool
+  default     = true
+}
+
+variable "associate_public_ip_address" {
+  description = "If using a non-default VPC, there is no public IP address assigned to the EC2 instance. If you specified a public subnet, you probably want to set this to true. Otherwise the EC2 instance won't have access to the internet"
+  type        = string
+  default     = null
+}
+
+variable "custom_shell_commands" {
+  description = "Additional commands to run on the EC2 instance, to customize the instance, like installing packages"
+  type        = list(string)
+  default     = []
+}
+
+variable "temporary_security_group_source_public_ip" {
+  description = "When enabled, use public IP of the host (obtained from https://checkip.amazonaws.com) as CIDR block to be authorized access to the instance, when packer is creating a temporary security group. Note: If you specify `security_group_id` then this input is ignored."
+  type        = bool
+  default     = false
+}
+
+variable "global_tags" {
+  description = "Tags to apply to everything"
+  type        = map(string)
+  default     = {}
+}
+
+variable "ami_tags" {
+  description = "Tags to apply to the AMI"
+  type        = map(string)
+  default     = {}
+}
+
+variable "snapshot_tags" {
+  description = "Tags to apply to the snapshot"
+  type        = map(string)
+  default     = {}
+}
+
+data "http" github_runner_release_json {
+  url = "https://api.github.com/repos/actions/runner/releases/latest"
+  request_headers = {
+    Accept = "application/vnd.github+json"
+    X-GitHub-Api-Version : "2022-11-28"
+  }
+}
+
+source "amazon-ebs" "githubrunner" {
+  ami_name                                  = "github-runner-win-builder-${formatdate("YYYYMMDDhhmm", timestamp())}"
+  communicator                              = "winrm"
+  instance_type                             = var.instance_type
+  iam_instance_profile                      = var.iam_instance_profile
+  region                                    = var.region
+  security_group_id                         = var.security_group_id
+  subnet_id                                 = var.subnet_id
+  associate_public_ip_address               = var.associate_public_ip_address
+  temporary_security_group_source_public_ip = var.temporary_security_group_source_public_ip
+
+  source_ami_filter {
+    filters = {
+      name                = "Windows_Server-2022-English-Core-Base-*"
+      root-device-type    = "ebs"
+      virtualization-type = "hvm"
+    }
+    most_recent = true
+    owners      = ["amazon"]
+  }
+  tags = merge(
+    var.global_tags,
+    var.ami_tags,
+  {
+    OS_Version    = "windows-core-2022"
+    Release       = "Latest"
+    Base_AMI_Name = "{{ .SourceAMIName }}"
+  })
+  snapshot_tags = merge(
+    var.global_tags,
+    var.snapshot_tags,
+  )
+  winrm_insecure       = true
+  winrm_use_ssl        = true
+  winrm_username       = "Administrator"
+
+  launch_block_device_mappings {
+    device_name           = "/dev/sda1"
+    volume_size           = "${var.root_volume_size_gb}"
+    delete_on_termination = "${var.ebs_delete_on_termination}"
+  }
+}
+
+build {
+  name = "githubactions-runner"
+  sources = [
+    "source.amazon-ebs.githubrunner"
+  ]
+
+  provisioner "powershell" {
+    inline = concat([
+      templatefile("./windows-provisioner.ps1", {
+        action_runner_url = "https://github.com/actions/runner/releases/download/v${local.runner_version}/actions-runner-win-x64-${local.runner_version}.zip"
+      })
+    ], var.custom_shell_commands)
+  }
+
+  provisioner "file" {
+    content = templatefile("./start-runner.ps1", { metadata_tags = "enabled" })
+    destination = "C:\\start-runner.ps1"
+  }
+
+  post-processor "manifest" {
+    output     = "manifest.json"
+    strip_path = true
+  }
+}

packer/start-runner.ps1

@@ -0,0 +1,165 @@
+
+## Retrieve instance metadata
+
+Write-Host  "Retrieving TOKEN from AWS API"
+$token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"}
+if ( ! $token ) {
+  $retrycount=0
+  do {
+    echo "Failed to retrieve token. Retrying in 5 seconds."
+    Start-Sleep 5
+    $token=Invoke-RestMethod -Method PUT -Uri "http://169.254.169.254/latest/api/token" -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "180"}
+    $retrycount=$retrycount + 1
+    if ( $retrycount -gt 40 )
+    {
+        break
+    }
+  } until ($token)
+}
+
+$ami_id=Invoke-RestMethod -Uri "http://169.254.169.254/latest/meta-data/ami-id" -Headers @{"X-aws-ec2-metadata-token" = $token}
+
+$metadata=Invoke-RestMethod -Uri "http://169.254.169.254/latest/dynamic/instance-identity/document" -Headers @{"X-aws-ec2-metadata-token" = $token}
+
+$Region = $metadata.region
+Write-Host  "Retrieved REGION from AWS API ($Region)"
+
+$InstanceId = $metadata.instanceId
+Write-Host  "Retrieved InstanceId from AWS API ($InstanceId)"
+
+$tags=aws ec2 describe-tags --region "$Region" --filters "Name=resource-id,Values=$InstanceId" | ConvertFrom-Json
+Write-Host  "Retrieved tags from AWS API"
+
+$environment=$tags.Tags.where( {$_.Key -eq 'ghr:environment'}).value
+Write-Host  "Retrieved ghr:environment tag - ($environment)"
+
+$runner_name_prefix=$tags.Tags.where( {$_.Key -eq 'ghr:runner_name_prefix'}).value
+Write-Host  "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)"
+
+$ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value
+Write-Host  "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)"
+
+$parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json
+Write-Host  "Retrieved parameters from AWS SSM"
+
+$run_as=$parameters.where( {$_.Name -eq "$ssm_config_path/run_as"}).value
+Write-Host  "Retrieved $ssm_config_path/run_as parameter - ($run_as)"
+
+$enable_cloudwatch_agent=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_cloudwatch"}).value
+Write-Host  "Retrieved $ssm_config_path/enable_cloudwatch parameter - ($enable_cloudwatch_agent)"
+
+$agent_mode=$parameters.where( {$_.Name -eq "$ssm_config_path/agent_mode"}).value
+Write-Host  "Retrieved $ssm_config_path/agent_mode parameter - ($agent_mode)"
+
+$disable_default_labels=$parameters.where( {$_.Name -eq "$ssm_config_path/disable_default_labels"}).value
+Write-Host  "Retrieved $ssm_config_path/disable_default_labels parameter - ($disable_default_labels)"
+
+$enable_jit_config=$parameters.where( {$_.Name -eq "$ssm_config_path/enable_jit_config"}).value
+Write-Host  "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_jit_config)"
+
+$token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value
+Write-Host  "Retrieved $ssm_config_path/token_path parameter - ($token_path)"
+
+
+if ($enable_cloudwatch_agent -eq "true")
+{
+    Write-Host  "Enabling CloudWatch Agent"
+    & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a fetch-config -m ec2 -s -c "ssm:$ssm_config_path/cloudwatch_agent_config_runner"
+}
+
+## Configure the runner
+
+Write-Host "Get GH Runner config from AWS SSM"
+$config = $null
+$i = 0
+do {
+    $config = (aws ssm get-parameters --names "$token_path/$InstanceId" --with-decryption --region $Region  --query "Parameters[*].{Name:Name,Value:Value}" | ConvertFrom-Json)[0].value
+    Write-Host "Waiting for GH Runner config to become available in AWS SSM ($i/30)"
+    Start-Sleep 1
+    $i++
+} while (($null -eq $config) -and ($i -lt 30))
+
+Write-Host "Delete GH Runner token from AWS SSM"
+aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region
+
+# Create or update user
+if (-not($run_as)) {
+  Write-Host "No user specified, using default ec2-user account"
+  $run_as="ec2-user"
+}
+Add-Type -AssemblyName "System.Web"
+$password = [System.Web.Security.Membership]::GeneratePassword(24, 4)
+$securePassword = ConvertTo-SecureString $password -AsPlainText -Force
+$username = $run_as
+if (!(Get-LocalUser -Name $username -ErrorAction Ignore)) {
+    New-LocalUser -Name $username -Password $securePassword
+    Write-Host "Created new user ($username)"
+}
+else {
+    Set-LocalUser -Name $username -Password $securePassword
+    Write-Host "Changed password for user ($username)"
+}
+# Add user to groups
+foreach ($group in @("Administrators", "docker-users")) {
+    if ((Get-LocalGroup -Name "$group" -ErrorAction Ignore) -and
+        !(Get-LocalGroupMember -Group "$group" -Member $username -ErrorAction Ignore)) {
+        Add-LocalGroupMember -Group "$group" -Member $username
+        Write-Host "Added $username to $group group"
+    }
+}
+
+# Disable User Access Control (UAC)
+# TODO investigate if this is needed or if its overkill - https://github.com/github-aws-runners/terraform-aws-github-runner/issues/1505
+Set-ItemProperty HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name ConsentPromptBehaviorAdmin -Value 0 -Force
+Write-Host "Disabled User Access Control (UAC)"
+
+$runnerExtraOptions = ""
+if ($disable_default_labels -eq "true") {
+    $runnerExtraOptions += "--no-default-labels"
+}
+
+if ($enable_jit_config -eq "false" -or $agent_mode -ne "ephemeral") {
+  $configCmd = ".\config.cmd --unattended --name $runner_name_prefix$InstanceId --work `"_work`" $runnerExtraOptions $config"
+  Write-Host "Configure GH Runner (non ephmeral / no JIT) as user $run_as"
+  Invoke-Expression $configCmd
+}
+
+$jsonBody = @(
+    @{
+        group='Runner Image'
+        detail="AMI id: $ami_id"
+    }
+)
+ConvertTo-Json -InputObject $jsonBody | Set-Content -Path "$pwd\.setup_info"
+
+
+Write-Host "Starting the runner in $agent_mode mode"
+Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))"
+
+if ($agent_mode -eq "ephemeral") {
+    if ($enable_jit_config -eq "true") {
+        Write-Host "Starting with jit config"
+        Invoke-Expression ".\run.cmd --jitconfig $${config}"
+    }
+    else {
+        Write-Host "Starting without jit config"
+        Invoke-Expression ".\run.cmd"
+    }
+    Write-Host "Runner has finished"
+
+    if ($enable_cloudwatch_agent)
+    {
+        Write-Host "Stopping CloudWatch Agent"
+        & 'C:\Program Files\Amazon\AmazonCloudWatchAgent\amazon-cloudwatch-agent-ctl.ps1' -a stop
+    }
+
+    Write-Host "Terminating instance"
+    aws ec2 terminate-instances --instance-ids "$InstanceId" --region "$Region"
+} else {
+    Write-Host  "Installing the runner as a service"
+
+    $action = New-ScheduledTaskAction -WorkingDirectory "$pwd" -Execute "run.cmd"
+    $trigger = Get-CimClass "MSFT_TaskRegistrationTrigger" -Namespace "Root/Microsoft/Windows/TaskScheduler"
+    Register-ScheduledTask -TaskName "runnertask" -Action $action -Trigger $trigger -User $username -Password $password -RunLevel Highest -Force
+    Write-Host "Starting runner after $(((get-date) - (gcim Win32_OperatingSystem).LastBootUpTime).tostring("hh':'mm':'ss''"))"
+}