Merge pull request #488 from fangn2/fix-build

Fix CI build by using pre-built rootfs image

Author: fangn2

.buildkite/pipeline.yml

@@ -61,11 +61,9 @@ steps:
       # Copy vmlinux and root-drive.img.
       - ln -s /var/lib/fc-ci/vmlinux.bin ${FC_TEST_DATA_PATH}/vmlinux
       - ln -s /var/lib/fc-ci/rootfs.ext4 ${FC_TEST_DATA_PATH}/root-drive.img
-      # Download Firecracker and its jailer.
-      - make deps
-      # Build a rootfs with SSH enabled.
-      - sudo -E FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH} make ${FC_TEST_DATA_PATH}/root-drive-ssh-key
-      - sudo chown $USER ${FC_TEST_DATA_PATH}/root-drive-ssh-key ${FC_TEST_DATA_PATH}/root-drive-with-ssh.img
+      # Download Firecracker and its jailer
+      # Download pre-built rootfs with SSH enabled and its ssh key.
+      - make deps  
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
@@ -90,6 +88,7 @@ steps:
 
   - label: ':book: examples'
     commands:
+      # Copy those deps from install step to save time re-downloading
       - cp ${FC_TEST_DATA_PATH}/root-drive-ssh-key ${FC_TEST_DATA_PATH}/root-drive-with-ssh.img ${FC_TEST_DATA_PATH}/vmlinux examples/cmd/snapshotting
       - cd examples/cmd/snapshotting
       - make all
@@ -112,7 +111,7 @@ steps:
     commands:
       - export FC_TEST_TAP=fc-root-tap${BUILDKITE_BUILD_NUMBER}
       - export FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH}
-      - make test EXTRAGOARGS="-exec 'sudo -E' -count=1 -race" DISABLE_ROOT_TESTS=
+      - make test EXTRAGOARGS="-exec 'sudo -E' -v -count=1 -race" DISABLE_ROOT_TESTS=
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
       distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"

Makefile

@@ -34,25 +34,21 @@ firecracker_version=v1.0.0
 release_url=https://github.com/firecracker-microvm/firecracker/releases/download/$(firecracker_version)/firecracker-$(firecracker_version)-$(arch).tgz
 
 testdata_objects = \
-$(FC_TEST_DATA_PATH)/vmlinux \
-$(FC_TEST_DATA_PATH)/root-drive.img \
-$(FC_TEST_DATA_PATH)/jailer \
 $(FC_TEST_DATA_PATH)/firecracker \
+$(FC_TEST_BIN_PATH)/host-local \
+$(FC_TEST_DATA_PATH)/jailer \
 $(FC_TEST_DATA_PATH)/ltag \
 $(FC_TEST_BIN_PATH)/ptp \
-$(FC_TEST_BIN_PATH)/host-local \
+$(FC_TEST_DATA_PATH)/root-drive.img \
+$(FC_TEST_DATA_PATH)/root-drive-with-ssh.img \
+$(FC_TEST_DATA_PATH)/root-drive-ssh-key \
 $(FC_TEST_BIN_PATH)/static \
-$(FC_TEST_BIN_PATH)/tc-redirect-tap
-
-# Enable pulling of artifacts from S3 instead of building
-# TODO: https://github.com/firecracker-microvm/firecracker-go-sdk/issues/418
-ifeq ($(GID), 0)
-testdata_objects += $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img $(FC_TEST_DATA_PATH)/root-drive-ssh-key
-endif
+$(FC_TEST_BIN_PATH)/tc-redirect-tap \
+$(FC_TEST_DATA_PATH)/vmlinux 
 
 testdata_dir = testdata/firecracker.tgz testdata/firecracker_spec-$(firecracker_version).yaml testdata/LICENSE testdata/NOTICE testdata/THIRD-PARTY
 
-# --location is needed to follow redirects on github.com
+# --location is needed to follow redirects
 curl = curl --location
 
 GO_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1,2)
@@ -96,26 +92,23 @@ $(FC_TEST_DATA_PATH)/vmlinux:
 
 $(FC_TEST_DATA_PATH)/firecracker $(FC_TEST_DATA_PATH)/jailer: $(FC_TEST_DATA_PATH)/fc.stamp
 
+# Download the pinned release version of firecracker and jailer version from github
 $(FC_TEST_DATA_PATH)/fc.stamp:
 	$(curl) ${release_url} | tar -xvzf - -C $(FC_TEST_DATA_PATH)
 	mv $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)/firecracker-$(firecracker_version)-$(arch) $(FC_TEST_DATA_PATH)/firecracker
 	mv $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)/jailer-$(firecracker_version)-$(arch) $(FC_TEST_DATA_PATH)/jailer
+	rm -rf $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)
 	touch $@
 
 $(FC_TEST_DATA_PATH)/root-drive.img:
 	$(curl) -o $@ https://s3.amazonaws.com/spec.ccfc.min/img/hello/fsfiles/hello-rootfs.ext4
 
+# Download pre-built rootfs image and its ssh key from S3
+# TODO: Change the pre-built rootfs to ubuntu 22.04 once firecracker team has it in the public S3 bucket
+# Currently the S3 bucket only has ubuntu 18.04 image
 $(FC_TEST_DATA_PATH)/root-drive-ssh-key $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img: 
-# Need root to move ssh key to testdata location
-ifeq ($(GID), 0)
-	$(MAKE) $(FIRECRACKER_DIR)
-	$(FIRECRACKER_DIR)/tools/devtool build_rootfs -m $(FC_TEST_DATA_PATH)/mnt
-	cp $(FIRECRACKER_DIR)/build/rootfs/bionic.rootfs.ext4 $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img
-	cp $(FIRECRACKER_DIR)/build/rootfs/ssh/id_rsa $(FC_TEST_DATA_PATH)/root-drive-ssh-key
-	rm -rf $(FIRECRACKER_DIR)
-else
-	$(error unable to place ssh key without root permissions)
-endif
+	$(curl) -o $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/$(arch)/ubuntu-18.04.ext4
+	$(curl) -o $(FC_TEST_DATA_PATH)/root-drive-ssh-key https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/$(arch)/ubuntu-18.04.id_rsa
 
 $(FC_TEST_BIN_PATH)/ptp:
 	$(call install_go,github.com/containernetworking/plugins/plugins/main/ptp,v1.1.1)
@@ -132,12 +125,14 @@ $(FC_TEST_BIN_PATH)/tc-redirect-tap:
 $(FC_TEST_DATA_PATH)/ltag:
 	$(call install_go,github.com/kunalkushwaha/ltag,v0.2.3)
 
-$(FIRECRACKER_DIR):
-	- git clone https://github.com/firecracker-microvm/firecracker.git $(FIRECRACKER_DIR)
-
+# test-images builds firecracker and jailer from main branch of firecracker
+# to test against HEAD of firecracker  
 .PHONY: test-images
 test-images: $(FIRECRACKER_BIN) $(JAILER_BIN)
 
+$(FIRECRACKER_DIR):
+	- git clone https://github.com/firecracker-microvm/firecracker.git $(FIRECRACKER_DIR)
+
 $(FIRECRACKER_BIN) $(JAILER_BIN): $(FIRECRACKER_DIR)
 	$(FIRECRACKER_DIR)/tools/devtool -y build --release
 	cp $(FIRECRACKER_DIR)/build/cargo_target/$(FIRECRACKER_TARGET)/release/firecracker $(FIRECRACKER_BIN)

examples/cmd/snapshotting/Makefile

@@ -21,6 +21,9 @@ PWD=$(shell pwd)
 GOBIN=$(PWD)/bin
 FC_TEST_DATA_PATH?=$(PWD)
 
+# --location is needed to follow redirects on github.com
+curl = curl --location
+
 GO_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1,2)
 ifeq ($(GO_VERSION), $(filter $(GO_VERSION),1.14 1.15))
     define install_go
@@ -50,28 +53,29 @@ bin/host-local: bin
 	$(call install_go,github.com/containernetworking/plugins/plugins/ipam/host-local,v1.1.1)
 
 vmlinux:
-	curl --location -o vmlinux https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/${ARCH}/kernels/vmlinux.bin
+	$(curl) -o vmlinux https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/${ARCH}/kernels/vmlinux.bin
 
 firecracker:
-	curl -L ${RELEASE_URL}/download/${VER}/firecracker-${VER}-${ARCH}.tgz | tar -xz
+	$(curl) ${RELEASE_URL}/download/${VER}/firecracker-${VER}-${ARCH}.tgz | tar -xz
 	mv release-${VER}-${ARCH}/firecracker-${VER}-${ARCH} firecracker
 	rm -rf release-${VER}-${ARCH}
 
+# Use rootfs and ssh key pre-built from S3 following firecracker getting-started guide
+# https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
+# TODO: Change the pre-built rootfs to ubuntu 22.04 once firecracker team has that in the public S3 bucket
+# Currently the S3 bucket only has ubuntu 18.04 image
 root-drive-with-ssh.img root-drive-ssh-key:
-	- mkdir temp
-	- git clone https://github.com/firecracker-microvm/firecracker temp
-	temp/tools/devtool build_rootfs
-	cp temp/build/rootfs/bionic.rootfs.ext4 root-drive-with-ssh.img
-	cp temp/build/rootfs/ssh/id_rsa root-drive-ssh-key
-	rm -rf temp
-
+	$(curl) -o root-drive-with-ssh.img https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/${ARCH}/ubuntu-18.04.ext4
+	$(curl) -o root-drive-ssh-key https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/${ARCH}/ubuntu-18.04.id_rsa
+	
+	
 snapshot-example:
 	go build -o $@
 
 run: snapshot-example
 	./snapshot-example
 
 clean:
-	rm -rf bin firecracker root-drive-ssh-key root-drive-with-ssh.img vmlinux
+	rm -rf bin firecracker root-drive-ssh-key root-drive-with-ssh.img snapshot-example vmlinux  
 
 .PHONY: all clean plugins run

examples/cmd/snapshotting/README.md

@@ -9,7 +9,7 @@ This test requires both KVM and root access.
 Run this test by first running
 
 ```
-sudo -E env PATH=$PATH make all
+make all
 ```
 
 followed by
@@ -18,7 +18,7 @@ followed by
 sudo -E env PATH=$PATH go run example_demo.go
 ```
 
-Alternatively, to do both of the above,
+or,
 ```
 sudo -E env PATH=$PATH make run
 ```
@@ -62,20 +62,5 @@ Pressing enter resumes execution of the program.
 You can remove dependencies via a simple `make clean`.
 
 ```
-sudo make clean
-```
-
-## Issues
-
-You may encounter an issue where the image does not build properly. This is often indicated via the following near the end of terminal output:
-
-```
-umount: /firecracker/build/rootfs/mnt: not mounted.
-```
-
-This is due to an issue in Firecracker's devtool command used to dynamically create an image. Fixing this is often as simple as rerunning the command.
-
-```
-sudo rm -rf root-drive-with-ssh.img root-drive-ssh-key
-sudo make image
+make clean
 ```

examples/cmd/snapshotting/example_demo.go

@@ -89,9 +89,9 @@ func createNewConfig(socketPath string, opts ...configOpt) sdk.Config {
 		SocketPath:      socketPath,
 		KernelImagePath: kernelImagePath,
 		MachineCfg: models.MachineConfiguration{
-			VcpuCount:   &vcpuCount,
-			MemSizeMib:  &memSizeMib,
-			Smt:         &smt,
+			VcpuCount:  &vcpuCount,
+			MemSizeMib: &memSizeMib,
+			Smt:        &smt,
 		},
 		Drives: []models.Drive{
 			{
@@ -412,13 +412,15 @@ func main() {
 	defer os.Remove(tempdir)
 	socketPath := filepath.Join(tempdir, "snapshotssh")
 
-	err = os.Mkdir("snapshotssh", 0777)
+	snapshotsshPath := filepath.Join(dir, "snapshotssh")
+	err = os.Mkdir(snapshotsshPath, 0777)
 	if err != nil && !errors.Is(err, os.ErrExist) {
 		log.Fatal(err)
 	}
+	defer os.RemoveAll(snapshotsshPath)
 
-	snapPath := filepath.Join(dir, "snapshotssh/SnapFile")
-	memPath := filepath.Join(dir, "snapshotssh/MemFile")
+	snapPath := filepath.Join(snapshotsshPath, "SnapFile")
+	memPath := filepath.Join(snapshotsshPath, "MemFile")
 
 	ctx := context.Background()