DLPX-94086 LTS 24.04: update linux-pkg for Ubuntu 24.04 appliance

PR URL: https://www.github.com/delphix/linux-pkg/pull/349

Author: palash-gandhi

.github/scripts/verify-query-packages.sh

@@ -30,14 +30,6 @@ read -r -a fields <<<"$(./query-packages.sh single -o git-url zfs 2>&1)"
 test ${#fields[@]} -eq 1
 test "${fields[0]}" == 'https://github.com/delphix/zfs.git'
 
-# Expect: "bpftrace	bcc	true	https://github.com/delphix/bpftrace.git"
-read -r -a fields <<<"$(./query-packages.sh single -o name,dependencies,can-update,git-url bpftrace 2>&1)"
-test ${#fields[@]} -eq 4
-test "${fields[0]}" == 'bpftrace'
-test "${fields[1]}" == 'bcc'
-test "${fields[2]}" == 'true'
-test "${fields[3]}" == 'https://github.com/delphix/bpftrace.git'
-
 # Expect that "list all" outputs all directory names under packages/
 diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort)
 

README.md

@@ -29,7 +29,7 @@ projects.
 
 ## System Requirements
 
-This framework is intended to be run on an Ubuntu 20.04 system with some basic
+This framework is intended to be run on an Ubuntu 24.04 system with some basic
 developer packages installed, such as git, and passwordless sudo enabled. Note
 that it will automatically install various build-dependencies on the system, so
 as a safety precaution it is currently restricted to only run on an AWS instance
@@ -280,12 +280,7 @@ of some of the scripts defined above.
 
 * **package_S3_URL**: Similar to the package_VAR variables above. This is used
   to override the default S3 location for where package build-dependencies are
-  fetched for a given linux-pkg package. For instance, if you are building
-  bpftrace, which has `PACKAGE_DEPENDENCIES="bcc"` in its config, the
-  `fetch_dependencies()` stage in the build will fetch the latest build
-  artifacts of the bcc package from a predetermined S3 location. If you pass
-  `BCC_S3_URL=s3://path/to/custom/bcc/artifacts` then those artifacts will be
-  fetched insteasd.
+  fetched for a given linux-pkg package.
 
 * **DELPHIX_PACKAGE_MIRROR_MAIN, DELPHIX_PACKAGE_MIRROR_SECONDARY**: When
   the [setup.sh](#setupsh) script is run, it will configure the apt sources

default-package-config.sh

@@ -35,11 +35,19 @@ function merge_with_upstream() {
 # The functions below are specific for the Linux kernel packages
 # and contain the majority of their common code.
 #
+# Ensure that Delphix's version of the rust toolchain is
+# installed rather than Ubuntu's version to satisfy the kernel's
+# dependency on the rust toolchain. Delphix's rust toolchain is
+# supplied via virtual packages and hence must be installed explicitly
+# otherwise apt installs the Ubuntu's version of the rust toolchain.
+#
 function kernel_prepare() {
 	logmust install_pkgs \
-		equivs \
+		"$DEPDIR"/delphix-rust/*.deb \
 		devscripts \
-		kernel-wedge
+		equivs \
+		kernel-wedge \
+		gawk
 }
 
 #
@@ -53,8 +61,8 @@ function kernel_build() {
 	local platform="$1"
 	#
 	# Note: Extra arguments can overwrite default arguments.
-	#       For example in this function we default skipdbg
-	#       to false, but if we pass "skipdbg=true" as an
+	#       For example in this function we default do_dbgsym_package
+	#       to false, but if we pass "do_dbgsym_package=true" as an
 	#       extra argument we will be overwriting this value
 	#       to true. This is because when a variable's value
 	#       is declared multiple times when invoking the
@@ -103,31 +111,29 @@ function kernel_build() {
 	echo "$kernel_version" >"$WORKDIR/artifacts/KERNEL_VERSION"
 
 	#
-	# skipdbg=false
+	# do_dbgsym_package=true
 	#   We need debug info for our debugging tools to work.
 	#   Don't skip them.
+	# do_tools_common=false
+	#   We do not need to build linux-tools-common package and we
+	#   install it directly from our package mirror.
+	# do_tools_host=false
+	#   We do not need to build linux-tools-host package.
 	# uefi_signed=false
 	#   This variable defaults to true but since we don't have
 	#   any intention and logic to provide signatures for now
 	#   we set it to false to avoid any misconfigurations down
 	#   the line.
-	# disable_d_i=true
-	#   This prevents udeb packages from being built as they are
-	#   not consumed by the Delphix Appliance.
-	# do_dkms_*=false
+	# dkms_exclude=*
 	#   This disables the build of various out-of-tree kernel modules
 	#   that we do not use in our product or that we provide separately.
 	#
 	local debian_rules_args=(
-		"skipdbg=false"
+		"do_dbgsym_package=true"
+		"do_tools_common=false"
+		"do_tools_host=false"
 		"uefi_signed=false"
-		"disable_d_i=true"
-		"do_zfs=false"
-		"do_dkms_nvidia=false"
-		"do_dkms_nvidia_server=false"
-		"do_dkms_vbox=false"
-		"do_dkms_wireguard=false"
-		"dkms_exclude=v4l2loopback"
+		"dkms_exclude=zfs ipu6 iwlwifi v4l2loopback usbio"
 		"flavours=$platform"
 		"abinum=${delphix_abinum}"
 	)
@@ -262,11 +268,7 @@ function kernel_update_upstream() {
 	local tag_prefix_flavour
 	case "${platform}" in
 	generic)
-		if [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then
-			tag_prefix_flavour="Ubuntu-hwe"
-		else
-			tag_prefix_flavour="Ubuntu"
-		fi
+		tag_prefix_flavour="Ubuntu"
 		;;
 	aws | azure | gcp | oracle)
 		tag_prefix_flavour="Ubuntu-${platform}"

lib/common.sh

@@ -28,7 +28,7 @@ export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"
 #
 export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"
 
-export UBUNTU_DISTRIBUTION="focal"
+export UBUNTU_DISTRIBUTION="noble"
 
 #
 # We currently support getting the linux kernel from 3 different sources:
@@ -616,6 +616,10 @@ function delphix_revision() {
 	echo "delphix.$(date '+%Y.%m.%d.%H.%M')"
 }
 
+function compare_versions() {
+	dpkg --compare-versions "$@"
+}
+
 function determine_dependencies_base_url() {
 	[[ -n "$DEPENDENCIES_BASE_URL" ]] && return
 
@@ -1109,21 +1113,13 @@ function get_kernel_version_for_platform_from_apt() {
 	# image for that particular platform. For instance, Ubuntu has a
 	# meta-package for AWS called 'linux-image-aws', which depends on
 	# package 'linux-image-4.15.0-1027-aws'. The latter is the linux image
-	# for kernel version '4.15.0-1027-aws'. We use this depenency to figure
+	# for kernel version '4.15.0-1027-aws'. We use this dependency to figure
 	# out the default kernel version for a given platform.
 	#
-	# The "generic" platform is a special case, since we want to use the
-	# hwe kernel image instead of the regular generic image.
-	#
 	# Note that while the default kernel is usually also the latest
 	# available, it is not always the case.
 	#
-
-	if [[ "$platform" != generic ]] && [[ "$UBUNTU_DISTRIBUTION" == focal ]]; then
-		package="linux-image-${platform}"
-	else
-		package="linux-image-${platform}-hwe-20.04"
-	fi
+	package="linux-image-${platform}"
 
 	if [[ "$(apt-cache show --no-all-versions "$package" \
 		2>/dev/null | grep Depends)" =~ linux-image-([^,]*-${platform}) ]]; then

package-lists/build/main.pkgs

@@ -2,9 +2,6 @@
 # List of non-kernel packages to be included in the Delphix Appliance.
 #
 
-adoptopenjdk
-bcc
-bpftrace
 challenge-response
 cloud-init
 crash-python
@@ -13,20 +10,17 @@ delphix-go
 delphix-platform
 delphix-rust
 delphix-sso-app
-drgn
 docker-python-image
+drgn
 dwarves
-fio
 fluentd-gems
 gdb-python
 grub2
 host-jdks
 libkdumpfile
-make-jpkg
 makedumpfile
 masking
 misc-debs
-mold
 nfs-utils
 performance-diagnostics
 ptools

packages/adoptopenjdk/config.sh

@@ -23,7 +23,7 @@ function prepare() {
 	logmust install_pkgs \
 		debhelper \
 		dpkg-dev \
-		llvm-12
+		llvm-14
 	logmust install_kernel_headers
 	logmust install_pkgs "$DEPDIR"/dwarves/*.deb
 }

packages/bcc/config.sh

@@ -29,17 +29,15 @@ source "$PWD/lib/common.sh"
 
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git"
 
-PACKAGE_DEPENDENCIES="adoptopenjdk"
 SKIP_COPYRIGHTS_CHECK=true
 
 function prepare() {
-	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+	echo "Nothing to prepare"
 }
 
 function build() {
 	export JAVA_HOME
-	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
-		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+	JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
 
 	logmust cd "$WORKDIR/repo"