LPD-68592 Import/export via creating and extracting tar files after image creation to avoid large Docker contexts

Author: holatuwol

.dockerignore

@@ -1 +1,17 @@
-**/.gitkeep
+**/.gitkeep
+.github/
+.gradle/
+build/
+buildSrc/
+data/
+dumps/
+exported_data/
+build.gradle
+gradle.properties
+gradle-local.properties
+gradlew
+gradlew.bat
+README.markdown
+settings.gradle
+source-formatter-suppressions.xml
+source-formatter.properties

Dockerfile.helper

@@ -1,5 +1,3 @@
-import com.bmuschko.gradle.docker.tasks.container.DockerCopyFileFromContainer
-
 import com.liferay.docker.workspace.environments.Util
 
 import groovy.json.JsonSlurper
@@ -32,26 +30,76 @@ withZone(ZoneId.systemDefault())
 
 String timestamp = formatter.format(now)
 
-tasks.register("exportContainerData", DockerCopyFileFromContainer) {
-	containerId = "${config.namespace}-data-helper"
-
+tasks.register("exportContainerData") {
 	doFirst {
-		mkdir hostPath.get()
+		String hostPath = file("exported_data/data_${config.namespace}_${timestamp}").absolutePath
 
-		waitForCommand("docker compose create data-helper")
-	}
+		getExistingVolumeNames().each {
+			String volumeName ->
 
-	doLast {
-		updateGradleLocalProperties(["lr.docker.environment.data.directory" : hostPath.get()])
+			String backupFileName = "${volumeName.substring(config.namespace.length() + 1)}.tar"
 
-		logger.lifecycle("\nUpdated gradle-local.properties with the new value:\nlr.docker.environment.data.directory=${hostPath.get()}")
-	}
+			if (backupFileName.equals("dumps.tar")) {
+				return
+			}
+
+			println "Creating backup of volume ${volumeName} in ${hostPath}/${backupFileName}"
+			waitForCommand("docker run --rm -v ${volumeName}:/source -v ${hostPath}:/target busybox:latest tar --create --file=/target/${backupFileName} --directory=/source .")
+			println "Created backup of volume ${volumeName} in ${hostPath}/${backupFileName}"
+		}
 
-	hostPath = provider {
-		"exported_data/data_${config.namespace}_${timestamp}"
+		updateGradleLocalProperties(["lr.docker.environment.data.directory" : hostPath])
+
+		logger.lifecycle("\nUpdated gradle-local.properties with the new value:\nlr.docker.environment.data.directory=${hostPath}")
 	}
+}
+
+tasks.register("importContainerData") {
+	mustRunAfter ":buildDockerImage"
+
+	doFirst {
+		List<String> previousVolumeNames = getExistingVolumeNames()
+
+		waitForCommand("docker compose build")
+		waitForCommand("docker compose create")
+
+		List<String> currentVolumeNames = getExistingVolumeNames()
+
+		project.file(config.dataDirectory).listFiles().each {
+			File backupFile ->
+
+			if (backupFile.isFile() && !backupFile.name.endsWith(".tar")) {
+				return
+			}
+
+			String volumeName = "${config.namespace}_${backupFile.name}"
+
+			if (backupFile.isFile()) {
+				volumeName = volumeName.substring(0, volumeName.length() - 4)
+			}
+
+			if (!currentVolumeNames.contains(volumeName)) {
+				return
+			}
+
+			if (previousVolumeNames.contains(volumeName)) {
+				return
+			}
+
+			println "Restoring backup of volume ${volumeName} using ${backupFile.absolutePath}"
 
-	remotePath = "/container-data"
+			if (backupFile.isFile()) {
+				waitForCommand("docker run --rm -v ${volumeName}:/target -v ${backupFile.absolutePath}:/source.tar busybox:latest tar --extract --file=/source.tar --directory=/target/")
+			}
+			else {
+				waitForCommand("docker run --rm -v ${volumeName}:/target -v ${backupFile.absolutePath}:/source busybox:latest sh -c 'cp --update --recursive /source/* /target/'")
+			}
+
+			println "Restored backup of volume ${volumeName} using ${backupFile.absolutePath}"
+
+			println waitForCommand("docker run --rm -v ${volumeName}:/${volumeName} busybox:latest du -sh /${volumeName}/")
+		}
+	}
 }
 
 tasks.register("importDocumentLibraryStructure") {
@@ -160,6 +208,7 @@ composeUp {
 	}
 
 	dependsOn ":buildDockerImage"
+	dependsOn ":importContainerData"
 	dependsOn ":importDatabaseDumps"
 
 	doFirst {

build.gradle

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   db2:/container-data/db2
     database:
         build: ./compose-recipes/db2
         container_name: ${NAMESPACE}-database-db2

compose-recipes/db2/service.db2.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   elasticsearch:/container-data/elasticsearch
     elasticsearch:
         build: ./compose-recipes/elasticsearch
         container_name: ${NAMESPACE}-elasticsearch

compose-recipes/elasticsearch/service.elasticsearch.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   liferay:/container-data/liferay
     liferay:
         container_name: ${NAMESPACE}-liferay
         env_file:

compose-recipes/liferay/service.liferay.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   mariadb:/container-data/mariadb
     database:
         build: ./compose-recipes/mariadb
         container_name: ${NAMESPACE}-database-mariadb

compose-recipes/mariadb/service.mariadb.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   mysql:/container-data/mysql
     database:
         build: ./compose-recipes/mysql
         container_name: ${NAMESPACE}-database-mysql

compose-recipes/mysql/service.mysql.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   postgres:/container-data/postgres
     database:
         build: ./compose-recipes/postgres
         container_name: ${NAMESPACE}-database-postgres

compose-recipes/postgres/service.postgres.yaml

@@ -1,7 +1,4 @@
 services:
-    data-helper:
-        volumes:
-            -   webserver-http:/container-data/webserver-http
     webserver:
         build: ./compose-recipes/webserver-http
         container_name: ${NAMESPACE}-webserver-http