AGENT-1193: Add --mirror-path flag to support pre-mirrored images

docs/appliance-config.md

@@ -17,7 +17,8 @@
 | userCorePass               |                                | Yes      | string  | Password of user 'core' for connecting from console.                                                                                                                                                                                                                                                                                                                                                          |                        
 | imageRegistry              |                                | Yes      |         | Local image registry details (used when building the appliance)                                                                                                                                                                                                                                                                                                                                               |
 | imageRegistry.uri          |         | Yes      | string  | The URI for the image.                                                                                                                                                                                                                                                                                                                                             |                                                                                                   
-| imageRegistry.port         | 5005                           | Yes      | integer | The image registry container TCP port to bind. A valid port number is between 1024 and 65535.                                                                                                                                                                                                                                                                                                                 |                                                                                  
+| imageRegistry.port         | 5005                           | Yes      | integer | The image registry container TCP port to bind. A valid port number is between 1024 and 65535.                                                                                                                                                                                                                                                                                                                 |
+| mirrorPath                 |                                | Yes      | string  | Path to pre-mirrored images from oc-mirror workspace. When provided, skips image mirroring and uses the pre-mirrored registry data. The path should point to an oc-mirror workspace directory containing a 'data' subdirectory.                                                                                                                                                                               |
 | stopLocalRegistry          | false                          | Yes      | bool    | Stop the local registry post cluster installation. Note that additional images and operators won't be available when stopped.                                                                                                                                                                                                                                                                                 |
 | createPinnedImageSets      | false                          | Yes      | bool    | Create PinnedImageSets for both the master and worker MCPs. The PinnedImageSets will include all the images included in the appliance disk image. Requires openshift version 4.16 or above. **WARNING:** As of 4.18, PinnedImageSets feature is still not GA. Thus, enabling it will set the cluster to tech preview, which means the cluster cannot be upgraded (i.e. should only be used for testing purposes). |
 | enableDefaultSources       | false                          | Yes      | bool    | Enable all default CatalogSources (on openshift-marketplace namespace). Should be disabled for disconnected environments.                                                                                                                                                                                                                                                                                     |

docs/user-guide.md

@@ -346,6 +346,27 @@ metadata:
   namespace: "openshift-cnv"
 ```
 
+### Using pre-mirrored images (Optional)
+
+If you have already mirrored OCP release images using `oc-mirror` on another system, you can skip the mirroring step during the build by providing the path to the oc-mirror workspace.
+
+Add the `mirrorPath` field to your `appliance-config.yaml`:
+```yaml
+apiVersion: v1beta1
+kind: ApplianceConfig
+ocpRelease:
+  version: 4.14
+  channel: candidate
+  cpuArchitecture: x86_64
+diskSizeGB: 200
+pullSecret: '{"auths":{<redacted>}}'
+sshKey: <redacted>
+# Path to oc-mirror workspace directory containing mirrored images
+mirrorPath: /path/to/mirror/workspace
+```
+
+The `mirrorPath` should point to the directory created by `oc-mirror` that contains a `data` subdirectory with the mirrored registry data.
+
 ### Build the disk image
 * Make sure you have enough free disk space.
   * The amount of space needed is defined by the configured `diskSizeGB` value mentioned above, which is at least 150GiB.

pkg/asset/config/appliance_config.go

@@ -153,6 +153,12 @@ pullSecret: pull-secret
   # [Optional]
   # useBinary: %t
 
+# Path to pre-mirrored images from oc-mirror workspace.
+# When provided, skips image mirroring and uses the pre-mirrored registry data.
+# The path should point to an oc-mirror workspace directory containing a 'data' subdirectory.
+# [Optional]
+# mirrorPath: /path/to/mirror/workspace
+
 # Enable all default CatalogSources (on openshift-marketplace namespace).
 # Should be disabled for disconnected environments.
 # Default: false
@@ -378,7 +384,7 @@ func (a *ApplianceConfig) GetRelease() (string, string, error) {
 				return "", "", nil
 			}
 			releaseDigest = strings.Trim(releaseDigest, "'")
-			releaseImage = fmt.Sprintf("%s@%s", strings.Split(releaseImage, ":")[0], releaseDigest)
+			releaseImage = fmt.Sprintf("%s@%s", releaseImage, releaseDigest)
 		}
 		logrus.Debugf("Release image: %s", releaseImage)
 	}
@@ -430,6 +436,11 @@ func (a *ApplianceConfig) validateConfig(f asset.FileFetcher) field.ErrorList {
 		}
 	}
 
+	// Validate mirrorPath
+	if err := a.validateMirrorPath(); err != nil {
+		allErrs = append(allErrs, err...)
+	}
+
 	return allErrs
 }
 
@@ -553,6 +564,41 @@ func (a *ApplianceConfig) validatePinnedImageSet() error {
 	return nil
 }
 
+func (a *ApplianceConfig) validateMirrorPath() field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if a.Config.MirrorPath != nil {
+		mirrorPath := swag.StringValue(a.Config.MirrorPath)
+		if mirrorPath != "" {
+			// Validate mirror path exists and is a directory
+			info, err := os.Stat(mirrorPath)
+			if err != nil {
+				if os.IsNotExist(err) {
+					allErrs = append(allErrs, field.Invalid(field.NewPath("mirrorPath"),
+						mirrorPath, "mirror path does not exist"))
+				} else {
+					allErrs = append(allErrs, field.Invalid(field.NewPath("mirrorPath"),
+						mirrorPath, fmt.Sprintf("failed to access mirror path: %v", err)))
+				}
+			} else if !info.IsDir() {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("mirrorPath"),
+					mirrorPath, "mirror path must be a directory"))
+			} else {
+				// Validate data subdirectory exists
+				dataDir := filepath.Join(mirrorPath, "data")
+				if _, err := os.Stat(dataDir); err != nil {
+					allErrs = append(allErrs, field.Invalid(field.NewPath("mirrorPath"),
+						mirrorPath, "mirror path must contain a 'data' subdirectory (expected oc-mirror workspace structure)"))
+				}
+			}
+
+			logrus.Infof("Using pre-mirrored images from: %s", mirrorPath)
+		}
+	}
+
+	return allErrs
+}
+
 func (a *ApplianceConfig) storePullSecret() error {
 	// Get home dir (~)
 	homeDir, err := os.UserHomeDir()

pkg/asset/data/data_iso.go

@@ -8,6 +8,7 @@ import (
 	"github.com/go-openapi/swag"
 	"github.com/openshift/appliance/pkg/asset/config"
 	"github.com/openshift/appliance/pkg/consts"
+	"github.com/openshift/appliance/pkg/executer"
 	"github.com/openshift/appliance/pkg/genisoimage"
 	"github.com/openshift/appliance/pkg/log"
 	"github.com/openshift/appliance/pkg/registry"
@@ -122,7 +123,39 @@ func (a *DataISO) Generate(dependencies asset.Parents) error {
 	)
 	spinner.FileToMonitor = dataIsoName
 	imageGen := genisoimage.NewGenIsoImage(nil)
-	if err = imageGen.GenerateImage(envConfig.CacheDir, dataIsoName, filepath.Join(envConfig.TempDir, dataDir), dataVolumeName); err != nil {
+
+	// When mirror-path is provided, copy the Docker registry data from mirror-path/data
+	// to temp/data so it's in the same location as the registry container image (images/registry/registry.tar)
+	registryDataSourcePath := filepath.Join(envConfig.TempDir, dataDir)
+	if applianceConfig.Config.MirrorPath != nil && swag.StringValue(applianceConfig.Config.MirrorPath) != "" {
+		mirrorDataPath := filepath.Join(swag.StringValue(applianceConfig.Config.MirrorPath), dataDir)
+		dockerSrcPath := filepath.Join(mirrorDataPath, "docker")
+		dockerDstPath := filepath.Join(registryDataSourcePath, "docker")
+
+		logrus.Infof("Copying Docker registry data from %s to %s", dockerSrcPath, dockerDstPath)
+
+		// Validate source directory exists
+		if _, err := os.Stat(dockerSrcPath); err != nil {
+			return log.StopSpinner(spinner, fmt.Errorf("docker registry data not found at %s (mirror-path may be invalid): %w", dockerSrcPath, err))
+		}
+
+		// Create destination directory
+		if err := os.MkdirAll(registryDataSourcePath, os.ModePerm); err != nil {
+			return log.StopSpinner(spinner, fmt.Errorf("failed to create directory for Docker registry data: %w", err))
+		}
+
+		// Copy directory recursively using cp command
+		// Note: Paths are safe here as they're program-generated from validated inputs
+		cpCmd := fmt.Sprintf("cp -r %s %s", dockerSrcPath, dockerDstPath)
+		exec := executer.NewExecuter()
+		if _, err := exec.Execute(cpCmd); err != nil {
+			return log.StopSpinner(spinner, fmt.Errorf("failed to copy Docker registry data from %s to %s: %w", dockerSrcPath, dockerDstPath, err))
+		}
+
+		logrus.Infof("Successfully copied Docker registry data")
+	}
+
+	if err = imageGen.GenerateImage(envConfig.CacheDir, dataIsoName, registryDataSourcePath, dataVolumeName); err != nil {
 		return log.StopSpinner(spinner, err)
 	}
 	return log.StopSpinner(spinner, a.updateAsset(envConfig))

pkg/release/release.go

@@ -99,9 +99,54 @@ func (r *release) GetImageFromRelease(imageName string) (string, error) {
 		return "", err
 	}
 
+	// Fix incomplete image references from local registries
+	image, err = r.fixImageReference(image, swag.StringValue(r.ApplianceConfig.Config.OcpRelease.URL))
+	if err != nil {
+		return "", err
+	}
+
 	return image, nil
 }
 
+// fixImageReference repairs incomplete image references returned by oc adm release info
+// when querying local registries with custom ports. The oc command may return references
+// like "registry.example.com@sha256:..." which are missing the port and repository path.
+// This function reconstructs the full reference from the release URL.
+func (r *release) fixImageReference(imageRef, releaseURL string) (string, error) {
+	// Check if this looks like an incomplete reference (has @ but no / after the hostname)
+	// Example: "virthost.ostest.test.metalkube.org@sha256:abc123"
+	if strings.Contains(imageRef, "@") && !strings.Contains(strings.Split(imageRef, "@")[0], "/") {
+		logrus.Debugf("Detected incomplete image reference: %s", imageRef)
+
+		// Extract digest from the incomplete reference
+		parts := strings.SplitN(imageRef, "@", 2)
+		if len(parts) != 2 {
+			return imageRef, nil // Return as-is if we can't parse it
+		}
+		digest := parts[1]
+
+		// Extract registry/port/repo from release URL
+		// Example: "virthost.ostest.test.metalkube.org:5000/openshift/release-images:tag"
+		// We want: "virthost.ostest.test.metalkube.org:5000/openshift/release-images"
+		releaseRef := releaseURL
+
+		// Remove tag or digest from release URL
+		if idx := strings.LastIndex(releaseRef, ":"); idx > strings.LastIndex(releaseRef, "/") {
+			releaseRef = releaseRef[:idx]
+		}
+		if idx := strings.Index(releaseRef, "@"); idx != -1 {
+			releaseRef = releaseRef[:idx]
+		}
+
+		// Reconstruct full image reference
+		fixedRef := fmt.Sprintf("%s@%s", releaseRef, digest)
+		logrus.Debugf("Fixed image reference: %s -> %s", imageRef, fixedRef)
+		return fixedRef, nil
+	}
+
+	return imageRef, nil
+}
+
 func (r *release) extractFileFromImage(image, file, outputDir string) (string, error) {
 	cmd := fmt.Sprintf(templateImageExtract, file, outputDir, image)
 	logrus.Debugf("extracting %s to %s, %s", file, outputDir, cmd)
@@ -138,35 +183,70 @@ func (r *release) execute(command string) (string, error) {
 }
 
 func (r *release) mirrorImages(imageSetFile, blockedImages, additionalImages, operators string) error {
-	if err := templates.RenderTemplateFile(
-		imageSetFile,
-		templates.GetImageSetTemplateData(r.ApplianceConfig, blockedImages, additionalImages, operators),
-		r.EnvConfig.TempDir); err != nil {
+	var tempDir string
+
+	// If a mirror path is provided in appliance-config, use it directly instead of running oc-mirror
+	var mirrorPath string
+	if r.ApplianceConfig.Config.MirrorPath != nil {
+		mirrorPath = *r.ApplianceConfig.Config.MirrorPath
+	}
+
+	if mirrorPath != "" {
+		logrus.Infof("Using pre-mirrored images from: %s", mirrorPath)
+		tempDir = mirrorPath
+	} else {
+		// Normal mirroring flow - run oc-mirror
+		if err := templates.RenderTemplateFile(
+			imageSetFile,
+			templates.GetImageSetTemplateData(r.ApplianceConfig, blockedImages, additionalImages, operators),
+			r.EnvConfig.TempDir); err != nil {
+			return err
+		}
+
+		imageSetFilePath, err := filepath.Abs(templates.GetFilePathByTemplate(imageSetFile, r.EnvConfig.TempDir))
+		if err != nil {
+			return err
+		}
+
+		tempDir = filepath.Join(r.EnvConfig.TempDir, "oc-mirror")
+		registryPort := swag.IntValue(r.ApplianceConfig.Config.ImageRegistry.Port)
+		cmd := fmt.Sprintf(ocMirror, imageSetFilePath, registryPort, tempDir)
+
+		logrus.Debugf("Fetching image from OCP release (%s)", cmd)
+		result, err := r.execute(cmd)
+		logrus.Debugf("mirroring result: %s", result)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Copy generated yaml files to cache dir (works for both mirror path and oc-mirror output)
+	if err := r.copyOutputYamls(tempDir); err != nil {
 		return err
 	}
 
-	imageSetFilePath, err := filepath.Abs(templates.GetFilePathByTemplate(imageSetFile, r.EnvConfig.TempDir))
-	if err != nil {
+	// Copy mapping file (works for both mirror path and oc-mirror output)
+	if err := r.copyMappingFile(tempDir); err != nil {
 		return err
 	}
 
-	tempDir := filepath.Join(r.EnvConfig.TempDir, "oc-mirror")
-	registryPort := swag.IntValue(r.ApplianceConfig.Config.ImageRegistry.Port)
-	cmd := fmt.Sprintf(ocMirror, imageSetFilePath, registryPort, tempDir)
+	return nil
+}
 
-	logrus.Debugf("Fetching image from OCP release (%s)", cmd)
-	result, err := r.execute(cmd)
-	logrus.Debugf("mirroring result: %s", result)
+func (r *release) copyMappingFile(ocMirrorDir string) error {
+	mappingFiles, err := filepath.Glob(filepath.Join(ocMirrorDir, fmt.Sprintf("results-*/%s", consts.OcMirrorMappingFileName)))
 	if err != nil {
 		return err
 	}
 
-	// Copy generated yaml files to cache dir
-	if err = r.copyOutputYamls(tempDir); err != nil {
-		return err
+	// The slice returned from Glob will have a single filename when running the application, but it will be empty when running the unit-tests since they don't create the files "oc mirror" generates
+	for _, mappingFile := range mappingFiles {
+		if err := fileutil.CopyFile(mappingFile, filepath.Join(r.EnvConfig.CacheDir, consts.OcMirrorMappingFileName)); err != nil {
+			return err
+		}
 	}
 
-	return err
+	return nil
 }
 
 func (r *release) copyOutputYamls(ocMirrorDir string) error {
@@ -186,6 +266,14 @@ func (r *release) copyOutputYamls(ocMirrorDir string) error {
 		internalRegistryURI := fmt.Sprintf("%s:%d", registry.RegistryDomain, registry.RegistryPort)
 		newYaml := strings.ReplaceAll(string(yamlBytes), buildRegistryURI, internalRegistryURI)
 
+		// Add IDMS entry for local registry mirror if using a custom release URL
+		if filepath.Base(yamlPath) == "idms-oc-mirror.yaml" {
+			newYaml, err = r.addLocalRegistryIDMS(newYaml, internalRegistryURI)
+			if err != nil {
+				return err
+			}
+		}
+
 		// Write edited yamls to cache
 		if err = r.OSInterface.MkdirAll(filepath.Join(r.EnvConfig.CacheDir, consts.OcMirrorResourcesDir), os.ModePerm); err != nil {
 			return err
@@ -198,6 +286,55 @@ func (r *release) copyOutputYamls(ocMirrorDir string) error {
 	return nil
 }
 
+// addLocalRegistryIDMS adds an IDMS entry for the local registry mirror when using
+// a custom release URL (not upstream quay.io). This ensures that pulls from the
+// registry mirror are redirected to the appliance's internal registry.
+func (r *release) addLocalRegistryIDMS(yamlContent, internalRegistryURI string) (string, error) {
+	releaseURL := swag.StringValue(r.ApplianceConfig.Config.OcpRelease.URL)
+
+	// Check if using a custom registry (not upstream quay.io)
+	if !strings.Contains(releaseURL, "quay.io") && !strings.Contains(releaseURL, "registry.ci.openshift.org") {
+		// Extract registry host:port from release URL
+		localRegistry := releaseURL
+		// Remove digest if present
+		if idx := strings.Index(localRegistry, "@"); idx != -1 {
+			localRegistry = localRegistry[:idx]
+		}
+		// Remove tag if present (after last colon that comes after last slash)
+		if lastSlash := strings.LastIndex(localRegistry, "/"); lastSlash != -1 {
+			if lastColon := strings.LastIndex(localRegistry[lastSlash:], ":"); lastColon != -1 {
+				localRegistry = localRegistry[:lastSlash+lastColon]
+			}
+		}
+
+		// Extract just the registry host:port (without repository path)
+		registryHost := localRegistry
+		if idx := strings.Index(localRegistry, "/"); idx != -1 {
+			registryHost = localRegistry[:idx]
+		}
+
+		logrus.Infof("Adding IDMS entry for registry mirror: %s -> %s", registryHost, internalRegistryURI)
+
+		// Append IDMS entry for registry mirror
+		// This maps all pulls from the registry mirror to the appliance's internal registry
+		additionalIDMS := fmt.Sprintf(`---
+apiVersion: config.openshift.io/v1
+kind: ImageDigestMirrorSet
+metadata:
+  name: local-registry-mirror
+spec:
+  imageDigestMirrors:
+  - mirrors:
+    - %s
+    source: %s
+`, internalRegistryURI, registryHost)
+
+		return yamlContent + "\n" + additionalIDMS, nil
+	}
+
+	return yamlContent, nil
+}
+
 func (r *release) generateImagesList(images *[]types.Image) string {
 	if images == nil {
 		return ""

pkg/types/appliance_config_type.go

@@ -18,6 +18,7 @@ type ApplianceConfig struct {
 	SshKey                *string        `json:"sshKey"`
 	UserCorePass          *string        `json:"userCorePass"`
 	ImageRegistry         *ImageRegistry `json:"imageRegistry"`
+	MirrorPath            *string        `json:"mirrorPath,omitempty"`
 	EnableDefaultSources  *bool          `json:"enableDefaultSources"`
 	EnableFips            *bool          `json:"enableFips"`
 	StopLocalRegistry     *bool          `json:"stopLocalRegistry"`