From f76509793c4c922fb50ef618264b2f6ff5117363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 5 May 2025 13:23:45 +0200 Subject: [PATCH 1/5] Ensure LayerArchiveSupport always contains valid layerName --- .../svm/hosted/NativeImageGenerator.java | 2 +- .../HostedImageLayerBuildingSupport.java | 9 ++++---- .../imagelayer/LayerArchiveSupport.java | 21 +++++++------------ .../imagelayer/LoadLayerArchiveSupport.java | 4 ++-- .../imagelayer/WriteLayerArchiveSupport.java | 7 +++---- 5 files changed, 19 insertions(+), 24 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index 5f6e5f986b3e..97335405c17d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -766,7 +766,7 @@ protected void doRun(Map entryPoints, JavaMainSupport j try (StopTimer t = TimerCollection.createTimerAndStart(TimerCollection.Registry.ARCHIVE_LAYER)) { if (ImageLayerBuildingSupport.buildingSharedLayer()) { ImageSingletonsSupportImpl.HostedManagement.persist(); - HostedImageLayerBuildingSupport.singleton().archiveLayer(imageName); + HostedImageLayerBuildingSupport.singleton().archiveLayer(); } } reporter.printCreationEnd(image.getImageFileSize(), heap.getLayerObjectCount(), image.getImageHeapSize(), codeCache.getCodeAreaSize(), numCompilations, image.getDebugInfoSize(), diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 176f8b4bcbf7..f9cb0337875d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -124,9 +124,9 @@ public LoadLayerArchiveSupport getLoadLayerArchiveSupport() { return loadLayerArchiveSupport; } - public void archiveLayer(String imageName) { + public void archiveLayer() { writer.dumpFiles(); - writeLayerArchiveSupport.write(imageName); + writeLayerArchiveSupport.write(); } public SharedLayerSnapshotCapnProtoSchemaHolder.SharedLayerSnapshot.Reader getSnapshot() { @@ -261,8 +261,9 @@ public static HostedImageLayerBuildingSupport initialize(HostedOptionValues valu WriteLayerArchiveSupport writeLayerArchiveSupport = null; ArchiveSupport archiveSupport = new ArchiveSupport(false); + String layerName = SubstrateOptions.Name.getValue(values); if (buildingSharedLayer) { - writeLayerArchiveSupport = new WriteLayerArchiveSupport(archiveSupport, imageClassLoader.classLoaderSupport.getLayerFile()); + writeLayerArchiveSupport = new WriteLayerArchiveSupport(layerName, archiveSupport, imageClassLoader.classLoaderSupport.getLayerFile()); } SVMImageLayerSingletonLoader singletonLoader = null; LoadLayerArchiveSupport loadLayerArchiveSupport = null; @@ -270,7 +271,7 @@ public static HostedImageLayerBuildingSupport initialize(HostedOptionValues valu List graphsChannels = null; if (buildingExtensionLayer) { Path layerFileName = getLayerUseValue(values); - loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerFileName, archiveSupport); + loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerName, layerFileName, archiveSupport); FilePaths filePaths = new FilePaths(loadLayerArchiveSupport.getSnapshotPath(), loadLayerArchiveSupport.getSnapshotGraphsPath()); List loadPaths = List.of(filePaths); snapshots = new ArrayList<>(); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index 7e6be584c245..440ff3f91b95 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -54,9 +54,9 @@ public class LayerArchiveSupport { protected final LayerProperties layerProperties; protected final ArchiveSupport archiveSupport; - public LayerArchiveSupport(ArchiveSupport archiveSupport) { + public LayerArchiveSupport(String layerName, ArchiveSupport archiveSupport) { this.archiveSupport = archiveSupport; - this.layerProperties = new LayerArchiveSupport.LayerProperties(); + this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); } protected static final Path layerPropertiesFileName = Path.of("META-INF/nilayer.properties"); @@ -73,8 +73,10 @@ public final class LayerProperties { private final Map properties; - LayerProperties() { - this.properties = new HashMap<>(); + LayerProperties(String layerName) { + properties = new HashMap<>(); + VMError.guarantee(layerName != null && !layerName.isEmpty(), "LayerProperties entry " + PROPERTY_KEY_IMAGE_LAYER_NAME + " requires non-empty layer-name"); + properties.put(PROPERTY_KEY_IMAGE_LAYER_NAME, layerName); } void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { @@ -87,6 +89,7 @@ void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { properties.putAll(ArchiveSupport.loadProperties(layerPropertiesFile)); verifyVersion(layerFileName); + info("Loaded layer %s from %s", layerName(), layerFileName); String niVendor = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, "unknown"); String javaVmVendor = System.getProperty("java.vm.vendor"); @@ -98,7 +101,6 @@ void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { // GR-55581 will enforce platform compatibility String currentPlatform = niPlatform.equals(platform()) ? "" : " != '" + platform() + "'"; String layerCreationTimestamp = properties.getOrDefault(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ""); - info("Loaded layer from %s", layerFileName); info("Layer created at '%s'", ArchiveSupport.parseTimestamp(layerCreationTimestamp)); info("Using version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", niVersion, currentVersion, niVendor, currentVendor, niPlatform, currentPlatform); } @@ -143,15 +145,8 @@ void write() { } } - public void writeLayerName(String layerName) { - properties.put(PROPERTY_KEY_IMAGE_LAYER_NAME, layerName); - } - public String layerName() { - VMError.guarantee(!properties.isEmpty(), "Property file is no loaded."); - String name = properties.get(PROPERTY_KEY_IMAGE_LAYER_NAME); - VMError.guarantee(name != null, "Property " + PROPERTY_KEY_IMAGE_LAYER_NAME + " must be set."); - return name; + return properties.get(PROPERTY_KEY_IMAGE_LAYER_NAME); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 527d44abab13..1f8978f00c37 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -38,8 +38,8 @@ public class LoadLayerArchiveSupport extends LayerArchiveSupport { private final AtomicBoolean deleteLayerRoot = new AtomicBoolean(); - public LoadLayerArchiveSupport(Path layerFile, ArchiveSupport archiveSupport) { - super(archiveSupport); + public LoadLayerArchiveSupport(String layerName, Path layerFile, ArchiveSupport archiveSupport) { + super(layerName, archiveSupport); Path inputLayerLocation = validateLayerFile(layerFile); expandedInputLayerDir = this.archiveSupport.createTempDir(LAYER_TEMP_DIR_PREFIX, deleteLayerRoot); this.archiveSupport.expandJarToDir(inputLayerLocation, expandedInputLayerDir, deleteLayerRoot); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 707b8a3333e5..549263cec067 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -41,8 +41,8 @@ public class WriteLayerArchiveSupport extends LayerArchiveSupport { /** The original location of the layer output file. */ private final Path outputLayerLocation; - public WriteLayerArchiveSupport(ArchiveSupport archiveSupport, Path layerFile) { - super(archiveSupport); + public WriteLayerArchiveSupport(String layerName, ArchiveSupport archiveSupport, Path layerFile) { + super(layerName, archiveSupport); this.outputLayerLocation = validateLayerFile(layerFile); } @@ -68,8 +68,7 @@ private static Path validateLayerFile(Path layerFile) { return layerFile; } - public void write(String imageName) { - layerProperties.writeLayerName(String.valueOf(imageName)); + public void write() { layerProperties.write(); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(outputLayerLocation), archiveSupport.createManifest())) { Path imageBuilderOutputDir = NativeImageGenerator.getOutputDirectory(); From 284c4bb9b00c8223c8bfdf2f0c63ff3544ee8c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 5 May 2025 14:01:55 +0200 Subject: [PATCH 2/5] Prevent using layer with mismatching platform or builder VM version --- .../imagelayer/LayerArchiveSupport.java | 72 ++++++++++++++----- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index 440ff3f91b95..6b5ed3ab1861 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Properties; import com.oracle.svm.core.OS; @@ -66,9 +67,7 @@ public final class LayerProperties { private static final String PROPERTY_KEY_LAYER_FILE_VERSION_MAJOR = "LayerFileVersionMajor"; private static final String PROPERTY_KEY_LAYER_FILE_VERSION_MINOR = "LayerFileVersionMinor"; private static final String PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP = "LayerFileCreationTimestamp"; - private static final String PROPERTY_KEY_NATIVE_IMAGE_PLATFORM = "NativeImagePlatform"; - private static final String PROPERTY_KEY_NATIVE_IMAGE_VENDOR = "NativeImageVendor"; - private static final String PROPERTY_KEY_NATIVE_IMAGE_VERSION = "NativeImageVersion"; + private static final String PROPERTY_KEY_LAYER_BUILDER_VM_PLATFORM = "BuilderVMPlatform"; private static final String PROPERTY_KEY_IMAGE_LAYER_NAME = "LayerName"; private final Map properties; @@ -79,6 +78,37 @@ public final class LayerProperties { properties.put(PROPERTY_KEY_IMAGE_LAYER_NAME, layerName); } + private record BuilderVMIdentifier(String vendor, String version) { + + private static final String PROPERTY_KEY_VM_VENDOR = "BuilderVMVendor"; + private static final String PROPERTY_KEY_VM_VERSION = "BuilderVMVersion"; + + BuilderVMIdentifier { + Objects.requireNonNull(vendor); + Objects.requireNonNull(version); + } + + static BuilderVMIdentifier system() { + return new BuilderVMIdentifier(System.getProperty("java.vm.vendor"), System.getProperty("java.vm.version")); + } + + static BuilderVMIdentifier load(Map properties) { + String vmVendor = properties.get(PROPERTY_KEY_VM_VENDOR); + String vmVersion = properties.get(PROPERTY_KEY_VM_VERSION); + return new BuilderVMIdentifier(vmVendor, vmVersion); + } + + public void store(Map properties) { + properties.put(PROPERTY_KEY_VM_VENDOR, vendor); + properties.put(PROPERTY_KEY_VM_VERSION, version); + } + + @Override + public String toString() { + return '\'' + vendor + ' ' + version + '\''; + } + } + void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { Path layerFileName = inputLayerLocation.getFileName(); Path layerPropertiesFile = expandedInputLayerDir.resolve(layerPropertiesFileName); @@ -88,24 +118,31 @@ void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { } properties.putAll(ArchiveSupport.loadProperties(layerPropertiesFile)); - verifyVersion(layerFileName); + verifyLayerFileVersion(layerFileName); info("Loaded layer %s from %s", layerName(), layerFileName); - String niVendor = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, "unknown"); - String javaVmVendor = System.getProperty("java.vm.vendor"); - String currentVendor = niVendor.equals(javaVmVendor) ? "" : " != '" + javaVmVendor + "'"; - String niVersion = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_VERSION, "unknown"); - String javaVmVersion = System.getProperty("java.vm.version"); - String currentVersion = niVersion.equals(javaVmVersion) ? "" : " != '" + javaVmVersion + "'"; - String niPlatform = properties.getOrDefault(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, "unknown"); - // GR-55581 will enforce platform compatibility - String currentPlatform = niPlatform.equals(platform()) ? "" : " != '" + platform() + "'"; + BuilderVMIdentifier layerBuilderVMIdentifier = BuilderVMIdentifier.load(properties); + if (!layerBuilderVMIdentifier.equals(BuilderVMIdentifier.system())) { + String message = String.format("The given layer file '%s' was created with an image builder running on %s. This image builder is using %s." + + " The given layer file can only be used with an image builder running the exact same version.", + layerFileName, layerBuilderVMIdentifier, BuilderVMIdentifier.system()); + throw UserError.abort(message); + } + + String niPlatform = properties.getOrDefault(PROPERTY_KEY_LAYER_BUILDER_VM_PLATFORM, "unknown"); + if (!niPlatform.equals(platform())) { + String message = String.format("The given layer file '%s' was created on platform '%s'. The current platform is '%s'." + + " The given layer file can only be used with an image builder running on that same platform.", + layerFileName, niPlatform, platform()); + throw UserError.abort(message); + } + String layerCreationTimestamp = properties.getOrDefault(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ""); info("Layer created at '%s'", ArchiveSupport.parseTimestamp(layerCreationTimestamp)); - info("Using version: '%s'%s (vendor '%s'%s) on platform: '%s'%s", niVersion, currentVersion, niVendor, currentVendor, niPlatform, currentPlatform); + info("Using version: %s on platform: '%s'", layerBuilderVMIdentifier, niPlatform); } - private void verifyVersion(Path layerFileName) { + private void verifyLayerFileVersion(Path layerFileName) { String fileVersionKey = PROPERTY_KEY_LAYER_FILE_VERSION_MAJOR; try { int major = Integer.parseInt(properties.getOrDefault(fileVersionKey, "-1")); @@ -127,9 +164,8 @@ private void verifyVersion(Path layerFileName) { void write() { properties.put(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ArchiveSupport.currentTime()); - properties.put(PROPERTY_KEY_NATIVE_IMAGE_PLATFORM, platform()); - properties.put(PROPERTY_KEY_NATIVE_IMAGE_VENDOR, System.getProperty("java.vm.vendor")); - properties.put(PROPERTY_KEY_NATIVE_IMAGE_VERSION, System.getProperty("java.vm.version")); + properties.put(PROPERTY_KEY_LAYER_BUILDER_VM_PLATFORM, platform()); + BuilderVMIdentifier.system().store(properties); Path layerPropertiesFile = NativeImageGenerator.getOutputDirectory().resolve(layerPropertiesFileName); Path parent = layerPropertiesFile.getParent(); if (parent == null) { From e05e1f0a14d1a6083143bc1e7cac7cf6306f3e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Mon, 5 May 2025 16:00:20 +0200 Subject: [PATCH 3/5] Allow LayerCreate ExtendedOption values as first entry --- .../svm/hosted/driver/LayerOptionsSupport.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java index 1ab8538c8f9b..ba75884a0228 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java @@ -26,9 +26,11 @@ import java.nio.file.Path; import java.util.List; +import java.util.stream.Stream; import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.imagelayer.LayerArchiveSupport; public class LayerOptionsSupport extends IncludeOptionsSupport { @@ -43,8 +45,20 @@ public static LayerOption parse(String layerOptionValue) { public static LayerOption parse(List options) { VMError.guarantee(!options.isEmpty()); - ExtendedOption[] extendedOptions = options.stream().skip(1).map(ExtendedOption::parse).toArray(ExtendedOption[]::new); - return new LayerOption(Path.of(options.getFirst()), extendedOptions); + + Stream optionsStream = options.stream(); + String fileName; + String first = options.getFirst(); + if (first.isEmpty() || first.endsWith(LayerArchiveSupport.LAYER_FILE_EXTENSION)) { + // First entry is empty or valid filename -> skip from parsing and use as filename + optionsStream = optionsStream.skip(1); + fileName = first; + } else { + // Assume first entry holds ExtendedOption value and use empty string as fileName + fileName = ""; + } + ExtendedOption[] extendedOptions = optionsStream.map(ExtendedOption::parse).toArray(ExtendedOption[]::new); + return new LayerOption(Path.of(fileName), extendedOptions); } } } From 18e87ecd20b38d469abc9967c48624ba7c927371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Wed, 14 May 2025 11:19:29 +0200 Subject: [PATCH 4/5] Consolidate LayerArchiveSupport and related classes * Make -H:TempDirectory available as early as possible * Ensure layerDir respects -H:TempDirectory like all other builder generated temp files * Do not expose layer-snapshot.lsb and layer-snapshot-graphs.big to BuildArtifacts * Remove LAYER_SNAPSHOT and LAYER_SNAPSHOT_GRAPHS artifact types * Clean up LayerArchiveSupport class hierarchy * Clean up HostedImageLayerBuildingSupport --- .../com/oracle/svm/core/BuildArtifacts.java | 2 - .../oracle/svm/core/util/ArchiveSupport.java | 9 +- .../com/oracle/svm/driver/BundleSupport.java | 2 +- .../svm/hosted/NativeImageGenerator.java | 35 +++--- .../TemporaryBuildDirectoryProviderImpl.java | 11 +- .../HostedImageLayerBuildingSupport.java | 104 ++++++------------ .../imagelayer/LayerArchiveSupport.java | 57 ++++++++-- .../imagelayer/LoadLayerArchiveSupport.java | 39 +------ .../imagelayer/SVMImageLayerSnapshotUtil.java | 14 +-- .../imagelayer/SVMImageLayerWriter.java | 61 ++++------ .../imagelayer/WriteLayerArchiveSupport.java | 42 +++---- .../com/oracle/svm/util/FileDumpingUtil.java | 72 ------------ 12 files changed, 159 insertions(+), 289 deletions(-) delete mode 100644 substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/FileDumpingUtil.java diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java index 21d90dd3e9e9..0d31744e4322 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/BuildArtifacts.java @@ -68,8 +68,6 @@ enum ArtifactType { BUILD_INFO("build_info"), /* For all debugging-related artifacts. */ DEBUG_INFO("debug_info"), - LAYER_SNAPSHOT("layer_snapshot"), - LAYER_SNAPSHOT_GRAPHS("layer_snapshot_graphs"), /* For C header files. */ C_HEADER("c_headers"), diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java index c402de7b7804..27a6b60522a8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/ArchiveSupport.java @@ -40,6 +40,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Predicate; import java.util.jar.Attributes; @@ -96,15 +97,15 @@ public Manifest createManifest(String mainClass) { return mf; } - public void expandJarToDir(Path inputJarFilePath, Path outputDir, AtomicBoolean outputDirDeleted) { - expandJarToDir(Function.identity(), inputJarFilePath, outputDir, outputDirDeleted); + public void expandJarToDir(Path inputJarFilePath, Path outputDir) { + expandJarToDir(Function.identity(), inputJarFilePath, outputDir, () -> false); } - public void expandJarToDir(Function relativizeEntry, Path inputJarFilePath, Path outputDir, AtomicBoolean outputDirDeleted) { + public void expandJarToDir(Function relativizeEntry, Path inputJarFilePath, Path outputDir, BooleanSupplier outputDirDeleted) { try { try (JarFile archive = new JarFile(inputJarFilePath.toFile())) { Enumeration jarEntries = archive.entries(); - while (jarEntries.hasMoreElements() && !outputDirDeleted.get()) { + while (jarEntries.hasMoreElements() && !outputDirDeleted.getAsBoolean()) { JarEntry jarEntry = jarEntries.nextElement(); Path originalEntry = outputDir.resolve(jarEntry.getName()); Path targetEntry = relativizeEntry.apply(originalEntry); diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java index 47d3a8d15d2f..4bfe8deb00e0 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/BundleSupport.java @@ -296,7 +296,7 @@ private BundleSupport(NativeImage nativeImage, String bundleFilenameArg) { outputDir = rootDir.resolve("output"); Path bundleFilePath = bundlePath.resolve(bundleName + BUNDLE_FILE_EXTENSION); - nativeImage.archiveSupport().expandJarToDir(e -> relativizeBundleEntry(getOriginalOutputDirName(), e), bundleFilePath, rootDir, deleteBundleRoot); + nativeImage.archiveSupport().expandJarToDir(e -> relativizeBundleEntry(getOriginalOutputDirName(), e), bundleFilePath, rootDir, deleteBundleRoot::get); if (deleteBundleRoot.get()) { /* Abort image build request without error message and exit with 0 */ diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java index 97335405c17d..ff0a69a96287 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java @@ -523,21 +523,23 @@ public void run(Map entryPoints, setSystemPropertiesForImageLate(k); var hostedOptionValues = new HostedOptionValues(optionProvider.getHostedValues()); - HostedImageLayerBuildingSupport imageLayerSupport = HostedImageLayerBuildingSupport.initialize(hostedOptionValues, loader); - ImageSingletonsSupportImpl.HostedManagement.install(new ImageSingletonsSupportImpl.HostedManagement(imageLayerSupport.buildingImageLayer), imageLayerSupport); - - ImageSingletons.add(LayeredImageSingletonSupport.class, (LayeredImageSingletonSupport) ImageSingletonsSupportImpl.get()); - ImageSingletons.add(ProgressReporter.class, reporter); - ImageSingletons.add(DeadlockWatchdog.class, loader.watchdog); - ImageSingletons.add(TimerCollection.class, timerCollection); - ImageSingletons.add(ImageBuildStatistics.TimerCollectionPrinter.class, timerCollection); - ImageSingletons.add(AnnotationExtractor.class, loader.classLoaderSupport.annotationExtractor); - ImageSingletons.add(BuildArtifacts.class, new BuildArtifactsImpl()); - ImageSingletons.add(HostedOptionValues.class, hostedOptionValues); - ImageSingletons.add(RuntimeOptionValues.class, new RuntimeOptionValues(optionProvider.getRuntimeValues(), allOptionNames)); - - try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl()) { + var tempDirectoryOptionValue = NativeImageOptions.TempDirectory.getValue(hostedOptionValues).lastValue().orElse(null); + try (TemporaryBuildDirectoryProviderImpl tempDirectoryProvider = new TemporaryBuildDirectoryProviderImpl(tempDirectoryOptionValue)) { + var builderTempDir = tempDirectoryProvider.getTemporaryBuildDirectory(); + HostedImageLayerBuildingSupport imageLayerSupport = HostedImageLayerBuildingSupport.initialize(hostedOptionValues, loader, builderTempDir); + ImageSingletonsSupportImpl.HostedManagement.install(new ImageSingletonsSupportImpl.HostedManagement(imageLayerSupport.buildingImageLayer), imageLayerSupport); + + ImageSingletons.add(LayeredImageSingletonSupport.class, (LayeredImageSingletonSupport) ImageSingletonsSupportImpl.get()); + ImageSingletons.add(ProgressReporter.class, reporter); + ImageSingletons.add(DeadlockWatchdog.class, loader.watchdog); + ImageSingletons.add(TimerCollection.class, timerCollection); + ImageSingletons.add(ImageBuildStatistics.TimerCollectionPrinter.class, timerCollection); + ImageSingletons.add(AnnotationExtractor.class, loader.classLoaderSupport.annotationExtractor); + ImageSingletons.add(BuildArtifacts.class, new BuildArtifactsImpl()); + ImageSingletons.add(HostedOptionValues.class, hostedOptionValues); + ImageSingletons.add(RuntimeOptionValues.class, new RuntimeOptionValues(optionProvider.getRuntimeValues(), allOptionNames)); ImageSingletons.add(TemporaryBuildDirectoryProvider.class, tempDirectoryProvider); + doRun(entryPoints, javaMainSupport, imageName, k, harnessSubstitutions); } finally { reporter.ensureCreationStageEndCompleted(); @@ -571,7 +573,7 @@ protected void doRun(Map entryPoints, JavaMainSupport j try (DebugContext debug = new Builder(options, new GraalDebugHandlersFactory(GraalAccess.getOriginalSnippetReflection())).build(); DebugCloseable featureCleanup = () -> featureHandler.forEachFeature(Feature::cleanup)) { - setupNativeImage(imageName, options, entryPoints, javaMainSupport, harnessSubstitutions, debug); + setupNativeImage(options, entryPoints, javaMainSupport, harnessSubstitutions, debug); boolean returnAfterAnalysis = runPointsToAnalysis(imageName, options, debug); if (returnAfterAnalysis) { @@ -917,7 +919,7 @@ protected boolean verifyAssignableTypes() { } @SuppressWarnings("try") - protected void setupNativeImage(String imageName, OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, + protected void setupNativeImage(OptionValues options, Map entryPoints, JavaMainSupport javaMainSupport, SubstitutionProcessor harnessSubstitutions, DebugContext debug) { try (Indent ignored = debug.logAndIndent("setup native-image builder")) { try (StopTimer ignored1 = TimerCollection.createTimerAndStart(TimerCollection.Registry.SETUP)) { @@ -996,7 +998,6 @@ protected void setupNativeImage(String imageName, OptionValues options, Map tempName = NativeImageOptions.TempDirectory.getValue().lastValue(); - if (tempName.isEmpty()) { + if (tempDirectoryOptionValue == null) { tempDirectory = Files.createTempDirectory("SVM-"); deleteTempDirectory = true; } else { - tempDirectory = tempName.get().resolve("SVM-" + TimeUtils.currentTimeMillis()); + tempDirectory = tempDirectoryOptionValue.resolve("SVM-" + TimeUtils.currentTimeMillis()); assert !Files.exists(tempDirectory); Files.createDirectories(tempDirectory); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index f9cb0337875d..2688e371dd4c 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -24,22 +24,16 @@ */ package com.oracle.svm.hosted.imagelayer; -import java.io.File; import java.io.IOException; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import com.oracle.svm.shaded.org.capnproto.ReaderOptions; -import com.oracle.svm.shaded.org.capnproto.Serialize; import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.ImageSingletons; import com.oracle.graal.pointsto.api.PointstoOptions; import com.oracle.graal.pointsto.util.AnalysisError; -import com.oracle.svm.core.BuildArtifacts; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport; import com.oracle.svm.core.option.HostedOptionValues; @@ -51,10 +45,13 @@ import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.ImageClassLoader; import com.oracle.svm.hosted.NativeImageClassLoaderSupport; -import com.oracle.svm.hosted.NativeImageGenerator; import com.oracle.svm.hosted.c.NativeLibraries; import com.oracle.svm.hosted.driver.IncludeOptionsSupport; import com.oracle.svm.hosted.driver.LayerOptionsSupport.LayerOption; +import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.SharedLayerSnapshot; +import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.SharedLayerSnapshot.Reader; +import com.oracle.svm.shaded.org.capnproto.ReaderOptions; +import com.oracle.svm.shaded.org.capnproto.Serialize; import com.oracle.svm.util.TypeResult; import jdk.graal.compiler.core.common.SuppressFBWarnings; @@ -71,23 +68,19 @@ private static String layerCreatePossibleOptions() { private SVMImageLayerWriter writer; private SVMImageLayerSingletonLoader singletonLoader; private final ImageClassLoader imageClassLoader; - private final List snapshots; - private final List graphsChannels; + private final SharedLayerSnapshot.Reader snapshot; + private final FileChannel graphsChannel; private final WriteLayerArchiveSupport writeLayerArchiveSupport; private final LoadLayerArchiveSupport loadLayerArchiveSupport; - public record FilePaths(Path snapshot, Path snapshotGraphs) { - } - - private HostedImageLayerBuildingSupport(SVMImageLayerSingletonLoader singletonLoader, ImageClassLoader imageClassLoader, - List snapshots, List graphsChannels, boolean buildingImageLayer, boolean buildingInitialLayer, - boolean buildingApplicationLayer, + private HostedImageLayerBuildingSupport(ImageClassLoader imageClassLoader, + Reader snapshot, FileChannel graphsChannel, + boolean buildingImageLayer, boolean buildingInitialLayer, boolean buildingApplicationLayer, WriteLayerArchiveSupport writeLayerArchiveSupport, LoadLayerArchiveSupport loadLayerArchiveSupport) { super(buildingImageLayer, buildingInitialLayer, buildingApplicationLayer); - this.singletonLoader = singletonLoader; this.imageClassLoader = imageClassLoader; - this.snapshots = snapshots; - this.graphsChannels = graphsChannels; + this.snapshot = snapshot; + this.graphsChannel = graphsChannel; this.writeLayerArchiveSupport = writeLayerArchiveSupport; this.loadLayerArchiveSupport = loadLayerArchiveSupport; } @@ -124,17 +117,21 @@ public LoadLayerArchiveSupport getLoadLayerArchiveSupport() { return loadLayerArchiveSupport; } + public WriteLayerArchiveSupport getWriteLayerArchiveSupport() { + return writeLayerArchiveSupport; + } + public void archiveLayer() { writer.dumpFiles(); writeLayerArchiveSupport.write(); } - public SharedLayerSnapshotCapnProtoSchemaHolder.SharedLayerSnapshot.Reader getSnapshot() { - return snapshots.get(0); + public SharedLayerSnapshot.Reader getSnapshot() { + return snapshot; } public FileChannel getGraphsChannel() { - return graphsChannels.get(0); + return graphsChannel; } public Class lookupClass(boolean optional, String className) { @@ -247,7 +244,7 @@ private static boolean isLayerUseOptionEnabled(OptionValues values) { return false; } - public static HostedImageLayerBuildingSupport initialize(HostedOptionValues values, ImageClassLoader imageClassLoader) { + public static HostedImageLayerBuildingSupport initialize(HostedOptionValues values, ImageClassLoader imageClassLoader, Path builderTempDir) { boolean buildingSharedLayer = isLayerCreateOptionEnabled(values); boolean buildingExtensionLayer = isLayerUseOptionEnabled(values); @@ -263,42 +260,33 @@ public static HostedImageLayerBuildingSupport initialize(HostedOptionValues valu ArchiveSupport archiveSupport = new ArchiveSupport(false); String layerName = SubstrateOptions.Name.getValue(values); if (buildingSharedLayer) { - writeLayerArchiveSupport = new WriteLayerArchiveSupport(layerName, archiveSupport, imageClassLoader.classLoaderSupport.getLayerFile()); + writeLayerArchiveSupport = new WriteLayerArchiveSupport(layerName, imageClassLoader.classLoaderSupport.getLayerFile(), builderTempDir, archiveSupport); } - SVMImageLayerSingletonLoader singletonLoader = null; LoadLayerArchiveSupport loadLayerArchiveSupport = null; - List snapshots = null; - List graphsChannels = null; + SharedLayerSnapshot.Reader snapshot = null; + FileChannel graphsChannel = null; if (buildingExtensionLayer) { Path layerFileName = getLayerUseValue(values); - loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerName, layerFileName, archiveSupport); - FilePaths filePaths = new FilePaths(loadLayerArchiveSupport.getSnapshotPath(), loadLayerArchiveSupport.getSnapshotGraphsPath()); - List loadPaths = List.of(filePaths); - snapshots = new ArrayList<>(); - graphsChannels = new ArrayList<>(); - for (FilePaths paths : loadPaths) { - try { - graphsChannels.add(FileChannel.open(paths.snapshotGraphs)); - - try (FileChannel ch = FileChannel.open(paths.snapshot)) { - MappedByteBuffer bb = ch.map(FileChannel.MapMode.READ_ONLY, ch.position(), ch.size()); - ReaderOptions opt = new ReaderOptions(Long.MAX_VALUE, ReaderOptions.DEFAULT_READER_OPTIONS.nestingLimit); - snapshots.add(Serialize.read(bb, opt).getRoot(SharedLayerSnapshotCapnProtoSchemaHolder.SharedLayerSnapshot.factory)); - // NOTE: buffer is never unmapped, but is read-only and pages can be evicted - } - } catch (IOException e) { - throw AnalysisError.shouldNotReachHere("Error during image layer snapshot loading", e); + loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerName, layerFileName, builderTempDir, archiveSupport); + try { + graphsChannel = FileChannel.open(loadLayerArchiveSupport.getSnapshotGraphsPath()); + + try (FileChannel ch = FileChannel.open(loadLayerArchiveSupport.getSnapshotPath())) { + MappedByteBuffer bb = ch.map(FileChannel.MapMode.READ_ONLY, ch.position(), ch.size()); + ReaderOptions opt = new ReaderOptions(Long.MAX_VALUE, ReaderOptions.DEFAULT_READER_OPTIONS.nestingLimit); + snapshot = Serialize.read(bb, opt).getRoot(SharedLayerSnapshot.factory); + // NOTE: buffer is never unmapped, but is read-only and pages can be evicted } + } catch (IOException e) { + throw AnalysisError.shouldNotReachHere("Error during image layer snapshot loading " + loadLayerArchiveSupport.getSnapshotGraphsPath(), e); } - - assert loadPaths.size() == 1 : "Currently only one path is supported for image layer loading " + loadPaths; } - HostedImageLayerBuildingSupport imageLayerBuildingSupport = new HostedImageLayerBuildingSupport(singletonLoader, imageClassLoader, snapshots, graphsChannels, buildingImageLayer, + HostedImageLayerBuildingSupport imageLayerBuildingSupport = new HostedImageLayerBuildingSupport(imageClassLoader, snapshot, graphsChannel, buildingImageLayer, buildingInitialLayer, buildingFinalLayer, writeLayerArchiveSupport, loadLayerArchiveSupport); if (buildingExtensionLayer) { - imageLayerBuildingSupport.setSingletonLoader(new SVMImageLayerSingletonLoader(imageLayerBuildingSupport, snapshots.get(0))); + imageLayerBuildingSupport.setSingletonLoader(new SVMImageLayerSingletonLoader(imageLayerBuildingSupport, snapshot)); } return imageLayerBuildingSupport; @@ -318,26 +306,4 @@ public static void setupSharedLayerLibrary(NativeLibraries nativeLibs) { HostedDynamicLayerInfo.singleton().registerLibName(libName); nativeLibs.addDynamicNonJniLibrary(libName); } - - public static void setupImageLayerArtifacts(String imageName) { - VMError.guarantee(!imageName.contains(File.separator), "Expected simple file name, found %s.", imageName); - - Path snapshotFile = NativeImageGenerator.getOutputDirectory().resolve(SVMImageLayerSnapshotUtil.snapshotFileName(imageName)); - Path snapshotFileName = getFileName(snapshotFile); - HostedImageLayerBuildingSupport.singleton().getWriter().setSnapshotFileInfo(snapshotFile, snapshotFileName.toString(), SVMImageLayerSnapshotUtil.FILE_EXTENSION); - BuildArtifacts.singleton().add(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT, snapshotFile); - - Path graphsFile = NativeImageGenerator.getOutputDirectory().resolve(SVMImageLayerSnapshotUtil.snapshotGraphsFileName(imageName)); - Path graphsFileName = getFileName(graphsFile); - HostedImageLayerBuildingSupport.singleton().getWriter().openGraphsOutput(graphsFile, graphsFileName.toString(), SVMImageLayerSnapshotUtil.FILE_EXTENSION); - BuildArtifacts.singleton().add(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT_GRAPHS, graphsFile); - } - - private static Path getFileName(Path path) { - Path fileName = path.getFileName(); - if (fileName == null) { - throw VMError.shouldNotReachHere("Layer snapshot file(s) missing."); - } - return fileName; - } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index 6b5ed3ab1861..b4611a390d98 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -39,7 +39,6 @@ import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.NativeImageGenerator; import com.oracle.svm.util.LogUtils; public class LayerArchiveSupport { @@ -48,19 +47,61 @@ public class LayerArchiveSupport { private static final int LAYER_FILE_FORMAT_VERSION_MINOR = 1; protected static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; - protected static final String LAYER_TEMP_DIR_PREFIX = "layerRoot-"; + protected static final String LAYER_TEMP_DIR_PREFIX = "_layerRoot_"; public static final String LAYER_FILE_EXTENSION = ".nil"; protected final LayerProperties layerProperties; + protected final Path layerFile; protected final ArchiveSupport archiveSupport; - public LayerArchiveSupport(String layerName, ArchiveSupport archiveSupport) { + /** The temp directory where the layer files reside in expanded form. */ + protected final Path layerDir; + + public LayerArchiveSupport(String layerName, Path layerFile, Path layerDir, ArchiveSupport archiveSupport) { this.archiveSupport = archiveSupport; + + validateLayerFile(layerFile); + this.layerFile = layerFile; + + this.layerDir = layerDir; + try { + Files.createDirectory(layerDir); + } catch (IOException e) { + throw UserError.abort("Unable to create temp directory " + layerDir + " where the layer files reside in expanded form.", e); + } + this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); } - protected static final Path layerPropertiesFileName = Path.of("META-INF/nilayer.properties"); + protected void validateLayerFile(Path layerFile) { + Path fileName = layerFile.getFileName(); + if (fileName == null || !fileName.toString().endsWith(LAYER_FILE_EXTENSION)) { + throw UserError.abort("The given layer file " + layerFile + " must end with '" + LAYER_FILE_EXTENSION + "'."); + } + + if (Files.isDirectory(layerFile)) { + throw UserError.abort("The given layer file " + layerFile + " is a directory and not a file."); + } + } + + public Path getSnapshotPath() { + return layerDir.resolve(SVMImageLayerSnapshotUtil.SNAPSHOT_FILE_NAME); + } + + public Path getSnapshotGraphsPath() { + return layerDir.resolve(SVMImageLayerSnapshotUtil.SNAPSHOT_GRAPHS_FILE_NAME); + } + + public Path getSharedLibraryPath() { + return layerDir.resolve(layerProperties.layerName() + ".so"); + } + + private static final Path layerPropertiesFileName = Path.of("META-INF/nilayer.properties"); + + protected Path getLayerPropertiesFile() { + return layerDir.resolve(layerPropertiesFileName); + } public final class LayerProperties { @@ -109,9 +150,9 @@ public String toString() { } } - void loadAndVerify(Path inputLayerLocation, Path expandedInputLayerDir) { - Path layerFileName = inputLayerLocation.getFileName(); - Path layerPropertiesFile = expandedInputLayerDir.resolve(layerPropertiesFileName); + void loadAndVerify() { + Path layerFileName = layerFile.getFileName(); + Path layerPropertiesFile = getLayerPropertiesFile(); if (!Files.isReadable(layerPropertiesFile)) { throw UserError.abort("The given layer file " + layerFileName + " does not contain a layer properties file"); @@ -166,7 +207,7 @@ void write() { properties.put(PROPERTY_KEY_LAYER_FILE_CREATION_TIMESTAMP, ArchiveSupport.currentTime()); properties.put(PROPERTY_KEY_LAYER_BUILDER_VM_PLATFORM, platform()); BuilderVMIdentifier.system().store(properties); - Path layerPropertiesFile = NativeImageGenerator.getOutputDirectory().resolve(layerPropertiesFileName); + Path layerPropertiesFile = getLayerPropertiesFile(); Path parent = layerPropertiesFile.getParent(); if (parent == null) { throw VMError.shouldNotReachHere("The layer properties file " + layerPropertiesFile + " doesn't have a parent directory."); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 1f8978f00c37..3877af5fd181 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -26,50 +26,23 @@ import java.nio.file.Files; import java.nio.file.Path; -import java.util.concurrent.atomic.AtomicBoolean; import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; public class LoadLayerArchiveSupport extends LayerArchiveSupport { - /** The temp directory where the input layer is expanded. */ - private final Path expandedInputLayerDir; - - private final AtomicBoolean deleteLayerRoot = new AtomicBoolean(); - - public LoadLayerArchiveSupport(String layerName, Path layerFile, ArchiveSupport archiveSupport) { - super(layerName, archiveSupport); - Path inputLayerLocation = validateLayerFile(layerFile); - expandedInputLayerDir = this.archiveSupport.createTempDir(LAYER_TEMP_DIR_PREFIX, deleteLayerRoot); - this.archiveSupport.expandJarToDir(inputLayerLocation, expandedInputLayerDir, deleteLayerRoot); - layerProperties.loadAndVerify(inputLayerLocation, expandedInputLayerDir); - } - - public Path getSharedLibraryPath() { - return expandedInputLayerDir.resolve(layerProperties.layerName() + ".so"); - } - - public Path getSnapshotPath() { - return expandedInputLayerDir.resolve(SVMImageLayerSnapshotUtil.snapshotFileName(layerProperties.layerName())); + public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, ArchiveSupport archiveSupport) { + super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "load"), archiveSupport); + this.archiveSupport.expandJarToDir(layerFile, layerDir); + layerProperties.loadAndVerify(); } - public Path getSnapshotGraphsPath() { - return expandedInputLayerDir.resolve(SVMImageLayerSnapshotUtil.snapshotGraphsFileName(layerProperties.layerName())); - } + protected void validateLayerFile(Path layerFile) { + super.validateLayerFile(layerFile); - private static Path validateLayerFile(Path layerFile) { - Path fileName = layerFile.getFileName(); - if (fileName == null || !fileName.toString().endsWith(LAYER_FILE_EXTENSION)) { - throw UserError.abort("The given layer file " + layerFile + " must end with '" + LAYER_FILE_EXTENSION + "'."); - } - if (Files.isDirectory(layerFile)) { - throw UserError.abort("The given layer file " + layerFile + " is a directory and not a file."); - } if (!Files.isReadable(layerFile)) { throw UserError.abort("The given layer file " + layerFile + " cannot be read."); } - return layerFile; } - } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java index dc8f59d4705f..0df33f3756d6 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java @@ -92,10 +92,8 @@ import jdk.vm.ci.meta.ConstantReflectionProvider; public class SVMImageLayerSnapshotUtil { - public static final String FILE_NAME_PREFIX = "layer-snapshot-"; - public static final String FILE_EXTENSION = ".lsb"; - public static final String GRAPHS_FILE_NAME_PREFIX = "layer-snapshot-graphs-"; - public static final String GRAPHS_FILE_EXTENSION = ".big"; + public static final String SNAPSHOT_FILE_NAME = "layer-snapshot.lsb"; + public static final String SNAPSHOT_GRAPHS_FILE_NAME = "layer-snapshot-graphs.big"; public static final String CONSTRUCTOR_NAME = ""; public static final String CLASS_INIT_NAME = ""; @@ -257,14 +255,6 @@ public void initializeExternalValues() { externalValues = ObjectCopier.Encoder.gatherExternalValues(externalValueFields); } - public static String snapshotFileName(String imageName) { - return FILE_NAME_PREFIX + imageName + FILE_EXTENSION; - } - - public static String snapshotGraphsFileName(String imageName) { - return GRAPHS_FILE_NAME_PREFIX + imageName + GRAPHS_FILE_EXTENSION; - } - public String getTypeDescriptor(AnalysisType type) { String javaName = type.toJavaName(true); if (javaName.contains(GENERATED_SERIALIZATION)) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerWriter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerWriter.java index 2bd43a698b37..d50323134453 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerWriter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerWriter.java @@ -30,6 +30,7 @@ import static com.oracle.svm.hosted.imagelayer.SVMImageLayerSnapshotUtil.UNDEFINED_FIELD_INDEX; import static com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.ClassInitializationInfo.Builder; +import java.io.FileOutputStream; import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.AnnotatedElement; @@ -42,6 +43,7 @@ import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; @@ -67,8 +69,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; -import jdk.graal.compiler.graph.NodeClass; -import jdk.graal.compiler.nodes.GraphEncoder; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.MapCursor; import org.graalvm.nativeimage.AnnotationAccess; @@ -162,15 +162,16 @@ import com.oracle.svm.shaded.org.capnproto.Text; import com.oracle.svm.shaded.org.capnproto.TextList; import com.oracle.svm.shaded.org.capnproto.Void; -import com.oracle.svm.util.FileDumpingUtil; import com.oracle.svm.util.LogUtils; import com.oracle.svm.util.ModuleSupport; import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.debug.Assertions; import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; import jdk.graal.compiler.java.LambdaUtils; import jdk.graal.compiler.nodes.EncodedGraph; +import jdk.graal.compiler.nodes.GraphEncoder; import jdk.graal.compiler.nodes.NodeClassMap; import jdk.graal.compiler.nodes.spi.IdentityHashCodeProvider; import jdk.graal.compiler.util.ObjectCopier; @@ -194,8 +195,7 @@ public class SVMImageLayerWriter extends ImageLayerWriter { private final Map methodsMap = new ConcurrentHashMap<>(); private final Map initialLayerOnlySingletonMap = new ConcurrentHashMap<>(); private final Map> polymorphicSignatureCallers = new ConcurrentHashMap<>(); - private FileInfo fileInfo; - private GraphsOutput graphsOutput; + private final GraphsOutput graphsOutput; private final boolean useSharedLayerGraphs; private final boolean useSharedLayerStrengthenedGraphs; @@ -213,9 +213,6 @@ private record ConstantParent(int constantId, int index) { static ConstantParent NONE = new ConstantParent(UNDEFINED_CONSTANT_ID, UNDEFINED_FIELD_INDEX); } - private record FileInfo(Path layerFilePath, String fileName, String suffix) { - } - private record MethodGraphsInfo(String analysisGraphLocation, boolean analysisGraphIsIntrinsic, String strengthenedGraphLocation) { @@ -233,26 +230,24 @@ MethodGraphsInfo withStrengthenedGraph(String location) { } private static class GraphsOutput { - private final Path path; - private final Path tempPath; - private final FileChannel tempChannel; + private final FileChannel channel; private final AtomicLong currentOffset = new AtomicLong(0); - GraphsOutput(Path path, String fileName, String suffix) { - this.path = path; - this.tempPath = FileDumpingUtil.createTempFile(path.getParent(), fileName, suffix); + GraphsOutput() { + Path snapshotGraphsPath = HostedImageLayerBuildingSupport.singleton().getWriteLayerArchiveSupport().getSnapshotGraphsPath(); try { - this.tempChannel = FileChannel.open(this.tempPath, EnumSet.of(StandardOpenOption.WRITE)); + Files.createFile(snapshotGraphsPath); + channel = FileChannel.open(snapshotGraphsPath, EnumSet.of(StandardOpenOption.WRITE)); } catch (IOException e) { - throw GraalError.shouldNotReachHere(e, "Error opening temporary graphs file."); + throw VMError.shouldNotReachHere("Error opening temporary graphs file " + snapshotGraphsPath, e); } } String add(byte[] encodedGraph) { long offset = currentOffset.getAndAdd(encodedGraph.length); try { - tempChannel.write(ByteBuffer.wrap(encodedGraph), offset); + channel.write(ByteBuffer.wrap(encodedGraph), offset); } catch (Exception e) { throw GraalError.shouldNotReachHere(e, "Error during graphs file dumping."); } @@ -261,10 +256,9 @@ String add(byte[] encodedGraph) { void finish() { try { - tempChannel.close(); - FileDumpingUtil.moveTryAtomically(tempPath, path); + channel.close(); } catch (Exception e) { - throw GraalError.shouldNotReachHere(e, "Error during graphs file dumping."); + throw VMError.shouldNotReachHere("Error during graphs file dumping.", e); } } } @@ -273,6 +267,7 @@ public SVMImageLayerWriter(SVMImageLayerSnapshotUtil imageLayerSnapshotUtil, boo this.imageLayerSnapshotUtil = imageLayerSnapshotUtil; this.useSharedLayerGraphs = useSharedLayerGraphs; this.useSharedLayerStrengthenedGraphs = useSharedLayerStrengthenedGraphs; + graphsOutput = new GraphsOutput(); } public void setInternedStringsIdentityMap(IdentityHashMap map) { @@ -283,10 +278,6 @@ public void setImageHeap(ImageHeap heap) { this.imageHeap = heap; } - public void setSnapshotFileInfo(Path layerSnapshotPath, String fileName, String suffix) { - fileInfo = new FileInfo(layerSnapshotPath, fileName, suffix); - } - public void setAnalysisUniverse(AnalysisUniverse aUniverse) { this.aUniverse = aUniverse; } @@ -299,26 +290,19 @@ public void setHostedUniverse(HostedUniverse hUniverse) { this.hUniverse = hUniverse; } - public void openGraphsOutput(Path layerGraphsPath, String fileName, String suffix) { - AnalysisError.guarantee(graphsOutput == null, "Graphs file has already been opened"); - graphsOutput = new GraphsOutput(layerGraphsPath, fileName, suffix); - } - public void dumpFiles() { SVMImageLayerSnapshotUtil.SVMGraphEncoder graphEncoder = imageLayerSnapshotUtil.getGraphEncoder(null); byte[] encodedNodeClassMap = ObjectCopier.encode(graphEncoder, nodeClassMap); String location = graphsOutput.add(encodedNodeClassMap); snapshotBuilder.setNodeClassMapLocation(location); - graphsOutput.finish(); - FileDumpingUtil.dumpFile(fileInfo.layerFilePath, fileInfo.fileName, fileInfo.suffix, outputStream -> { - try { - Serialize.write(Channels.newChannel(outputStream), snapshotFileBuilder); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + Path snapshotFile = HostedImageLayerBuildingSupport.singleton().getWriteLayerArchiveSupport().getSnapshotPath(); + try (FileOutputStream outputStream = new FileOutputStream(snapshotFile.toFile())) { + Serialize.write(Channels.newChannel(outputStream), snapshotFileBuilder); + } catch (IOException e) { + throw VMError.shouldNotReachHere("Unable to write " + snapshotFile, e); + } } public void initializeExternalValues() { @@ -830,9 +814,8 @@ private void persistConstant(ImageHeapConstant imageHeapConstant, ConstantParent } case ImageHeapPrimitiveArray imageHeapPrimitiveArray -> persistConstantPrimitiveArray(builder.initPrimitiveData(), imageHeapPrimitiveArray.getType().getComponentType().getJavaKind(), imageHeapPrimitiveArray.getArray()); - case ImageHeapRelocatableConstant relocatableConstant -> { + case ImageHeapRelocatableConstant relocatableConstant -> builder.initRelocatable().setKey(relocatableConstant.getConstantData().key); - } default -> throw AnalysisError.shouldNotReachHere("Unexpected constant type " + imageHeapConstant); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 549263cec067..afc80a190fa4 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -38,23 +38,14 @@ /* Builds an image layer, either initial or intermediate. */ public class WriteLayerArchiveSupport extends LayerArchiveSupport { - /** The original location of the layer output file. */ - private final Path outputLayerLocation; - - public WriteLayerArchiveSupport(String layerName, ArchiveSupport archiveSupport, Path layerFile) { - super(layerName, archiveSupport); - this.outputLayerLocation = validateLayerFile(layerFile); + public WriteLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, ArchiveSupport archiveSupport) { + super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "write"), archiveSupport); } - private static Path validateLayerFile(Path layerFile) { - Path fileName = layerFile.getFileName(); - if (fileName == null || !fileName.toString().endsWith(LAYER_FILE_EXTENSION)) { - throw UserError.abort("The given layer file " + layerFile + " must end with '" + LAYER_FILE_EXTENSION + "'."); - } + protected void validateLayerFile(Path layerFile) { + super.validateLayerFile(layerFile); + Path layerFilePath = layerFile.toAbsolutePath(); - if (Files.isDirectory(layerFilePath)) { - throw UserError.abort("The given layer file " + layerFile + " is a directory and not a file."); - } Path layerParentPath = layerFilePath.getParent(); if (layerParentPath == null) { throw UserError.abort("The given layer file " + layerFile + " doesn't have a parent directory."); @@ -65,31 +56,26 @@ private static Path validateLayerFile(Path layerFile) { if (Files.exists(layerFilePath) && !Files.isWritable(layerFilePath)) { throw UserError.abort("The given layer file " + layerFile + " is not writeable."); } - return layerFile; } public void write() { layerProperties.write(); - try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(outputLayerLocation), archiveSupport.createManifest())) { - Path imageBuilderOutputDir = NativeImageGenerator.getOutputDirectory(); + try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(layerFile), archiveSupport.createManifest())) { // disable compression for significant (un)archiving speedup at the cost of file size jarOutStream.setLevel(0); // copy the layer snapshot file and its graphs file to the jar - Path snapshotFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT).getFirst(); - archiveSupport.addFileToJar(imageBuilderOutputDir, snapshotFile, outputLayerLocation, jarOutStream); - Path snapshotGraphsFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.LAYER_SNAPSHOT_GRAPHS).getFirst(); - archiveSupport.addFileToJar(imageBuilderOutputDir, snapshotGraphsFile, outputLayerLocation, jarOutStream); + archiveSupport.addFileToJar(layerDir, getSnapshotPath(), layerFile, jarOutStream); + archiveSupport.addFileToJar(layerDir, getSnapshotGraphsPath(), layerFile, jarOutStream); // copy the shared object file to the jar Path sharedLibFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.IMAGE_LAYER).getFirst(); - archiveSupport.addFileToJar(imageBuilderOutputDir, sharedLibFile, outputLayerLocation, jarOutStream); + archiveSupport.addFileToJar(NativeImageGenerator.getOutputDirectory(), sharedLibFile, layerFile, jarOutStream); // copy the properties file to the jar - Path propertiesFile = imageBuilderOutputDir.resolve(layerPropertiesFileName); - archiveSupport.addFileToJar(imageBuilderOutputDir, propertiesFile, outputLayerLocation, jarOutStream); - BuildArtifacts.singleton().add(ArtifactType.IMAGE_LAYER_BUNDLE, outputLayerLocation); + layerProperties.write(); + archiveSupport.addFileToJar(layerDir, getLayerPropertiesFile(), layerFile, jarOutStream); + BuildArtifacts.singleton().add(ArtifactType.IMAGE_LAYER_BUNDLE, layerFile); } catch (IOException e) { - throw UserError.abort("Failed to create Native Image Layer file " + outputLayerLocation.getFileName(), e); + throw UserError.abort("Failed to create Native Image Layer file " + layerFile.getFileName(), e); } - info("Layer written to %s", outputLayerLocation); + info("Layer written to %s", layerFile); } - } diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/FileDumpingUtil.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/FileDumpingUtil.java deleted file mode 100644 index e7194e4ec78f..000000000000 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/FileDumpingUtil.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.svm.util; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.AtomicMoveNotSupportedException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.function.Consumer; - -import jdk.graal.compiler.core.common.SuppressFBWarnings; -import jdk.graal.compiler.debug.GraalError; - -public class FileDumpingUtil { - public static Path createTempFile(Path directory, String name, String suffix) { - try { - long start = System.nanoTime(); - String filePrefix = String.format(name + "-%d", start); - return Files.createTempFile(directory, filePrefix, suffix); - } catch (Exception e) { - throw GraalError.shouldNotReachHere(e, "Error during temporary file creation."); - } - } - - @SuppressFBWarnings(value = "", justification = "FB reports null pointer dereferencing although it is not possible in this case.") - public static void dumpFile(Path path, String name, String suffix, Consumer streamConsumer) { - Path tempPath = createTempFile(path.getParent(), name, suffix); - try { - try (FileOutputStream fileOutputStream = new FileOutputStream(tempPath.toFile())) { - streamConsumer.accept(fileOutputStream); - } - moveTryAtomically(tempPath, path); - } catch (Exception e) { - throw GraalError.shouldNotReachHere(e, "Error during file dumping."); - } - } - - public static void moveTryAtomically(Path source, Path target) throws IOException { - try { - Files.move(source, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } catch (AtomicMoveNotSupportedException e) { - LogUtils.warning("Could not move temporary file (" + source.toAbsolutePath() + ") to (" + target.toAbsolutePath() + ") atomically. " + - "This might result in inconsistencies while reading the file."); - Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); - } - } -} From e426d60273cd4868fcd25718ba0ae8bf1d27659b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Thu, 15 May 2025 20:40:05 +0200 Subject: [PATCH 5/5] Fist draft of builder arguments compatibility checking --- .../HostedImageLayerBuildingSupport.java | 3 +- .../imagelayer/LayerArchiveSupport.java | 20 +++- .../imagelayer/LoadLayerArchiveSupport.java | 73 ++++++++++++ .../imagelayer/SVMImageLayerSnapshotUtil.java | 2 - .../imagelayer/WriteLayerArchiveSupport.java | 20 +++- .../com/oracle/svm/hosted/util/DiffTool.java | 105 ++++++++++++++++++ 6 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/DiffTool.java diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 2688e371dd4c..78e393ee65b0 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -260,7 +260,7 @@ public static HostedImageLayerBuildingSupport initialize(HostedOptionValues valu ArchiveSupport archiveSupport = new ArchiveSupport(false); String layerName = SubstrateOptions.Name.getValue(values); if (buildingSharedLayer) { - writeLayerArchiveSupport = new WriteLayerArchiveSupport(layerName, imageClassLoader.classLoaderSupport.getLayerFile(), builderTempDir, archiveSupport); + writeLayerArchiveSupport = new WriteLayerArchiveSupport(layerName, imageClassLoader.classLoaderSupport, builderTempDir, archiveSupport); } LoadLayerArchiveSupport loadLayerArchiveSupport = null; SharedLayerSnapshot.Reader snapshot = null; @@ -268,6 +268,7 @@ public static HostedImageLayerBuildingSupport initialize(HostedOptionValues valu if (buildingExtensionLayer) { Path layerFileName = getLayerUseValue(values); loadLayerArchiveSupport = new LoadLayerArchiveSupport(layerName, layerFileName, builderTempDir, archiveSupport); + loadLayerArchiveSupport.verifyCompatibility(imageClassLoader.classLoaderSupport); try { graphsChannel = FileChannel.open(loadLayerArchiveSupport.getSnapshotGraphsPath()); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index b4611a390d98..99d761c02d04 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -28,7 +28,9 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -46,11 +48,16 @@ public class LayerArchiveSupport { private static final int LAYER_FILE_FORMAT_VERSION_MAJOR = 0; private static final int LAYER_FILE_FORMAT_VERSION_MINOR = 1; - protected static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; - protected static final String LAYER_TEMP_DIR_PREFIX = "_layerRoot_"; + private static final String BUILDER_ARGUMENTS_FILE_NAME = "builder-arguments.txt"; + private static final String SNAPSHOT_FILE_NAME = "layer-snapshot.lsb"; + private static final String SNAPSHOT_GRAPHS_FILE_NAME = "layer-snapshot-graphs.big"; + private static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; + protected static final String LAYER_TEMP_DIR_PREFIX = "layerRoot_"; public static final String LAYER_FILE_EXTENSION = ".nil"; + protected final List builderArguments; + protected final LayerProperties layerProperties; protected final Path layerFile; protected final ArchiveSupport archiveSupport; @@ -72,6 +79,7 @@ public LayerArchiveSupport(String layerName, Path layerFile, Path layerDir, Arch } this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); + this.builderArguments = new ArrayList<>(); } protected void validateLayerFile(Path layerFile) { @@ -86,11 +94,11 @@ protected void validateLayerFile(Path layerFile) { } public Path getSnapshotPath() { - return layerDir.resolve(SVMImageLayerSnapshotUtil.SNAPSHOT_FILE_NAME); + return layerDir.resolve(SNAPSHOT_FILE_NAME); } public Path getSnapshotGraphsPath() { - return layerDir.resolve(SVMImageLayerSnapshotUtil.SNAPSHOT_GRAPHS_FILE_NAME); + return layerDir.resolve(SNAPSHOT_GRAPHS_FILE_NAME); } public Path getSharedLibraryPath() { @@ -103,6 +111,10 @@ protected Path getLayerPropertiesFile() { return layerDir.resolve(layerPropertiesFileName); } + protected Path getBuilderArgumentsFilePath() { + return layerDir.resolve(BUILDER_ARGUMENTS_FILE_NAME); + } + public final class LayerProperties { private static final String PROPERTY_KEY_LAYER_FILE_VERSION_MAJOR = "LayerFileVersionMajor"; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 3877af5fd181..3131aa7b73fe 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -24,11 +24,19 @@ */ package com.oracle.svm.hosted.imagelayer; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; +import com.oracle.svm.core.option.OptionOrigin; import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.hosted.NativeImageClassLoaderSupport; +import com.oracle.svm.hosted.util.DiffTool; +import com.oracle.svm.hosted.util.DiffTool.DiffResult.Kind; +import com.oracle.svm.util.LogUtils; public class LoadLayerArchiveSupport extends LayerArchiveSupport { @@ -36,6 +44,15 @@ public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, A super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "load"), archiveSupport); this.archiveSupport.expandJarToDir(layerFile, layerDir); layerProperties.loadAndVerify(); + loadBuilderArgumentsFile(); + } + + private void loadBuilderArgumentsFile() { + try (Stream lines = Files.lines(getBuilderArgumentsFilePath())) { + lines.forEach(builderArguments::add); + } catch (IOException e) { + throw UserError.abort("Unable to load builder arguments from file " + getBuilderArgumentsFilePath()); + } } protected void validateLayerFile(Path layerFile) { @@ -45,4 +62,60 @@ protected void validateLayerFile(Path layerFile) { throw UserError.abort("The given layer file " + layerFile + " cannot be read."); } } + + public void verifyCompatibility(NativeImageClassLoaderSupport classLoaderSupport) { + // var errorMessagePrefix = "Layer Compatibility Error: "; + var strippedBuilderArguments = builderArguments.stream() + .map(argument -> splitArgumentOrigin(argument).argument) + .toList(); + List currentBuilderArguments = classLoaderSupport.getHostedOptionParser().getArguments(); + var strippedCurrentBuilderArguments = currentBuilderArguments.stream() + .map(argument -> splitArgumentOrigin(argument).argument) + .toList(); + + var diffResults = DiffTool.diffResults(strippedBuilderArguments, strippedCurrentBuilderArguments); + for (var diffResult : diffResults) { + if (diffResult.kind() == Kind.Removed) { + ArgumentOrigin argumentOrigin = splitArgumentOrigin(diffResult.getEntry(builderArguments, currentBuilderArguments)); + OptionOrigin origin = OptionOrigin.from(argumentOrigin.origin); + LogUtils.warning("The used parent layer '" + layerProperties.layerName() + "' specified via layer-file '" + layerFile.getFileName() + "'" + + " was built with option argument '" + argumentOrigin.argument + "' from " + origin + "." + + " The same option argument is required to also be used at the same place for this layered image build."); + } + } + + System.out.println("============================================================"); + for (var diffResult : diffResults) { + System.out.println(diffResult.toString(builderArguments, currentBuilderArguments)); + } + System.out.println("============================================================"); + } + + record ArgumentOrigin(String argument, String origin) { + } + + private static ArgumentOrigin splitArgumentOrigin(String argumentWithOrigin) { + int booleanPrefixPos = argumentWithOrigin.indexOf(':') + 1; + char booleanPrefixChar = argumentWithOrigin.charAt(booleanPrefixPos); + boolean booleanOption = booleanPrefixChar == '+' || booleanPrefixChar == '-'; + String argument = argumentWithOrigin; + String origin = ""; + if (booleanOption) { + // e.g. -R:+Bar@originInfoB -> -R:+Bar, originInfoB + int originSeperatorPos = argumentWithOrigin.lastIndexOf('@'); + if (originSeperatorPos >= 0) { + argument = argumentWithOrigin.substring(0, originSeperatorPos); + origin = argumentWithOrigin.substring(originSeperatorPos + 1); + } + } else { + // e.g. -H:Foo@originInfoA=value1 -> -H:Foo=value1, originInfoA + int originSeparatorPos = argumentWithOrigin.indexOf('@'); + if (originSeparatorPos >= 0) { + int keyValueSeparatorPos = argumentWithOrigin.indexOf('='); + argument = argumentWithOrigin.substring(0, originSeparatorPos) + argumentWithOrigin.substring(keyValueSeparatorPos); + origin = argumentWithOrigin.substring(originSeparatorPos + 1, keyValueSeparatorPos); + } + } + return new ArgumentOrigin(argument, origin); + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java index 0df33f3756d6..2b443ec95594 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerSnapshotUtil.java @@ -92,8 +92,6 @@ import jdk.vm.ci.meta.ConstantReflectionProvider; public class SVMImageLayerSnapshotUtil { - public static final String SNAPSHOT_FILE_NAME = "layer-snapshot.lsb"; - public static final String SNAPSHOT_GRAPHS_FILE_NAME = "layer-snapshot-graphs.big"; public static final String CONSTRUCTOR_NAME = ""; public static final String CLASS_INIT_NAME = ""; diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index afc80a190fa4..2991e4dd0108 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -33,13 +33,15 @@ import com.oracle.svm.core.BuildArtifacts.ArtifactType; import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.hosted.NativeImageClassLoaderSupport; import com.oracle.svm.hosted.NativeImageGenerator; /* Builds an image layer, either initial or intermediate. */ public class WriteLayerArchiveSupport extends LayerArchiveSupport { - public WriteLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, ArchiveSupport archiveSupport) { - super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "write"), archiveSupport); + public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport classLoaderSupport, Path tempDir, ArchiveSupport archiveSupport) { + super(layerName, classLoaderSupport.getLayerFile(), tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "write"), archiveSupport); + builderArguments.addAll(classLoaderSupport.getHostedOptionParser().getArguments()); } protected void validateLayerFile(Path layerFile) { @@ -58,18 +60,28 @@ protected void validateLayerFile(Path layerFile) { } } + private void writeBuilderArgumentsFile() { + try { + Files.write(getBuilderArgumentsFilePath(), builderArguments); + } catch (IOException e) { + throw UserError.abort("Unable to write builder arguments to file " + getBuilderArgumentsFilePath()); + } + } + public void write() { - layerProperties.write(); try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(layerFile), archiveSupport.createManifest())) { // disable compression for significant (un)archiving speedup at the cost of file size jarOutStream.setLevel(0); + // write builder arguments file and add to jar + writeBuilderArgumentsFile(); + archiveSupport.addFileToJar(layerDir, getBuilderArgumentsFilePath(), layerFile, jarOutStream); // copy the layer snapshot file and its graphs file to the jar archiveSupport.addFileToJar(layerDir, getSnapshotPath(), layerFile, jarOutStream); archiveSupport.addFileToJar(layerDir, getSnapshotGraphsPath(), layerFile, jarOutStream); // copy the shared object file to the jar Path sharedLibFile = BuildArtifacts.singleton().get(BuildArtifacts.ArtifactType.IMAGE_LAYER).getFirst(); archiveSupport.addFileToJar(NativeImageGenerator.getOutputDirectory(), sharedLibFile, layerFile, jarOutStream); - // copy the properties file to the jar + // write properties file and add to jar layerProperties.write(); archiveSupport.addFileToJar(layerDir, getLayerPropertiesFile(), layerFile, jarOutStream); BuildArtifacts.singleton().add(ArtifactType.IMAGE_LAYER_BUNDLE, layerFile); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/DiffTool.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/DiffTool.java new file mode 100644 index 000000000000..b4ed8ebcbcc1 --- /dev/null +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/util/DiffTool.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.hosted.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiPredicate; + +import com.oracle.svm.hosted.util.DiffTool.DiffResult.Kind; + +public class DiffTool { + + public record DiffResult(Kind kind, int index) { + + public enum Kind { + Equal, + Added, + Removed + } + + public String toString(List left, List right) { + return String.valueOf(switch (kind) { + case Equal -> ' '; + case Added -> '+'; + case Removed -> '-'; + }) + getEntry(left, right); + } + + public T getEntry(List left, List right) { + List holder = switch (kind) { + case Equal, Removed -> left; + case Added -> right; + }; + return holder.get(index); + } + + @Override + public String toString() { + return kind + " on " + switch (kind) { + case Equal, Removed -> "left"; + case Added -> "right"; + } + "Index=" + index; + } + } + + public static List> diffResults(List left, List right) { + return diffResults(left, right, T::equals); + } + + public static List> diffResults(List left, List right, BiPredicate equality) { + int[][] length = buildLCSLength(left, right, equality); + List> result = new ArrayList<>(); + generateDiff(length, left, right, left.size(), right.size(), equality, result); + return result; + } + + private static int[][] buildLCSLength(List left, List right, BiPredicate equality) { + int[][] length = new int[left.size() + 1][right.size() + 1]; + for (int leftIndex = 0; leftIndex < left.size(); leftIndex++) { + for (int rightIndex = 0; rightIndex < right.size(); rightIndex++) { + if (equality.test(left.get(leftIndex), right.get(rightIndex))) { + length[leftIndex + 1][rightIndex + 1] = length[leftIndex][rightIndex] + 1; + } else { + length[leftIndex + 1][rightIndex + 1] = Math.max(length[leftIndex + 1][rightIndex], length[leftIndex][rightIndex + 1]); + } + } + } + return length; + } + + private static void generateDiff(int[][] length, List left, List right, int leftIndex, int rightIndex, BiPredicate equality, List> result) { + if (leftIndex > 0 && rightIndex > 0 && equality.test(left.get(leftIndex - 1), right.get(rightIndex - 1))) { + generateDiff(length, left, right, leftIndex - 1, rightIndex - 1, equality, result); + result.add(new DiffResult<>(Kind.Equal, leftIndex - 1)); + } else if (rightIndex > 0 && (leftIndex == 0 || length[leftIndex][rightIndex - 1] >= length[leftIndex - 1][rightIndex])) { + generateDiff(length, left, right, leftIndex, rightIndex - 1, equality, result); + result.add(new DiffResult<>(Kind.Added, rightIndex - 1)); + } else if (leftIndex > 0 && (rightIndex == 0 || length[leftIndex][rightIndex - 1] < length[leftIndex - 1][rightIndex])) { + generateDiff(length, left, right, leftIndex - 1, rightIndex, equality, result); + result.add(new DiffResult<>(Kind.Removed, leftIndex - 1)); + } + } +}