From 4681cbc7f9f13799a8aae43f17670e2ac5241191 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 3 Oct 2024 21:53:13 +0200 Subject: [PATCH 01/49] Embed some data files in `libxamarin-app.so` --- .../Tasks/GeneratePackageManagerJava.cs | 14 ++++++++++++++ .../Utilities/DSOWrapperGenerator.cs | 2 +- .../Utilities/MonoAndroidHelper.cs | 5 +++++ .../Xamarin.Android.Common.targets | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 6f095c30cb2..29205ebb739 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -61,6 +61,9 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } + [Required] + public string AndroidBinUtilsDirectory { get; set; } + public bool EnableMarshalMethods { get; set; } public string RuntimeConfigBinFilePath { get; set; } public string BoundExceptionType { get; set; } @@ -320,6 +323,17 @@ void AddEnvironment () } bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); + if (haveRuntimeConfigBlob) { + List objectFilePaths = ELFEmbeddingHelper.EmbedBinary ( + Log, + SupportedAbis, + AndroidBinUtilsDirectory, + RuntimeConfigBinFilePath, + ELFEmbeddingHelper.KnownEmbedItems.RuntimeConfig, + EnvironmentOutputDirectory + ); + } + var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { UsesMonoAOT = usesMonoAOT, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs index bedd1dff72e..e20f5228027 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOWrapperGenerator.cs @@ -102,7 +102,7 @@ public static string WrapIt (TaskLoggingHelper log, Config config, AndroidTarget File.Copy (stubPath, outputFile, overwrite: true); string quotedOutputFile = MonoAndroidHelper.QuoteFileNameArgument (outputFile); - string objcopy = Path.Combine (config.AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (config.AndroidBinUtilsDirectory, "llvm-objcopy")); + string objcopy = MonoAndroidHelper.GetLlvmObjcopyPath (config.AndroidBinUtilsDirectory);; var args = new List { "--add-section", $"payload={MonoAndroidHelper.QuoteFileNameArgument (payloadFilePath)}", diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index cb7f516ddab..4372ebd34ce 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -794,5 +794,10 @@ public static string QuoteFileNameArgument (string fileName) builder.AppendFileNameIfNotNull (fileName); return builder.ToString (); } + + public static string GetLlvmObjcopyPath (string androidBinUtilsDirectory) + { + return Path.Combine (androidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (androidBinUtilsDirectory, "llvm-objcopy")); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index d34717d1ff5..0274ba729a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1746,6 +1746,7 @@ because xbuild doesn't support framework reference assemblies. UseAssemblyStore="$(AndroidUseAssemblyStore)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" + AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" > From 3f531465bb8dce7f6cedf4f1fab53d01e38998d4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 7 Oct 2024 21:48:51 +0200 Subject: [PATCH 02/49] Use llvm-mc to embed the binary, this way we have full control --- .../Tasks/GeneratePackageManagerJava.cs | 7 + .../Utilities/ELFEmbeddingHelper.cs | 191 ++++++++++++++++++ .../Utilities/MonoAndroidHelper.cs | 7 +- .../Xamarin.Android.Common.targets | 3 + src/native/monodroid/monodroid-glue.cc | 5 + .../xamarin-app-stub/application_dso_stub.cc | 3 + src/native/xamarin-app-stub/xamarin-app.hh | 2 + 7 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 29205ebb739..038c167d8df 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -64,6 +64,9 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public string AndroidBinUtilsDirectory { get; set; } + [Output] + public ITaskItem[] EmbeddedObjectFiles { get; set; } + public bool EnableMarshalMethods { get; set; } public string RuntimeConfigBinFilePath { get; set; } public string BoundExceptionType { get; set; } @@ -332,6 +335,10 @@ void AddEnvironment () ELFEmbeddingHelper.KnownEmbedItems.RuntimeConfig, EnvironmentOutputDirectory ); + + EmbeddedObjectFiles = objectFilePaths.ToArray (); + } else { + EmbeddedObjectFiles = Array.Empty (); } var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs new file mode 100644 index 00000000000..5195c4b2d17 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class ELFEmbeddingHelper +{ + public sealed class EmbedItem + { + public readonly string SymbolName; + public readonly string BaseFileName; + + public EmbedItem (string symbolName, string baseFileName) + { + SymbolName = symbolName; + BaseFileName = baseFileName; + } + } + + public static class KnownEmbedItems + { + public static readonly EmbedItem RuntimeConfig = new ("embedded_runtime_config", "runtime_config"); + } + + sealed class LlvmMcTargetConfig + { + public readonly string TargetArch; + public readonly string TripleArch; + public readonly string TripleApiPrefix; + public readonly string AssemblerDirectivePrefix; + public readonly string SizeType; + public readonly uint WordSize; + + public LlvmMcTargetConfig (string targetArch, string tripleArch, string tripleApiPrefix, string assemblerDirectivePrefix, string sizeType, uint wordSize) + { + TargetArch = targetArch; + TripleArch = tripleArch; + TripleApiPrefix = tripleApiPrefix; + AssemblerDirectivePrefix = assemblerDirectivePrefix; + SizeType = sizeType; + WordSize = wordSize; + } + } + + static readonly Dictionary llvmMcConfigs = new () { + { AndroidTargetArch.Arm64, new ("aarch64", "aarch64", "android", "@", ".xword", 8) }, + { AndroidTargetArch.Arm, new ("arm", "armv7a", "androideabi", "%", ".long", 4) }, + { AndroidTargetArch.X86_64, new ("x86-64", "x86_64", "android", "@", ".quad", 8) }, + { AndroidTargetArch.X86, new ("x86", "i686", "android", "@", ".long", 4) }, + }; + + static readonly Encoding asmFileEncoding = new UTF8Encoding (false); + + public static List EmbedBinary ( + TaskLoggingHelper log, + ICollection supportedAbis, + string androidBinUtilsDirectory, + string inputFile, + EmbedItem embedItem, + string outputDirectory) + { + if (supportedAbis.Count < 1) { + throw new ArgumentException ("At least one target ABI must be present", nameof (supportedAbis)); + } + + string llvmMcPath = GetLlvmMcPath (androidBinUtilsDirectory); + var ret = new List (); + foreach (string abi in supportedAbis) { + EmbedBinary ( + log, + ret, + llvmMcPath, + abi, + inputFile, + outputDirectory, + embedItem + ); + } + + return ret; + } + + public static List EmbedBinary ( + TaskLoggingHelper log, + string abi, + string androidBinUtilsDirectory, + string inputFile, + EmbedItem embedItem, + string outputDirectory) + { + if (String.IsNullOrEmpty (abi)) { + throw new ArgumentException ("Must be a supported ABI name", nameof (abi)); + } + + var ret = new List (); + EmbedBinary ( + log, + ret, + GetLlvmMcPath (androidBinUtilsDirectory), + abi, + inputFile, + outputDirectory, + embedItem + ); + return ret; + } + + static void EmbedBinary ( + TaskLoggingHelper log, + List resultItems, + string llvmMcPath, + string abi, + string inputFile, + string outputDirectory, + EmbedItem embedItem) + { + string outputFile = Path.Combine (outputDirectory, $"embed_{embedItem.BaseFileName}.{abi.ToLowerInvariant ()}.o"); + DoEmbed (log, MonoAndroidHelper.AbiToTargetArch (abi), llvmMcPath, inputFile, outputFile, embedItem); + if (!File.Exists (outputFile)) { + return; + } + + var taskItem = new TaskItem (outputFile); + taskItem.SetMetadata ("Abi", abi); + taskItem.SetMetadata ("RuntimeIdentifier", MonoAndroidHelper.AbiToRid (abi)); + resultItems.Add (taskItem); + } + + static void DoEmbed ( + TaskLoggingHelper log, + AndroidTargetArch arch, + string llvmMcPath, + string inputFile, + string outputFile, + EmbedItem item) + { + if (!llvmMcConfigs.TryGetValue (arch, out LlvmMcTargetConfig cfg)) { + throw new NotSupportedException ($"Internal error: unsupported target arch '{arch}'"); + } + + var fi = new FileInfo (inputFile); + long inputFileSize = fi.Length; + string asmInputFile = Path.ChangeExtension (outputFile, ".s"); + + using var fs = File.Open (asmInputFile, FileMode.Create, FileAccess.Write, FileShare.Read); + using var sw = new StreamWriter (fs, asmFileEncoding); + + string symbolName = item.SymbolName; + sw.WriteLine ($".section .rodata,\"a\",{cfg.AssemblerDirectivePrefix}progbits"); + sw.WriteLine (".p2align 3, 0x00"); // Put the data at 4k boundary + sw.WriteLine (); + sw.WriteLine ($".global {symbolName}"); + sw.WriteLine ($".type {symbolName},{cfg.AssemblerDirectivePrefix}object"); + sw.WriteLine ($"{symbolName}:"); + sw.WriteLine ($"\t.incbin \"{inputFile}\""); + sw.WriteLine ($"\t.size {symbolName}, {inputFileSize}"); + sw.WriteLine (); + + symbolName += "_size"; + sw.WriteLine ($".global {symbolName}"); + sw.WriteLine ($"{symbolName}:"); + sw.WriteLine ($"\t{cfg.SizeType}\t{inputFileSize}"); + sw.WriteLine ($"\t.size {symbolName}, {cfg.WordSize}"); + + sw.Flush (); + sw.Close (); + + var args = new List { + $"--arch={cfg.TargetArch}", + "--assemble", + "--filetype=obj", + "-g", + $"--triple={cfg.TripleArch}-linux-{cfg.TripleApiPrefix}{XABuildConfig.AndroidMinimumDotNetApiLevel}", + "-o", MonoAndroidHelper.QuoteFileNameArgument (outputFile), + MonoAndroidHelper.QuoteFileNameArgument (asmInputFile), + }; + + int ret = MonoAndroidHelper.RunProcess (llvmMcPath, String.Join (" ", args), log); + if (ret != 0) { + return; + } + } + + static string GetLlvmMcPath (string androidBinUtilsDirectory) => MonoAndroidHelper.GetLlvmMcPath (androidBinUtilsDirectory); +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 4372ebd34ce..61734c55cb4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -795,9 +795,12 @@ public static string QuoteFileNameArgument (string fileName) return builder.ToString (); } - public static string GetLlvmObjcopyPath (string androidBinUtilsDirectory) + public static string GetLlvmObjcopyPath (string androidBinUtilsDirectory) => GetBinUtilsToolPath (androidBinUtilsDirectory, "llvm-objcopy"); + public static string GetLlvmMcPath (string androidBinUtilsDirectory) => GetBinUtilsToolPath (androidBinUtilsDirectory, "llvm-mc"); + + static string GetBinUtilsToolPath (string androidBinUtilsDirectory, string toolName) { - return Path.Combine (androidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (androidBinUtilsDirectory, "llvm-objcopy")); + return Path.Combine (androidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (androidBinUtilsDirectory, toolName)); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 0274ba729a6..e1e3910beb2 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1709,6 +1709,7 @@ because xbuild doesn't support framework reference assemblies. <_GeneratePackageManagerJavaInputs Include="@(_GenerateJavaStubsInputs)" /> + <_GeneratePackageManagerJavaInputs Include="$(_BinaryRuntimeConfigPath)" /> @@ -1748,10 +1749,12 @@ because xbuild doesn't support framework reference assemblies. CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" > + + diff --git a/src/native/monodroid/monodroid-glue.cc b/src/native/monodroid/monodroid-glue.cc index 03c8ead558e..2e99550f849 100644 --- a/src/native/monodroid/monodroid-glue.cc +++ b/src/native/monodroid/monodroid-glue.cc @@ -1377,6 +1377,11 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl Logger::init_logging_categories (mono_log_mask_raw, mono_log_level_raw); + log_warn (LOG_DEFAULT, "Embedded runtime config size: %zu", embedded_runtime_config_size); + if (embedded_runtime_config_size > 0) { + log_warn (LOG_DEFAULT, "First byte of embedded runtime config: 0x%x", embedded_runtime_config[0]); + } + std::unique_ptr mono_log_mask (mono_log_mask_raw); std::unique_ptr mono_log_level (mono_log_level_raw); diff --git a/src/native/xamarin-app-stub/application_dso_stub.cc b/src/native/xamarin-app-stub/application_dso_stub.cc index f840005c204..93f25897f6b 100644 --- a/src/native/xamarin-app-stub/application_dso_stub.cc +++ b/src/native/xamarin-app-stub/application_dso_stub.cc @@ -303,3 +303,6 @@ const JniRemappingTypeReplacementEntry jni_remapping_type_replacements[] = { .replacement = "another/replacement/java/type", }, }; + +size_t embedded_runtime_config_size = 0; +uint8_t embedded_runtime_config[0]; diff --git a/src/native/xamarin-app-stub/xamarin-app.hh b/src/native/xamarin-app-stub/xamarin-app.hh index 426b4913e55..3b3bc8e3843 100644 --- a/src/native/xamarin-app-stub/xamarin-app.hh +++ b/src/native/xamarin-app-stub/xamarin-app.hh @@ -396,4 +396,6 @@ using get_function_pointer_fn = void(*)(uint32_t mono_image_index, uint32_t clas MONO_API MONO_API_EXPORT void xamarin_app_init (JNIEnv *env, get_function_pointer_fn fn) noexcept; #endif // def RELEASE +MONO_API MONO_API_EXPORT size_t embedded_runtime_config_size; +MONO_API MONO_API_EXPORT uint8_t embedded_runtime_config[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From a117cac89503cb99c95ce0e18ea34392af6d58b9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 8 Oct 2024 11:51:24 +0200 Subject: [PATCH 03/49] Runtime config binary blob is now fully embedded --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 2 +- .../Utilities/ELFEmbeddingHelper.cs | 6 +++++- src/native/monodroid/embedded-assemblies-zip.cc | 14 +++++++------- src/native/monodroid/monodroid-glue.cc | 15 +++++++++------ src/native/runtime-base/android-system.cc | 2 ++ 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 024b181fc66..95067b681cf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -218,7 +218,7 @@ void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supp apk.Flush (); } - AddRuntimeConfigBlob (dsoWrapperConfig, apk); +// AddRuntimeConfigBlob (apk); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); AddNativeLibraries (files, supportedAbis); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 5195c4b2d17..62da88708c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -144,9 +144,13 @@ static void DoEmbed ( throw new NotSupportedException ($"Internal error: unsupported target arch '{arch}'"); } + inputFile = Path.GetFullPath (inputFile); + outputFile = Path.GetFullPath (outputFile); + var fi = new FileInfo (inputFile); long inputFileSize = fi.Length; string asmInputFile = Path.ChangeExtension (outputFile, ".s"); + string sanitizedInputFilePath = inputFile.Replace ("\\", "\\\\"); using var fs = File.Open (asmInputFile, FileMode.Create, FileAccess.Write, FileShare.Read); using var sw = new StreamWriter (fs, asmFileEncoding); @@ -158,7 +162,7 @@ static void DoEmbed ( sw.WriteLine ($".global {symbolName}"); sw.WriteLine ($".type {symbolName},{cfg.AssemblerDirectivePrefix}object"); sw.WriteLine ($"{symbolName}:"); - sw.WriteLine ($"\t.incbin \"{inputFile}\""); + sw.WriteLine ($"\t.incbin \"{sanitizedInputFilePath}\""); sw.WriteLine ($"\t.size {symbolName}, {inputFileSize}"); sw.WriteLine (); diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 3f91b71dc9a..68f163fa740 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -61,13 +61,13 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector 0) { size_t blob_time_index; if (FastTiming::enabled ()) [[unlikely]] { blob_time_index = internal_timing->start_event (TimingEventKind::RuntimeConfigBlob); } runtime_config_args.kind = 1; - embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); +// embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); + runtime_config_args.runtimeconfig.data.data = reinterpret_cast(embedded_runtime_config); + runtime_config_args.runtimeconfig.data.data_len = static_cast(embedded_runtime_config_size); monovm_runtimeconfig_initialize (&runtime_config_args, cleanup_runtime_config, nullptr); if (FastTiming::enabled ()) [[unlikely]] { @@ -1377,10 +1380,10 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl Logger::init_logging_categories (mono_log_mask_raw, mono_log_level_raw); - log_warn (LOG_DEFAULT, "Embedded runtime config size: %zu", embedded_runtime_config_size); - if (embedded_runtime_config_size > 0) { - log_warn (LOG_DEFAULT, "First byte of embedded runtime config: 0x%x", embedded_runtime_config[0]); - } + // log_warn (LOG_DEFAULT, "Embedded runtime config size: %zu", embedded_runtime_config_size); + // if (embedded_runtime_config_size > 0) { + // log_warn (LOG_DEFAULT, "First byte of embedded runtime config: 0x%x", embedded_runtime_config[0]); + // } std::unique_ptr mono_log_mask (mono_log_mask_raw); std::unique_ptr mono_log_level (mono_log_level_raw); diff --git a/src/native/runtime-base/android-system.cc b/src/native/runtime-base/android-system.cc index 025362628ae..eb9d38fe86b 100644 --- a/src/native/runtime-base/android-system.cc +++ b/src/native/runtime-base/android-system.cc @@ -741,10 +741,12 @@ AndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_arr if (have_split_apks) { if (Util::ends_with (apk, SharedConstants::split_config_abi_apk_name)) { + log_warn (LOG_ASSEMBLY, "Here #1"); add_apk_libdir (apk, number_of_added_directories, abi); break; } } else { + log_warn (LOG_ASSEMBLY, "Here #2"); add_apk_libdir (apk, number_of_added_directories, abi); } } From eabdee55c268279646932d49858adec80e3c8948 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 8 Oct 2024 17:53:04 +0200 Subject: [PATCH 04/49] Remove debug spam --- src/native/runtime-base/android-system.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/native/runtime-base/android-system.cc b/src/native/runtime-base/android-system.cc index eb9d38fe86b..025362628ae 100644 --- a/src/native/runtime-base/android-system.cc +++ b/src/native/runtime-base/android-system.cc @@ -741,12 +741,10 @@ AndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_arr if (have_split_apks) { if (Util::ends_with (apk, SharedConstants::split_config_abi_apk_name)) { - log_warn (LOG_ASSEMBLY, "Here #1"); add_apk_libdir (apk, number_of_added_directories, abi); break; } } else { - log_warn (LOG_ASSEMBLY, "Here #2"); add_apk_libdir (apk, number_of_added_directories, abi); } } From daa654c05dded43babf4c4f6a5c2a4be3c394932 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Oct 2024 17:23:08 +0200 Subject: [PATCH 05/49] New target to build embedded assembly store --- .../Microsoft.Android.Sdk.After.targets | 1 + ...crosoft.Android.Sdk.AssemblyStores.targets | 31 +++++++++++++++++++ .../Tasks/CreateEmbeddedAssemblyStore.cs | 14 +++++++++ 3 files changed, 46 insertions(+) create mode 100644 src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets index 8a64f834b80..c01bf0248cd 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets @@ -29,4 +29,5 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets. + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets new file mode 100644 index 00000000000..afa905adcac --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -0,0 +1,31 @@ + + + + + + + + <_EmbeddedAssemblyStoreObjectFile Include="$(_NativeAssemblySourceDir)embed_assembly_store.%(_BuildTargetAbis.Identity).o" /> + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs new file mode 100644 index 00000000000..d68e6d386f5 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -0,0 +1,14 @@ +using Xamarin.Android.Tools; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class CreateEmbeddedAssemblyStore : AndroidTask +{ + public override string TaskPrefix => "CEAS"; + + public override bool RunTask () + { + return !Log.HasLoggedErrors; + } +} From 70c7b7413f16a7f876b01ad0da219f055eb1824f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Oct 2024 17:24:11 +0200 Subject: [PATCH 06/49] New task to build embedded assembly store --- .../Tasks/CreateEmbeddedAssemblyStore.cs | 22 ++++++++++++++++++- .../Utilities/ELFEmbeddingHelper.cs | 1 + .../Xamarin.Android.Common.targets | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs index d68e6d386f5..357575e5fdd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -1,5 +1,7 @@ -using Xamarin.Android.Tools; + using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -7,6 +9,24 @@ public class CreateEmbeddedAssemblyStore : AndroidTask { public override string TaskPrefix => "CEAS"; + [Required] + public string AndroidBinUtilsDirectory { get; set; } + + [Required] + public string AppSharedLibrariesDir { get; set; } + + [Required] + public ITaskItem[] ResolvedUserAssemblies { get; set; } + + [Required] + public ITaskItem[] ResolvedFrameworkAssemblies { get; set; } + + [Output] + public ITaskItem[] NativeAssemblySources { get; set; } + + [Output] + public ITaskItem[] EmbeddedObjectFiles { get; set; } + public override bool RunTask () { return !Log.HasLoggedErrors; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 62da88708c7..56f399913f2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -26,6 +26,7 @@ public EmbedItem (string symbolName, string baseFileName) public static class KnownEmbedItems { public static readonly EmbedItem RuntimeConfig = new ("embedded_runtime_config", "runtime_config"); + public static readonly EmbedItem AssemblyStore = new ("embedded_assembly_store", "assembly_store"); } sealed class LlvmMcTargetConfig diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index e1e3910beb2..6c0d200e27b 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -327,6 +327,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) + <_AndroidEmbedAssemblyStoreInRuntime Condition=" '$(AndroidUseAssemblyStore)' == 'True' And '$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">True @@ -1998,7 +1999,7 @@ because xbuild doesn't support framework reference assemblies. Date: Wed, 9 Oct 2024 18:03:38 +0200 Subject: [PATCH 07/49] Move some of assembly packaging code to a helper class --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 2 +- .../Utilities/AssemblyPackagingHelper.cs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 95067b681cf..fa2346ef796 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -459,7 +459,7 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, GetCompressionMethod (inArchivePath)); } - void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) + void DoAddAssembliesFromArchCollection (AndroidTargetArch arch, Dictionary assemblies) { // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyPackagingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyPackagingHelper.cs index 1599a8e581e..ca9bd62ea33 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyPackagingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyPackagingHelper.cs @@ -10,7 +10,11 @@ namespace Xamarin.Android.Tasks; static class AssemblyPackagingHelper { +<<<<<<< HEAD public static void AddAssembliesFromCollection (TaskLoggingHelper Log, ICollection SupportedAbis, ICollection assemblies, Action doAddAssembly) +======= + public static void AddAssembliesFromCollection (TaskLoggingHelper Log, ICollection SupportedAbis, ICollection assemblies, Action> doAddAssemblies) +>>>>>>> c4d883405 (Move some of assembly packaging code to a helper class) { Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( assemblies, @@ -28,6 +32,7 @@ public static void AddAssembliesFromCollection (TaskLoggingHelper Log, ICollecti foreach (var kvp in perArchAssemblies) { Log.LogDebugMessage ($"Adding assemblies for architecture '{kvp.Key}'"); +<<<<<<< HEAD DoAddAssembliesFromArchCollection (Log, kvp.Key, kvp.Value, doAddAssembly); } } @@ -40,6 +45,9 @@ static void DoAddAssembliesFromArchCollection (TaskLoggingHelper Log, AndroidTar } doAddAssembly (Log, arch, assembly); +======= + doAddAssemblies (kvp.Key, kvp.Value); +>>>>>>> c4d883405 (Move some of assembly packaging code to a helper class) } } } From d34cffc724f6fdc6192b96dc6f95bf8cf4c71db6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Oct 2024 18:25:26 +0200 Subject: [PATCH 08/49] Move more assembly packaging code to the helper class --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 10 +--------- .../Utilities/AssemblyPackagingHelper.cs | 8 -------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index fa2346ef796..c457e22784e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -459,7 +459,7 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, GetCompressionMethod (inArchivePath)); } - void DoAddAssembliesFromArchCollection (AndroidTargetArch arch, Dictionary assemblies) + void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) { // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. @@ -501,14 +501,6 @@ void DoAddAssembliesFromArchCollection (AndroidTargetArch arch, Dictionary SupportedAbis, ICollection assemblies, Action doAddAssembly) -======= - public static void AddAssembliesFromCollection (TaskLoggingHelper Log, ICollection SupportedAbis, ICollection assemblies, Action> doAddAssemblies) ->>>>>>> c4d883405 (Move some of assembly packaging code to a helper class) { Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( assemblies, @@ -32,7 +28,6 @@ public static void AddAssembliesFromCollection (TaskLoggingHelper Log, ICollecti foreach (var kvp in perArchAssemblies) { Log.LogDebugMessage ($"Adding assemblies for architecture '{kvp.Key}'"); -<<<<<<< HEAD DoAddAssembliesFromArchCollection (Log, kvp.Key, kvp.Value, doAddAssembly); } } @@ -45,9 +40,6 @@ static void DoAddAssembliesFromArchCollection (TaskLoggingHelper Log, AndroidTar } doAddAssembly (Log, arch, assembly); -======= - doAddAssemblies (kvp.Key, kvp.Value); ->>>>>>> c4d883405 (Move some of assembly packaging code to a helper class) } } } From 623d7bd7dc6a4fc2c5359591c5bd7384ae05302c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 9 Oct 2024 20:43:18 +0200 Subject: [PATCH 09/49] Almost there, TBC --- ...crosoft.Android.Sdk.AssemblyStores.targets | 7 ++- .../Microsoft.Android.Sdk.BuildOrder.targets | 1 - .../Tasks/BuildApk.cs | 8 +-- .../Tasks/CreateEmbeddedAssemblyStore.cs | 54 +++++++++++++++++++ .../Xamarin.Android.Common.targets | 9 +++- 5 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index afa905adcac..ba76fad8ce1 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -17,8 +17,13 @@ + ResolvedFrameworkAssemblies="@(_ShrunkFrameworkAssemblies)" + SupportedAbis="@(_BuildTargetAbis)"> diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index ee627e65d03..d937c238c6f 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -62,7 +62,6 @@ properties that determine build ordering. <_PrepareBuildApkDependsOnTargets> _SetLatestTargetFrameworkVersion; _GetLibraryImports; - _RemoveRegisterAttribute; _ResolveAssemblies; _ResolveSatellitePaths; _CreatePackageWorkspace; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index c457e22784e..672cd34d774 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -118,6 +118,9 @@ public class BuildApk : AndroidTask [Required] public string ProjectFullPath { get; set; } + [Required] + public string CompressedAssembliesDir { get; set; } + [Output] public ITaskItem[] OutputFiles { get; set; } @@ -354,7 +357,7 @@ public override bool RunTask () if (compress) { string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); + Log.LogDebugMessage ($"[{TaskPrefix}] Retrieving assembly compression info with key '{key}'"); compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); if (compressedAssembliesInfo == null) throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); @@ -424,7 +427,6 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap { string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; - string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); AssemblyStoreBuilder? storeBuilder = null; if (UseAssemblyStore) { @@ -507,7 +509,7 @@ string CompressAssembly (ITaskItem assembly) return assembly.ItemSpec; } - return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, compressedOutputDir); + return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, CompressedAssembliesDir); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs index 357575e5fdd..3d272f3b933 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -1,6 +1,9 @@ +using System; +using System.Collections.Generic; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -15,12 +18,27 @@ public class CreateEmbeddedAssemblyStore : AndroidTask [Required] public string AppSharedLibrariesDir { get; set; } + [Required] + public string CompressedAssembliesDir { get; set; } + + [Required] + public bool Debug { get; set; } + + [Required] + public bool EnableCompression { get; set; } + + [Required] + public string ProjectFullPath { get; set; } + [Required] public ITaskItem[] ResolvedUserAssemblies { get; set; } [Required] public ITaskItem[] ResolvedFrameworkAssemblies { get; set; } + [Required] + public string [] SupportedAbis { get; set; } + [Output] public ITaskItem[] NativeAssemblySources { get; set; } @@ -29,6 +47,42 @@ public class CreateEmbeddedAssemblyStore : AndroidTask public override bool RunTask () { + bool compress = !Debug && EnableCompression; + IDictionary>? compressedAssembliesInfo = null; + + if (compress) { + string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); + Log.LogDebugMessage ($"[{TaskPrefix}] Retrieving assembly compression info with key '{key}'"); + compressedAssembliesInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); + if (compressedAssembliesInfo == null) { + throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); + } + } + + var storeBuilder = new AssemblyStoreBuilder (Log); + + // Add user assemblies + AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, DoAddAssembliesFromArchCollection); + + // Add framework assemblies + AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection); + return !Log.HasLoggedErrors; + + void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) + { + string sourcePath = CompressAssembly (assembly); + storeBuilder.AddAssembly (sourcePath, assembly, includeDebugSymbols: Debug); + return; + } + + string CompressAssembly (ITaskItem assembly) + { + if (!compress) { + return assembly.ItemSpec; + } + + return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, CompressedAssembliesDir); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 6c0d200e27b..13ce1dc14e7 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -292,7 +292,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidFastDeployEnvironmentFiles Condition=" '$(_AndroidFastDeployEnvironmentFiles)' == '' And '$(EmbedAssembliesIntoApk)' == 'False' ">True <_AndroidFastDeployEnvironmentFiles Condition=" '$(_AndroidFastDeployEnvironmentFiles)' == '' ">False - + <_AndroidCompressedAssembliesDir>$(IntermediateOutputPath)android\lz4 @@ -1869,6 +1869,7 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; + _RemoveRegisterAttribute; _CreateApplicationSharedLibraries; _GetMonoPlatformJarPath; _GetLibraryImports; @@ -1999,7 +2000,7 @@ because xbuild doesn't support framework reference assemblies. + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> Date: Thu, 10 Oct 2024 11:33:07 +0200 Subject: [PATCH 10/49] Assembly store embedding works --- ...crosoft.Android.Sdk.AssemblyStores.targets | 1 + .../Tasks/CreateEmbeddedAssemblyStore.cs | 37 +++++++++++++++++++ .../Utilities/AssemblyStoreGenerator.cs | 4 +- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index ba76fad8ce1..c7185129668 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -17,6 +17,7 @@ (); + var sourceFiles = new List (); + Dictionary assemblyStorePaths = storeBuilder.Generate (Path.Combine (AppSharedLibrariesDir, "embedded")); + foreach (var kvp in assemblyStorePaths) { + string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); + string inputFile = kvp.Value; + + List items = ELFEmbeddingHelper.EmbedBinary ( + Log, + abi, + AndroidBinUtilsDirectory, + inputFile, + ELFEmbeddingHelper.KnownEmbedItems.AssemblyStore, + AssemblySourcesDir + ); + + if (items.Count == 0) { + continue; + } + + objectFiles.AddRange (items); + foreach (ITaskItem objectItem in items) { + var sourceItem = new TaskItem ( + Path.ChangeExtension (objectItem.ItemSpec, ".s"), + objectItem.CloneCustomMetadata () + ); + sourceFiles.Add (sourceItem); + } + } + + NativeAssemblySources = sourceFiles.ToArray (); + EmbeddedObjectFiles = objectFiles.ToArray (); + return !Log.HasLoggedErrors; void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 896c17e43c3..728be5dc285 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -101,7 +101,8 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List (); var descriptors = new List (); ulong namesSize = 0; @@ -115,6 +116,7 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List Date: Thu, 10 Oct 2024 18:05:33 +0200 Subject: [PATCH 11/49] Actually use the embedded assembly store --- .../Tasks/BuildApk.cs | 8 ++ .../Xamarin.Android.Common.targets | 4 + .../monodroid/embedded-assemblies-zip.cc | 87 +++++++++++-------- src/native/monodroid/embedded-assemblies.hh | 2 + .../xamarin-app-stub/application_dso_stub.cc | 3 + src/native/xamarin-app-stub/xamarin-app.hh | 3 + 6 files changed, 70 insertions(+), 37 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 672cd34d774..d387c06ea80 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -121,6 +121,9 @@ public class BuildApk : AndroidTask [Required] public string CompressedAssembliesDir { get; set; } + [Required] + public bool AssemblyStoreEmbeddedInRuntime { get; set; } + [Output] public ITaskItem[] OutputFiles { get; set; } @@ -430,6 +433,11 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap AssemblyStoreBuilder? storeBuilder = null; if (UseAssemblyStore) { + if (AssemblyStoreEmbeddedInRuntime) { + // We don't need to do anything here, the store is in `libxamarin-app.so` + return; + } + storeBuilder = new AssemblyStoreBuilder (Log); } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 13ce1dc14e7..127c8f848f7 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2118,6 +2118,7 @@ because xbuild doesn't support framework reference assemblies. AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" IntermediateOutputPath="$(IntermediateOutputPath)"> CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> c } } + +[[gnu::always_inline]] void +EmbeddedAssemblies::verify_assembly_store_and_set_info (void *data_start, const char *name) noexcept +{ + auto header = static_cast(data_start); + + if (header->magic != ASSEMBLY_STORE_MAGIC) { + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Assembly store '%s' is not a valid .NET for Android assembly store file", + entry_name.get () + ) + ); + } + + if (header->version != ASSEMBLY_STORE_FORMAT_VERSION) { + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Assembly store '%s' uses format version 0x%x, instead of the expected 0x%x", + entry_name.get (), + header->version, + ASSEMBLY_STORE_FORMAT_VERSION + ) + ); + } + + constexpr size_t header_size = sizeof(AssemblyStoreHeader); + + assembly_store.data_start = static_cast(data_start); + assembly_store.assembly_count = header->entry_count; + assembly_store.index_entry_count = header->index_entry_count; + assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); + assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); + + number_of_found_assemblies += assembly_store.assembly_count; + number_of_mapped_assembly_stores++; + have_and_want_debug_symbols = register_debug_symbols; +} + inline void EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept { @@ -210,41 +251,7 @@ EmbeddedAssemblies::map_assembly_store (dynamic_local_string auto [payload_start, payload_size] = get_wrapper_dso_payload_pointer_and_size (assembly_store_map, entry_name.get ()); log_debug (LOG_ASSEMBLY, "Adjusted assembly store pointer: %p; size: %zu", payload_start, payload_size); - auto header = static_cast(payload_start); - - if (header->magic != ASSEMBLY_STORE_MAGIC) { - Helpers::abort_application ( - LOG_ASSEMBLY, - Util::monodroid_strdup_printf ( - "Assembly store '%s' is not a valid .NET for Android assembly store file", - entry_name.get () - ) - ); - } - - if (header->version != ASSEMBLY_STORE_FORMAT_VERSION) { - Helpers::abort_application ( - LOG_ASSEMBLY, - Util::monodroid_strdup_printf ( - "Assembly store '%s' uses format version 0x%x, instead of the expected 0x%x", - entry_name.get (), - header->version, - ASSEMBLY_STORE_FORMAT_VERSION - ) - ); - } - - constexpr size_t header_size = sizeof(AssemblyStoreHeader); - - assembly_store.data_start = static_cast(payload_start); - assembly_store.assembly_count = header->entry_count; - assembly_store.index_entry_count = header->index_entry_count; - assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); - assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); - - number_of_found_assemblies += assembly_store.assembly_count; - number_of_mapped_assembly_stores++; - have_and_want_debug_symbols = register_debug_symbols; + verify_assembly_store_and_set_info (payload_start, entry_name.get ()); } force_inline void @@ -255,9 +262,15 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& } dynamic_local_string entry_name; - bool assembly_store_found = false; + bool assembly_store_found = embedded_assembly_store_size != 0; + if (assembly_store_found) { + log_debug (LOG_ASSEMBLY, "Got embedded assembly store, size %zu", embedded_assembly_store_size); + verify_assembly_store_and_set_info (embedded_assembly_store, "embedded"); + log_debug (LOG_ASSEMBLY, "Looking for DSOs in APK"); + } else { + log_debug (LOG_ASSEMBLY, "Looking for assembly store ('%s') and DSOs in APK", assembly_store_file_path.data ()); + } - log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK ('%s)", assembly_store_file_path.data ()); for (size_t i = 0uz; i < num_entries; i++) { if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index ce9c9068ffe..e7ddb6258c4 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -353,6 +353,8 @@ namespace xamarin::android::internal { void set_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; + + void verify_assembly_store_and_set_info (void *data_start, const char *name) noexcept; void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; const AssemblyStoreIndexEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept; void store_individual_assembly_data (dynamic_local_string const& entry_name, ZipEntryLoadState const& state, monodroid_should_register should_register) noexcept; diff --git a/src/native/xamarin-app-stub/application_dso_stub.cc b/src/native/xamarin-app-stub/application_dso_stub.cc index 93f25897f6b..3c8b937c3d7 100644 --- a/src/native/xamarin-app-stub/application_dso_stub.cc +++ b/src/native/xamarin-app-stub/application_dso_stub.cc @@ -306,3 +306,6 @@ const JniRemappingTypeReplacementEntry jni_remapping_type_replacements[] = { size_t embedded_runtime_config_size = 0; uint8_t embedded_runtime_config[0]; + +size_t embedded_assembly_store_size = 0; +uint8_t embedded_assembly_store[0]; diff --git a/src/native/xamarin-app-stub/xamarin-app.hh b/src/native/xamarin-app-stub/xamarin-app.hh index 3b3bc8e3843..1eeb26e4954 100644 --- a/src/native/xamarin-app-stub/xamarin-app.hh +++ b/src/native/xamarin-app-stub/xamarin-app.hh @@ -398,4 +398,7 @@ MONO_API MONO_API_EXPORT void xamarin_app_init (JNIEnv *env, get_function_pointe MONO_API MONO_API_EXPORT size_t embedded_runtime_config_size; MONO_API MONO_API_EXPORT uint8_t embedded_runtime_config[]; + +MONO_API MONO_API_EXPORT size_t embedded_assembly_store_size; +MONO_API MONO_API_EXPORT uint8_t embedded_assembly_store[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H From a6a0e2b5c2bd40dfcf0b7ce4934945c9263685d9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 10 Oct 2024 22:28:02 +0200 Subject: [PATCH 12/49] A handful of bugfixes and performance tweaks --- .../Tasks/GeneratePackageManagerJava.cs | 4 ++++ ...pplicationConfigNativeAssemblyGenerator.cs | 19 +++++++++++++++++++ .../Xamarin.Android.Common.targets | 2 ++ src/native/monodroid/monodroid-glue.cc | 2 +- src/native/runtime-base/android-system.cc | 18 +++--------------- src/native/runtime-base/timing-internal.cc | 2 -- src/native/runtime-base/util.cc | 10 ++++------ src/native/runtime-base/util.hh | 2 +- 8 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 038c167d8df..e5207abdeae 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -64,6 +64,9 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public string AndroidBinUtilsDirectory { get; set; } + [Required] + public bool AssemblyStoreEmbeddedInRuntime { get; set; } + [Output] public ITaskItem[] EmbeddedObjectFiles { get; set; } @@ -366,6 +369,7 @@ void AddEnvironment () JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, MarshalMethodsEnabled = EnableMarshalMethods, IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), + AssemblyStoreEmbeddedInRuntime = UseAssemblyStore && AssemblyStoreEmbeddedInRuntime, }; LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 9a0bc213d6c..45e4ae7a4e0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -188,6 +188,7 @@ sealed class XamarinAndroidBundledAssembly public List NativeLibraries { get; set; } public bool MarshalMethodsEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } + public bool AssemblyStoreEmbeddedInRuntime { get; set; } public ApplicationConfigNativeAssemblyGenerator (IDictionary environmentVariables, IDictionary systemProperties, TaskLoggingHelper log) : base (log) @@ -294,6 +295,24 @@ protected override void Construct (LlvmIrModule module) module.Add (bundled_assemblies); AddAssemblyStores (module); + + if (AssemblyStoreEmbeddedInRuntime) { + return; + } + + // Need these to keep ABI compatibility with `libxamarin-app.so` used at the runtime's build time + var embedded_assembly_store_size = new LlvmIrGlobalVariable ( + (ulong)0, + "embedded_assembly_store_size", + LlvmIrVariableOptions.GlobalConstant + ); + module.Add (embedded_assembly_store_size); + + var embedded_assembly_store = new LlvmIrGlobalVariable (typeof (byte[]), "embedded_assembly_store", LlvmIrVariableOptions.GlobalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = 0, + }; + module.Add (embedded_assembly_store); } void AddAssemblyStores (LlvmIrModule module) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 127c8f848f7..03de60fd0d4 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -328,6 +328,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) <_AndroidEmbedAssemblyStoreInRuntime Condition=" '$(AndroidUseAssemblyStore)' == 'True' And '$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">True + <_AndroidEmbedAssemblyStoreInRuntime Condition="'$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">False @@ -1749,6 +1750,7 @@ because xbuild doesn't support framework reference assemblies. EnableMarshalMethods="$(_AndroidUseMarshalMethods)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)" > diff --git a/src/native/monodroid/monodroid-glue.cc b/src/native/monodroid/monodroid-glue.cc index 605abba1ae6..c02ab5796dc 100644 --- a/src/native/monodroid/monodroid-glue.cc +++ b/src/native/monodroid/monodroid-glue.cc @@ -1126,7 +1126,7 @@ MonodroidRuntime::set_profile_options () .append (OUTPUT_ARG) .append (output_path.get (), output_path.length ()); } - if (Util::create_directory (AndroidSystem::override_dirs[0], 0) < 0) { + if (Util::create_directory (AndroidSystem::override_dirs[0], 0777, 000) < 0) { log_warn (LOG_DEFAULT, "Failed to create directory '%s'. %s", AndroidSystem::override_dirs[0], std::strerror (errno)); } diff --git a/src/native/runtime-base/android-system.cc b/src/native/runtime-base/android-system.cc index 025362628ae..98176e84223 100644 --- a/src/native/runtime-base/android-system.cc +++ b/src/native/runtime-base/android-system.cc @@ -262,26 +262,14 @@ AndroidSystem::monodroid_get_system_property_from_overrides ([[maybe_unused]] co return 0; } -// TODO: review this. Do we really have to create the dir in release? void AndroidSystem::create_update_dir (char *override_dir) noexcept { -#if defined (RELEASE) - /* - * Don't create .__override__ on Release builds, because Google requires - * that pre-loaded apps not create world-writable directories. - * - * However, if any logging is enabled (which should _not_ happen with - * pre-loaded apps!), we need the .__override__ directory... - */ - if (log_categories == 0 && monodroid_get_system_property (SharedConstants::DEBUG_MONO_PROFILE_PROPERTY, nullptr) == 0) { - return; - } -#endif // def RELEASE - override_dirs [0] = override_dir; +#if defined(DEBUG) + log_debug (LOG_DEFAULT, "Creating public update directory: `%s`", override_dir); Util::create_public_directory (override_dir); - log_warn (LOG_DEFAULT, "Creating public update directory: `%s`", override_dir); +#endif } bool diff --git a/src/native/runtime-base/timing-internal.cc b/src/native/runtime-base/timing-internal.cc index eaaee8c17ef..0d86c1e0a28 100644 --- a/src/native/runtime-base/timing-internal.cc +++ b/src/native/runtime-base/timing-internal.cc @@ -22,8 +22,6 @@ FastTiming::really_initialize (bool log_immediately) noexcept if (immediate_logging) { return; } - - log_write (LOG_TIMING, LogLevel::Info, "[2/1] To get timing results, send the mono.android.app.DUMP_TIMING_DATA intent to the application"); } void diff --git a/src/native/runtime-base/util.cc b/src/native/runtime-base/util.cc index 7a58a2836c1..7ea54bbf857 100644 --- a/src/native/runtime-base/util.cc +++ b/src/native/runtime-base/util.cc @@ -146,16 +146,14 @@ Util::path_combine (const char *path1, const char *path2) void Util::create_public_directory (const char *dir) { - mode_t m = umask (0); - int ret = mkdir (dir, 0777); + int ret = create_directory (dir, 0777, 0); if (ret < 0) { - log_warn (LOG_DEFAULT, "Failed to create directory '%s'. %s", dir, std::strerror (errno)); + log_warn (LOG_DEFAULT, "Failed to create public directory '%s'. %s", dir, std::strerror (errno)); } - umask (m); } int -Util::create_directory (const char *pathname, mode_t mode) +Util::create_directory (const char *pathname, mode_t mode, mode_t mask) { if (mode <= 0) mode = DEFAULT_DIRECTORY_MODE; @@ -164,7 +162,7 @@ Util::create_directory (const char *pathname, mode_t mode) errno = EINVAL; return -1; } - mode_t oldumask = umask (022); + mode_t oldumask = umask (mask); std::unique_ptr path {strdup_new (pathname)}; int rv, ret = 0; for (char *d = path.get (); d != nullptr && *d; ++d) { diff --git a/src/native/runtime-base/util.hh b/src/native/runtime-base/util.hh index 8dd7f432664..61292935bed 100644 --- a/src/native/runtime-base/util.hh +++ b/src/native/runtime-base/util.hh @@ -71,7 +71,7 @@ namespace xamarin::android static char *monodroid_strdup_vprintf (const char *format, va_list vargs); static char* path_combine (const char *path1, const char *path2); static void create_public_directory (const char *dir); - static int create_directory (const char *pathname, mode_t mode); + static int create_directory (const char *pathname, mode_t mode, mode_t mask = 022); static void set_world_accessable (const char *path); static void set_user_executable (const char *path); static bool file_exists (const char *file); From 17e8d3e0b32122235c4953f324ede98a7f570d60 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 11 Oct 2024 11:42:08 +0200 Subject: [PATCH 13/49] Fix DSO count when building Also, runtime config blob will no longer be packaged in the APK, stop checking whether it was found. --- .../Tasks/GeneratePackageManagerJava.cs | 15 +++++++++++++++ .../ApplicationConfigNativeAssemblyGenerator.cs | 3 ++- src/native/monodroid/embedded-assemblies.hh | 5 ++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index e5207abdeae..bee1418de9b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -306,6 +306,10 @@ void AddEnvironment () } var uniqueNativeLibraries = new List (); + + // Number of DSOs that will be packaged, it may be different to the number of items in the above + // `uniqueNativeLibraries` list. + uint packagedNativeLibrariesCount = 0; var seenNativeLibraryNames = new HashSet (StringComparer.OrdinalIgnoreCase); if (NativeLibraries != null) { foreach (ITaskItem item in NativeLibraries) { @@ -315,9 +319,19 @@ void AddEnvironment () continue; } + if (!ELFHelper.IsEmptyAOTLibrary (Log, item.ItemSpec)) { + packagedNativeLibrariesCount++; + } + seenNativeLibraryNames.Add (name); uniqueNativeLibraries.Add (item); } + + // libxamarin-app.so is not in NativeLibraries, but we must count it + if (!seenNativeLibraryNames.Contains ("libxamarin-app.so")) { + uniqueNativeLibraries.Add (new TaskItem ("libxamarin-app.so")); + packagedNativeLibrariesCount++; + } } ConcurrentDictionary? nativeCodeGenStates = null; @@ -361,6 +375,7 @@ void AddEnvironment () BundledAssemblyNameWidth = assemblyNameWidth, MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, + PackagedNativeLibrariesCount = packagedNativeLibrariesCount, HaveAssemblyStore = UseAssemblyStore, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, JNIEnvInitializeToken = jnienv_initialize_method_token, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 45e4ae7a4e0..0280d7fd1ea 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -186,6 +186,7 @@ sealed class XamarinAndroidBundledAssembly public MonoComponent MonoComponents { get; set; } public PackageNamingPolicy PackageNamingPolicy { get; set; } public List NativeLibraries { get; set; } + public uint PackagedNativeLibrariesCount { get; set; } public bool MarshalMethodsEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } public bool AssemblyStoreEmbeddedInRuntime { get; set; } @@ -236,7 +237,7 @@ protected override void Construct (LlvmIrModule module) environment_variable_count = (uint)(environmentVariables == null ? 0 : environmentVariables.Count * 2), system_property_count = (uint)(systemProperties == null ? 0 : systemProperties.Count * 2), number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk, - number_of_shared_libraries = (uint)NativeLibraries.Count, + number_of_shared_libraries = PackagedNativeLibrariesCount, bundled_assembly_name_width = (uint)BundledAssemblyNameWidth, number_of_dso_cache_entries = (uint)dsoCache.Count, number_of_aot_cache_entries = (uint)aotDsoCache.Count, diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index e7ddb6258c4..77f4f05fab7 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -331,8 +331,7 @@ namespace xamarin::android::internal { bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == number_of_assembly_store_files && number_of_zip_dso_entries >= application_config.number_of_shared_libraries - && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob); + number_of_mapped_assembly_stores == number_of_assembly_store_files && number_of_zip_dso_entries >= application_config.number_of_shared_libraries; } force_inline static c_unique_ptr to_utf8 (const MonoString *s) noexcept @@ -439,7 +438,7 @@ namespace xamarin::android::internal { md_mmap_info runtime_config_blob_mmap{}; void *runtime_config_data = nullptr; size_t runtime_config_data_size = 0uz; - bool runtime_config_blob_found = false; + bool runtime_config_blob_found = embedded_runtime_config_size > 0u; uint32_t number_of_mapped_assembly_stores = 0u; uint32_t number_of_zip_dso_entries = 0u; bool need_to_scan_more_apks = true; From fad0c8b0ae9b4cd53e46c63bb849cf62ae18de23 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 11 Oct 2024 12:14:24 +0200 Subject: [PATCH 14/49] Don't allocate memory for ZIP Central Directory data Instead use `mmap` --- .../monodroid/embedded-assemblies-zip.cc | 50 ++++++++++--------- src/native/monodroid/embedded-assemblies.hh | 9 ++-- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 08c6901ab41..3002e762a24 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -15,7 +15,7 @@ using namespace xamarin::android::internal; force_inline bool -EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::span const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept { entry_name.clear (); @@ -141,7 +141,7 @@ EmbeddedAssemblies::store_individual_assembly_data (dynamic_local_string const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_individual_assembly_entries (std::span const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept { // TODO: do away with all the string manipulation here. Replace it with generating xxhash for the entry name dynamic_local_string entry_name; @@ -255,7 +255,7 @@ EmbeddedAssemblies::map_assembly_store (dynamic_local_string } force_inline void -EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_assembly_store_entries (std::span const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { if (all_required_zip_entries_found ()) { return; @@ -326,26 +326,28 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus ) ); } -#ifdef DEBUG - log_info (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); - log_info (LOG_ASSEMBLY, "Central directory size: %u", cd_size); - log_info (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); -#endif - off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); - if (retval < 0) { - Helpers::abort_application ( - LOG_ASSEMBLY, - Util::monodroid_strdup_printf ( - "Failed to seek to central directory position in APK: %s. retval=%d errno=%d, File=%s", - std::strerror (errno), - retval, - errno, - apk_name - ) - ); - } - std::vector buf (cd_size); + md_mmap_info apk_map = md_mmap_apk_file (fd, cd_offset, cd_size, apk_name); + + log_debug (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); + log_debug (LOG_ASSEMBLY, "Central directory size: %u", cd_size); + log_debug (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); + + // off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); + // if (retval < 0) { + // Helpers::abort_application ( + // LOG_ASSEMBLY, + // Util::monodroid_strdup_printf ( + // "Failed to seek to central directory position in APK: %s. retval=%d errno=%d, File=%s", + // std::strerror (errno), + // retval, + // errno, + // apk_name + // ) + // ); + // } + + std::span buf (reinterpret_cast(apk_map.area), apk_map.size); const auto [prefix, prefix_len] = get_assemblies_prefix_and_length (); ZipEntryLoadState state { .file_fd = fd, @@ -381,6 +383,8 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } else { zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } + + // TODO: unmap here } template @@ -636,7 +640,7 @@ EmbeddedAssemblies::zip_read_field (T const& buf, size_t index, size_t count, dy } bool -EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state) +EmbeddedAssemblies::zip_read_entry_info (std::span const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state) { constexpr size_t CD_COMPRESSION_METHOD_OFFSET = 10uz; constexpr size_t CD_UNCOMPRESSED_SIZE_OFFSET = 24uz; diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 77f4f05fab7..4649e742d07 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -248,9 +249,9 @@ namespace xamarin::android::internal { void get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); - void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; - void zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; - bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; + void zip_load_individual_assembly_entries (std::span const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; + void zip_load_assembly_store_entries (std::span const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; + bool zip_load_entry_common (size_t entry_index, std::span const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); @@ -272,7 +273,7 @@ namespace xamarin::android::internal { template bool zip_read_field (T const& buf, size_t index, size_t count, dynamic_local_string& characters) const noexcept; - bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); + bool zip_read_entry_info (std::span const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); [[gnu::always_inline]] static std::tuple get_wrapper_dso_payload_pointer_and_size (md_mmap_info const& map_info, const char *file_name) noexcept From 2e2fde63686dd99dec7c9430815fd283a044efa0 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 11 Oct 2024 19:43:06 +0200 Subject: [PATCH 15/49] Experimenting with zip scanning performance --- .../monodroid/embedded-assemblies-zip.cc | 10 ++--- src/native/monodroid/embedded-assemblies.cc | 36 ------------------ src/native/monodroid/embedded-assemblies.hh | 37 ++++++++++++++++++- 3 files changed, 40 insertions(+), 43 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 3002e762a24..5bd7379992c 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -310,6 +310,7 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::span const& b } } +[[gnu::flatten]] void EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) { @@ -317,7 +318,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus uint32_t cd_size; uint16_t cd_entries; - if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) { + if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) [[unlikely]] { Helpers::abort_application ( LOG_ASSEMBLY, Util::monodroid_strdup_printf ( @@ -327,14 +328,12 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus ); } - md_mmap_info apk_map = md_mmap_apk_file (fd, cd_offset, cd_size, apk_name); - log_debug (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); log_debug (LOG_ASSEMBLY, "Central directory size: %u", cd_size); log_debug (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); // off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); - // if (retval < 0) { + // if (retval < 0) [[unlikely]] { // Helpers::abort_application ( // LOG_ASSEMBLY, // Util::monodroid_strdup_printf ( @@ -347,7 +346,6 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus // ); // } - std::span buf (reinterpret_cast(apk_map.area), apk_map.size); const auto [prefix, prefix_len] = get_assemblies_prefix_and_length (); ZipEntryLoadState state { .file_fd = fd, @@ -384,7 +382,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } - // TODO: unmap here + //delete[] raw_data; } template diff --git a/src/native/monodroid/embedded-assemblies.cc b/src/native/monodroid/embedded-assemblies.cc index 6a20df0bc59..8db5e4c0ee0 100644 --- a/src/native/monodroid/embedded-assemblies.cc +++ b/src/native/monodroid/embedded-assemblies.cc @@ -875,42 +875,6 @@ EmbeddedAssemblies::typemap_managed_to_java (MonoReflectionType *reflection_type return ret; } -EmbeddedAssemblies::md_mmap_info -EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) -{ - md_mmap_info file_info; - md_mmap_info mmap_info; - - size_t pageSize = static_cast(Util::monodroid_getpagesize ()); - size_t offsetFromPage = offset % pageSize; - size_t offsetPage = offset - offsetFromPage; - size_t offsetSize = size + offsetFromPage; - - mmap_info.area = mmap (nullptr, offsetSize, PROT_READ, MAP_PRIVATE, fd, static_cast(offsetPage)); - - if (mmap_info.area == MAP_FAILED) { - Helpers::abort_application ( - LOG_ASSEMBLY, - Util::monodroid_strdup_printf ( - "Could not mmap APK fd %d: %s; File=%s", - fd, - strerror (errno), - filename - ) - ); - } - - mmap_info.size = offsetSize; - file_info.area = (void*)((const char*)mmap_info.area + offsetFromPage); - file_info.size = size; - - log_info (LOG_ASSEMBLY, " mmap_start: %08p mmap_end: %08p mmap_len: % 12u file_start: %08p file_end: %08p file_len: % 12u apk descriptor: %d file: %s", - mmap_info.area, reinterpret_cast (mmap_info.area) + mmap_info.size, mmap_info.size, - file_info.area, reinterpret_cast (file_info.area) + file_info.size, file_info.size, fd, filename); - - return file_info; -} - void EmbeddedAssemblies::gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register) { diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 4649e742d07..97cfc22d6e5 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -27,6 +27,7 @@ #include "cppcompat.hh" #include "shared-constants.hh" #include "xxhash.hh" +#include "util.hh" #include @@ -239,7 +240,41 @@ namespace xamarin::android::internal { const TypeMapEntry *typemap_managed_to_java (const char *managed_type_name) noexcept; #endif // DEBUG - static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename); + [[gnu::always_inline]] + static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename, md_mmap_info &original_info, md_mmap_info &adjusted_info) noexcept + { + size_t pageSize = static_cast(Util::monodroid_getpagesize ()); + size_t offsetFromPage = offset % pageSize; + size_t offsetPage = offset - offsetFromPage; + size_t offsetSize = size + offsetFromPage; + + original_info.area = mmap (nullptr, offsetSize, PROT_READ, MAP_PRIVATE, fd, static_cast(offsetPage)); + + if (original_info.area == MAP_FAILED) { + log_fatal (LOG_DEFAULT, "Could not `mmap` apk fd %d entry `%s`: %s", fd, filename, strerror (errno)); + Helpers::abort_application (); + } + + original_info.size = offsetSize; + adjusted_info.area = (void*)((const char*)original_info.area + offsetFromPage); + adjusted_info.size = size; + + log_info (LOG_ASSEMBLY, " mmap_start: %08p mmap_end: %08p mmap_len: % 12u file_start: %08p file_end: %08p file_len: % 12u apk descriptor: %d file: %s", + original_info.area, reinterpret_cast (original_info.area) + original_info.size, original_info.size, + adjusted_info.area, reinterpret_cast (adjusted_info.area) + adjusted_info.size, adjusted_info.size, fd, filename); + + return adjusted_info; + } + + [[gnu::flatten, gnu::always_inline]] + static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) noexcept + { + md_mmap_info file_info; + md_mmap_info mmap_info; + + return md_mmap_apk_file (fd, offset, size, filename, mmap_info, file_info); + } + static MonoAssembly* open_from_bundles_full (MonoAssemblyName *aname, char **assemblies_path, void *user_data); static MonoAssembly* open_from_bundles (MonoAssemblyLoadContextGCHandle alc_gchandle, MonoAssemblyName *aname, char **assemblies_path, void *user_data, MonoError *error); From 88dc31e6a36198583c88e7b1a33b4cb9b979170d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 15 Oct 2024 12:16:50 +0200 Subject: [PATCH 16/49] Post rebase fixups --- src/native/monodroid/embedded-assemblies.hh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 97cfc22d6e5..4dc86ec4052 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -251,17 +251,25 @@ namespace xamarin::android::internal { original_info.area = mmap (nullptr, offsetSize, PROT_READ, MAP_PRIVATE, fd, static_cast(offsetPage)); if (original_info.area == MAP_FAILED) { - log_fatal (LOG_DEFAULT, "Could not `mmap` apk fd %d entry `%s`: %s", fd, filename, strerror (errno)); - Helpers::abort_application (); + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Could not mmap APK fd %d: %s; File=%s", + fd, + strerror (errno), + filename + ) + ); } original_info.size = offsetSize; adjusted_info.area = (void*)((const char*)original_info.area + offsetFromPage); adjusted_info.size = size; - log_info (LOG_ASSEMBLY, " mmap_start: %08p mmap_end: %08p mmap_len: % 12u file_start: %08p file_end: %08p file_len: % 12u apk descriptor: %d file: %s", + log_info (LOG_ASSEMBLY, "mmap_start: %08p mmap_end: %08p mmap_len: % 12u file_start: %08p file_end: %08p file_len: % 12u; apk descriptor: %d; file: %s", original_info.area, reinterpret_cast (original_info.area) + original_info.size, original_info.size, - adjusted_info.area, reinterpret_cast (adjusted_info.area) + adjusted_info.size, adjusted_info.size, fd, filename); + adjusted_info.area, reinterpret_cast (adjusted_info.area) + adjusted_info.size, adjusted_info.size, fd, filename + ); return adjusted_info; } From 492fb28e74dfe70bd7cf11fd9a7558af04ca6918 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 15 Oct 2024 12:31:45 +0200 Subject: [PATCH 17/49] More post rebase fixups --- src/native/monodroid/embedded-assemblies-zip.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 5bd7379992c..dbed3a73a5c 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -184,7 +184,7 @@ EmbeddedAssemblies::verify_assembly_store_and_set_info (void *data_start, const LOG_ASSEMBLY, Util::monodroid_strdup_printf ( "Assembly store '%s' is not a valid .NET for Android assembly store file", - entry_name.get () + name ) ); } @@ -194,7 +194,7 @@ EmbeddedAssemblies::verify_assembly_store_and_set_info (void *data_start, const LOG_ASSEMBLY, Util::monodroid_strdup_printf ( "Assembly store '%s' uses format version 0x%x, instead of the expected 0x%x", - entry_name.get (), + name, header->version, ASSEMBLY_STORE_FORMAT_VERSION ) From 4848efb7efcb192c46f7e45da1e776929aabbb58 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 15 Oct 2024 16:29:28 +0200 Subject: [PATCH 18/49] seek and ye shall read --- .../monodroid/embedded-assemblies-zip.cc | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index dbed3a73a5c..26cc878d1a0 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -332,19 +332,19 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus log_debug (LOG_ASSEMBLY, "Central directory size: %u", cd_size); log_debug (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); - // off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); - // if (retval < 0) [[unlikely]] { - // Helpers::abort_application ( - // LOG_ASSEMBLY, - // Util::monodroid_strdup_printf ( - // "Failed to seek to central directory position in APK: %s. retval=%d errno=%d, File=%s", - // std::strerror (errno), - // retval, - // errno, - // apk_name - // ) - // ); - // } + off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); + if (retval < 0) [[unlikely]] { + Helpers::abort_application ( + LOG_ASSEMBLY, + Util::monodroid_strdup_printf ( + "Failed to seek to central directory position in APK: %s. retval=%d errno=%d, File=%s", + std::strerror (errno), + retval, + errno, + apk_name + ) + ); + } const auto [prefix, prefix_len] = get_assemblies_prefix_and_length (); ZipEntryLoadState state { From b1a3863060ab47f460e20cebd905a60a74c5bd1e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Oct 2024 14:05:16 +0200 Subject: [PATCH 19/49] Details, details... --- .../Xamarin.Android.Common.targets | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 03de60fd0d4..02e4693b81a 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2118,8 +2118,8 @@ because xbuild doesn't support framework reference assemblies. ZipAlignmentPages="$(AndroidZipAlignment)" UseAssemblyStore="$(AndroidUseAssemblyStore)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - IntermediateOutputPath="$(IntermediateOutputPath)"> - CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + IntermediateOutputPath="$(IntermediateOutputPath)" + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)" AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> @@ -2160,8 +2160,8 @@ because xbuild doesn't support framework reference assemblies. ZipAlignmentPages="$(AndroidZipAlignment)" UseAssemblyStore="$(AndroidUseAssemblyStore)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - IntermediateOutputPath="$(IntermediateOutputPath)"> - CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + IntermediateOutputPath="$(IntermediateOutputPath)" + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)" AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> @@ -2235,8 +2235,8 @@ because xbuild doesn't support framework reference assemblies. ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - IntermediateOutputPath="$(IntermediateOutputPath)"> - CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + IntermediateOutputPath="$(IntermediateOutputPath)" + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)" AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> @@ -2276,8 +2276,8 @@ because xbuild doesn't support framework reference assemblies. ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - IntermediateOutputPath="$(IntermediateOutputPath)"> - CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)"> + IntermediateOutputPath="$(IntermediateOutputPath)" + CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)" AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"> From 2941030fd7aa4d3b281b7a1839700079a3382834 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Oct 2024 17:38:54 +0200 Subject: [PATCH 20/49] Cleanup --- .../Tasks/BuildApk.cs | 21 ------------------- .../Xamarin.Android.Common.targets | 7 ++----- .../monodroid/embedded-assemblies-zip.cc | 11 ---------- src/native/monodroid/embedded-assemblies.hh | 5 ----- src/native/monodroid/monodroid-glue.cc | 7 ------- 5 files changed, 2 insertions(+), 49 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index d387c06ea80..0e92abe4c0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -99,8 +99,6 @@ public class BuildApk : AndroidTask public string CheckedBuild { get; set; } - public string RuntimeConfigBinFilePath { get; set; } - public bool UseAssemblyStore { get; set; } public string ZipFlushFilesLimit { get; set; } @@ -224,7 +222,6 @@ void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supp apk.Flush (); } -// AddRuntimeConfigBlob (apk); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); AddNativeLibraries (files, supportedAbis); @@ -409,27 +406,9 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) return new Regex (sb.ToString (), options); } - void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk) - { - // We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter - // our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99% - // of cases these days, since AAB enforces that split). `base.apk` contains only ABI-agnostic file, while one of the split config files contains only - // ABI-specific data+code. - if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { - foreach (string abi in SupportedAbis) { - // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we - // like since we can finish scanning the zip central directory earlier at startup. - string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so"); - string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath)); - AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath)); - } - } - } - void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { string sourcePath; - AssemblyCompression.AssemblyData compressedAssembly = null; AssemblyStoreBuilder? storeBuilder = null; if (UseAssemblyStore) { diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 02e4693b81a..4fbefbffb6f 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -292,6 +292,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidFastDeployEnvironmentFiles Condition=" '$(_AndroidFastDeployEnvironmentFiles)' == '' And '$(EmbedAssembliesIntoApk)' == 'False' ">True <_AndroidFastDeployEnvironmentFiles Condition=" '$(_AndroidFastDeployEnvironmentFiles)' == '' ">False + <_AndroidCompressedAssembliesDir>$(IntermediateOutputPath)android\lz4 @@ -1871,7 +1872,7 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; - _RemoveRegisterAttribute; + _RemoveRegisterAttribute; _CreateApplicationSharedLibraries; _GetMonoPlatformJarPath; _GetLibraryImports; @@ -2110,7 +2111,6 @@ because xbuild doesn't support framework reference assemblies. ProjectFullPath="$(MSBuildProjectFullPath)" IncludeWrapSh="$(AndroidIncludeWrapSh)" CheckedBuild="$(_AndroidCheckedBuild)" - RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" IncludeFiles="@(AndroidPackagingOptionsInclude)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" @@ -2152,7 +2152,6 @@ because xbuild doesn't support framework reference assemblies. ProjectFullPath="$(MSBuildProjectFullPath)" IncludeWrapSh="$(AndroidIncludeWrapSh)" CheckedBuild="$(_AndroidCheckedBuild)" - RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" IncludeFiles="@(AndroidPackagingOptionsInclude)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" @@ -2230,7 +2229,6 @@ because xbuild doesn't support framework reference assemblies. ProjectFullPath="$(MSBuildProjectFullPath)" IncludeWrapSh="$(AndroidIncludeWrapSh)" CheckedBuild="$(_AndroidCheckedBuild)" - RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" @@ -2271,7 +2269,6 @@ because xbuild doesn't support framework reference assemblies. ProjectFullPath="$(MSBuildProjectFullPath)" IncludeWrapSh="$(AndroidIncludeWrapSh)" CheckedBuild="$(_AndroidCheckedBuild)" - RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" ExcludeFiles="@(AndroidPackagingOptionsExclude)" diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 26cc878d1a0..d2888cd97f0 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -61,14 +61,6 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::span const& b dynamic_local_string entry_name; bool assembly_store_found = embedded_assembly_store_size != 0; if (assembly_store_found) { - log_debug (LOG_ASSEMBLY, "Got embedded assembly store, size %zu", embedded_assembly_store_size); verify_assembly_store_and_set_info (embedded_assembly_store, "embedded"); log_debug (LOG_ASSEMBLY, "Looking for DSOs in APK"); } else { @@ -381,8 +372,6 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } else { zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } - - //delete[] raw_data; } template diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 4dc86ec4052..4e67985c3da 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -175,11 +175,6 @@ namespace xamarin::android::internal { runtime_config_blob_mmap.size = 0; } - bool have_runtime_config_blob () const noexcept - { - return application_config.have_runtime_config_blob && runtime_config_blob_mmap.area != nullptr; - } - bool keep_scanning () const noexcept { return need_to_scan_more_apks; diff --git a/src/native/monodroid/monodroid-glue.cc b/src/native/monodroid/monodroid-glue.cc index c02ab5796dc..30c83c81642 100644 --- a/src/native/monodroid/monodroid-glue.cc +++ b/src/native/monodroid/monodroid-glue.cc @@ -712,7 +712,6 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); - // if (embeddedAssemblies.have_runtime_config_blob ()) { if (embedded_runtime_config_size > 0) { size_t blob_time_index; if (FastTiming::enabled ()) [[unlikely]] { @@ -720,7 +719,6 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks } runtime_config_args.kind = 1; -// embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); runtime_config_args.runtimeconfig.data.data = reinterpret_cast(embedded_runtime_config); runtime_config_args.runtimeconfig.data.data_len = static_cast(embedded_runtime_config_size); monovm_runtimeconfig_initialize (&runtime_config_args, cleanup_runtime_config, nullptr); @@ -1380,11 +1378,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl Logger::init_logging_categories (mono_log_mask_raw, mono_log_level_raw); - // log_warn (LOG_DEFAULT, "Embedded runtime config size: %zu", embedded_runtime_config_size); - // if (embedded_runtime_config_size > 0) { - // log_warn (LOG_DEFAULT, "First byte of embedded runtime config: 0x%x", embedded_runtime_config[0]); - // } - std::unique_ptr mono_log_mask (mono_log_mask_raw); std::unique_ptr mono_log_level (mono_log_level_raw); From d69c5d6fbbb23fcaec84a66809d80155266e1d08 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Oct 2024 17:39:06 +0200 Subject: [PATCH 21/49] Teach assembly store explorer about embedded stores --- .../Utilities/ELFHelper.Basic.cs | 61 ++++++++++++++++ .../Utilities/ELFHelper.cs | 26 +------ .../AssemblyStore/ELFPayloadError.cs | 1 + .../AssemblyStore/StoreReader_V2.cs | 47 ++++++++---- .../AssemblyStore/Utils.cs | 73 +++++++++++++++---- .../assembly-store-reader.csproj | 1 + 6 files changed, 155 insertions(+), 54 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.Basic.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.Basic.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.Basic.cs new file mode 100644 index 00000000000..df401b2f462 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.Basic.cs @@ -0,0 +1,61 @@ +using System; + +using ELFSharp.ELF; +using ELFSharp.ELF.Sections; + +namespace Xamarin.Android.Tasks; + +static partial class ELFHelper +{ + public static ISymbolTable? GetSymbolTable (IELF elf, string sectionName) + { + ISection? section = GetSection (elf, sectionName); + if (section == null) { + return null; + } + + var symtab = section as ISymbolTable; + if (symtab == null) { + return null; + } + + return symtab; + } + + public static ISection? GetSection (IELF elf, string sectionName) + { + if (!elf.TryGetSection (sectionName, out ISection section)) { + return null; + } + + return section; + } + + public static SymbolEntry? FindSymbol (ISymbolTable? symbolTable, string symbolName) where T: struct + { + if (symbolTable == null) { + return null; + } + + ISymbolEntry? symbol = null; + foreach (ISymbolEntry entry in symbolTable.Entries) { + if (String.Compare (entry.Name, symbolName, StringComparison.Ordinal) != 0) { + continue; + } + + symbol = entry; + break; + } + + if (symbol == null) { + return null; + } + + Type t = typeof(T); + if (t == typeof(ulong) || t == typeof(uint)) { + return (SymbolEntry)symbol; + } + + throw new InvalidOperationException ($"Only `ulong` and `uint` types are accepted"); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs index 9bee404bb7c..2100bfbb1c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs @@ -16,7 +16,7 @@ namespace Xamarin.Android.Tasks { - static class ELFHelper + static partial class ELFHelper { public static void AssertValidLibraryAlignment (TaskLoggingHelper log, int alignmentInPages, string path, ITaskItem? item) { @@ -218,29 +218,5 @@ bool IsNonEmptyCodeSymbol (SymbolEntry? symbolEntry) where T : struct return size != 0 && symbolEntry.PointedSection.Type == ELFSectionType.ProgBits; } } - - static ISymbolTable? GetSymbolTable (IELF elf, string sectionName) - { - ISection? section = GetSection (elf, sectionName); - if (section == null) { - return null; - } - - var symtab = section as ISymbolTable; - if (symtab == null) { - return null; - } - - return symtab; - } - - static ISection? GetSection (IELF elf, string sectionName) - { - if (!elf.TryGetSection (sectionName, out ISection section)) { - return null; - } - - return section; - } } } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs b/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs index 932f151c80f..78372c26853 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/ELFPayloadError.cs @@ -8,4 +8,5 @@ enum ELFPayloadError NotSharedLibrary, NotLittleEndian, NoPayloadSection, + NoEmbeddedStore, } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index a43960102e2..1fc93530a66 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -86,6 +86,19 @@ public StoreReader_V2 (Stream store, string path) protected override ulong GetStoreStartDataOffset () => elfOffset; + string PayloadErrorToString (ELFPayloadError error) + { + return error switch { + ELFPayloadError.NotELF => $"Store '{StorePath}' is not a valid ELF binary", + ELFPayloadError.LoadFailed => $"Store '{StorePath}' could not be loaded", + ELFPayloadError.NotSharedLibrary => $"Store '{StorePath}' is not a shared ELF library", + ELFPayloadError.NotLittleEndian => $"Store '{StorePath}' is not a little-endian ELF image", + ELFPayloadError.NoPayloadSection => $"Store '{StorePath}' does not contain the 'payload' section", + ELFPayloadError.NoEmbeddedStore => $"Store '{StorePath}' does not contain embedded data blob", + _ => $"Unknown ELF payload section error for store '{StorePath}': {error}" + }; + } + protected override bool IsSupported () { StoreStream.Seek (0, SeekOrigin.Begin); @@ -94,22 +107,19 @@ protected override bool IsSupported () uint magic = reader.ReadUInt32 (); if (magic == Utils.ELF_MAGIC) { ELFPayloadError error; - (elfOffset, _, error) = Utils.FindELFPayloadSectionOffsetAndSize (StoreStream); - + (elfOffset, _, error) = Utils.FindEmbeddedStoreOffsetAndSize (StoreStream); if (error != ELFPayloadError.None) { - string message = error switch { - ELFPayloadError.NotELF => $"Store '{StorePath}' is not a valid ELF binary", - ELFPayloadError.LoadFailed => $"Store '{StorePath}' could not be loaded", - ELFPayloadError.NotSharedLibrary => $"Store '{StorePath}' is not a shared ELF library", - ELFPayloadError.NotLittleEndian => $"Store '{StorePath}' is not a little-endian ELF image", - ELFPayloadError.NoPayloadSection => $"Store '{StorePath}' does not contain the 'payload' section", - _ => $"Unknown ELF payload section error for store '{StorePath}': {error}" - }; - Log.Debug (message); - } else if (elfOffset >= 0) { - StoreStream.Seek ((long)elfOffset, SeekOrigin.Begin); - magic = reader.ReadUInt32 (); + MaybeLogError (error); + + (elfOffset, _, error) = Utils.FindELFPayloadSectionOffsetAndSize (StoreStream); + if (error != ELFPayloadError.None) { + MaybeLogError (error); + return false; + } } + + StoreStream.Seek ((long)elfOffset, SeekOrigin.Begin); + magic = reader.ReadUInt32 (); } if (magic != Utils.ASSEMBLY_STORE_MAGIC) { @@ -129,6 +139,15 @@ protected override bool IsSupported () header = new Header (magic, version, entry_count, index_entry_count, index_size); return true; + + void MaybeLogError (ELFPayloadError error) + { + if (error == ELFPayloadError.None) { + return; + } + + Log.Debug (PayloadErrorToString (error)); + } } protected override void Prepare () diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs index 284f501c9ac..2e2f03e5e52 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs @@ -4,6 +4,7 @@ using ELFSharp.ELF; using ELFSharp.ELF.Sections; +using Xamarin.Android.Tasks; using Xamarin.Tools.Zip; namespace Xamarin.Android.AssemblyStore; @@ -29,31 +30,28 @@ static class Utils public static readonly ArrayPool BytePool = ArrayPool.Shared; - public static (ulong offset, ulong size, ELFPayloadError error) FindELFPayloadSectionOffsetAndSize (Stream stream) + static (bool is64, IELF? elf, ELFPayloadError error) TryOpenELF (Stream stream) { + bool is64 = false; stream.Seek (0, SeekOrigin.Begin); Class elfClass = ELFReader.CheckELFType (stream); if (elfClass == Class.NotELF) { - return ReturnError (null, ELFPayloadError.NotELF); + return ReturnError (is64, null, ELFPayloadError.NotELF); } if (!ELFReader.TryLoad (stream, shouldOwnStream: false, out IELF? elf)) { - return ReturnError (elf, ELFPayloadError.LoadFailed); + return ReturnError (is64, elf, ELFPayloadError.LoadFailed); } if (elf.Type != FileType.SharedObject) { - return ReturnError (elf, ELFPayloadError.NotSharedLibrary); + return ReturnError (is64, elf, ELFPayloadError.NotSharedLibrary); } if (elf.Endianess != ELFSharp.Endianess.LittleEndian) { - return ReturnError (elf, ELFPayloadError.NotLittleEndian); - } - - if (!elf.TryGetSection ("payload", out ISection? payloadSection)) { - return ReturnError (elf, ELFPayloadError.NoPayloadSection); + return ReturnError (is64, elf, ELFPayloadError.NotLittleEndian); } - bool is64 = elf.Machine switch { + is64 = elf.Machine switch { Machine.ARM => false, Machine.Intel386 => false, @@ -63,6 +61,51 @@ public static (ulong offset, ulong size, ELFPayloadError error) FindELFPayloadSe _ => throw new NotSupportedException ($"Unsupported ELF architecture '{elf.Machine}'") }; + return (is64, elf, ELFPayloadError.None); + + (bool is64, IELF? elf, ELFPayloadError error) ReturnError (bool is64, IELF? elf, ELFPayloadError error) + { + elf?.Dispose (); + + return (is64, null, error); + } + } + + public static (ulong offset, ulong size, ELFPayloadError error) FindEmbeddedStoreOffsetAndSize (Stream stream) + { + (bool is64, IELF? elf, ELFPayloadError error) = TryOpenELF (stream); + if (elf == null || error != ELFPayloadError.None) { + return ReturnError (elf, error); + } + + const string SymbolName = "embedded_assembly_store"; + ISymbolTable? dynsym = ELFHelper.GetSymbolTable (elf, ".dynsym"); + if (is64) { + SymbolEntry? symbol = ELFHelper.FindSymbol (dynsym, SymbolName); + if (symbol != null) { + return (symbol.Value, symbol.Size, ELFPayloadError.None); + } + } else { + SymbolEntry? symbol = ELFHelper.FindSymbol (dynsym, SymbolName); + if (symbol != null) { + return ((ulong)symbol.Value, (ulong)symbol.Size, ELFPayloadError.None); + } + } + + return (0, 0, ELFPayloadError.NoEmbeddedStore); + } + + public static (ulong offset, ulong size, ELFPayloadError error) FindELFPayloadSectionOffsetAndSize (Stream stream) + { + (bool is64, IELF? elf, ELFPayloadError error) = TryOpenELF (stream); + if (elf == null || error != ELFPayloadError.None) { + return ReturnError (elf, error); + } + + if (!elf.TryGetSection ("payload", out ISection? payloadSection)) { + return ReturnError (elf, ELFPayloadError.NoPayloadSection); + } + ulong offset; ulong size; @@ -84,13 +127,13 @@ public static (ulong offset, ulong size, ELFPayloadError error) FindELFPayloadSe { return ((ulong)payload.Offset, (ulong)payload.Size); } + } - (ulong offset, ulong size, ELFPayloadError error) ReturnError (IELF? elf, ELFPayloadError error) - { - elf?.Dispose (); + static (ulong offset, ulong size, ELFPayloadError error) ReturnError (IELF? elf, ELFPayloadError error) + { + elf?.Dispose (); - return (0, 0, error); - } + return (0, 0, error); } public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index ec4c90fecb1..3bd2852d04e 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -24,6 +24,7 @@ + From bde06acdf62557eb383b4005107b648de226a5a6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Oct 2024 17:47:22 +0200 Subject: [PATCH 22/49] Teach store v2 reader to look in `libxamarin-app.so`, too --- .../AssemblyStore/StoreReader_V2.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index 1fc93530a66..1ba0cb8b488 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -40,6 +40,11 @@ static StoreReader_V2 () GetArchPath (AndroidTargetArch.Arm), GetArchPath (AndroidTargetArch.X86_64), GetArchPath (AndroidTargetArch.X86), + + GetArchPath (AndroidTargetArch.Arm64, embeddedBlob: true), + GetArchPath (AndroidTargetArch.Arm, embeddedBlob: true), + GetArchPath (AndroidTargetArch.X86_64, embeddedBlob: true), + GetArchPath (AndroidTargetArch.X86, embeddedBlob: true), }; ApkPaths = paths.AsReadOnly (); AabBasePaths = ApkPaths; @@ -50,10 +55,15 @@ static StoreReader_V2 () GetArchPath (AndroidTargetArch.Arm, AabBaseDir), GetArchPath (AndroidTargetArch.X86_64, AabBaseDir), GetArchPath (AndroidTargetArch.X86, AabBaseDir), + + GetArchPath (AndroidTargetArch.Arm64, AabBaseDir, embeddedBlob: true), + GetArchPath (AndroidTargetArch.Arm, AabBaseDir, embeddedBlob: true), + GetArchPath (AndroidTargetArch.X86_64, AabBaseDir, embeddedBlob: true), + GetArchPath (AndroidTargetArch.X86, AabBaseDir, embeddedBlob: true), }; AabPaths = paths.AsReadOnly (); - string GetArchPath (AndroidTargetArch arch, string? root = null) + string GetArchPath (AndroidTargetArch arch, string? root = null, bool embeddedBlob = false) { const string LibDirName = "lib"; @@ -65,7 +75,9 @@ string GetArchPath (AndroidTargetArch arch, string? root = null) root = LibDirName; } parts.Add (abi); - parts.Add (GetBlobName (abi)); + parts.Add ( + embeddedBlob ? "libxamarin-app.so" : GetBlobName (abi) + ); return MonoAndroidHelper.MakeZipArchivePath (root, parts); } From 94a09e38446eed2c61dba3e686082d7fa1aec3f7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 16 Oct 2024 17:48:55 +0200 Subject: [PATCH 23/49] Nicer --- .../AssemblyStore/StoreReader_V2.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index 1ba0cb8b488..fbccf1bdb94 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -75,9 +75,7 @@ string GetArchPath (AndroidTargetArch arch, string? root = null, bool embeddedBl root = LibDirName; } parts.Add (abi); - parts.Add ( - embeddedBlob ? "libxamarin-app.so" : GetBlobName (abi) - ); + parts.Add (GetBlobName (abi, embeddedBlob)); return MonoAndroidHelper.MakeZipArchivePath (root, parts); } @@ -94,7 +92,7 @@ public StoreReader_V2 (Stream store, string path) }; } - static string GetBlobName (string abi) => $"libassemblies.{abi}.blob.so"; + static string GetBlobName (string abi, bool embeddedBlob) => embeddedBlob ? "libxamarin-app.so" : $"libassemblies.{abi}.blob.so"; protected override ulong GetStoreStartDataOffset () => elfOffset; From 4607b6f9561379688a98c657deabd83a85a366b1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 17 Oct 2024 10:02:47 +0200 Subject: [PATCH 24/49] The runtime configuration blob is now part of `libxamarin-app.so` --- .../Tests/Xamarin.Android.Build.Tests/PackagingTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index f9394a98ca1..ee29af8429f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -129,7 +129,6 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto "System.Collections.dll", "System.Collections.Concurrent.dll", "System.Text.RegularExpressions.dll", - "libarc.bin.so", }; using (var b = CreateApkBuilder ()) { From 872f66293f0cc4f98e8bca15b89683e4733abe5e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 17 Oct 2024 11:14:14 +0200 Subject: [PATCH 25/49] Optionally produce "empty" embed.*.s files Even though we might not have any content to embed, we need the symbols to be present, because they're part of the `libxamarin-app.so` ABI expected by `libmonodroid.so` --- .../Tasks/CreateEmbeddedAssemblyStore.cs | 3 +- .../Tasks/GeneratePackageManagerJava.cs | 29 +++++----- .../Utilities/ELFEmbeddingHelper.cs | 58 ++++++++++++++----- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs index 26c5394ff9e..288f52be3db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -84,7 +84,8 @@ public override bool RunTask () AndroidBinUtilsDirectory, inputFile, ELFEmbeddingHelper.KnownEmbedItems.AssemblyStore, - AssemblySourcesDir + AssemblySourcesDir, + missingContentOK: false ); if (items.Count == 0) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index bee1418de9b..01caab086ac 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -61,7 +61,9 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } - [Required] + // This property should be required but it can't, because during design time builds there's no + // value to pass and MSBuild signals an error that a required property wasn't given a value. + //[Required] public string AndroidBinUtilsDirectory { get; set; } [Required] @@ -343,20 +345,17 @@ void AddEnvironment () } bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); - if (haveRuntimeConfigBlob) { - List objectFilePaths = ELFEmbeddingHelper.EmbedBinary ( - Log, - SupportedAbis, - AndroidBinUtilsDirectory, - RuntimeConfigBinFilePath, - ELFEmbeddingHelper.KnownEmbedItems.RuntimeConfig, - EnvironmentOutputDirectory - ); - - EmbeddedObjectFiles = objectFilePaths.ToArray (); - } else { - EmbeddedObjectFiles = Array.Empty (); - } + List objectFilePaths = ELFEmbeddingHelper.EmbedBinary ( + Log, + SupportedAbis, + AndroidBinUtilsDirectory, + RuntimeConfigBinFilePath, + ELFEmbeddingHelper.KnownEmbedItems.RuntimeConfig, + EnvironmentOutputDirectory, + missingContentOK: !haveRuntimeConfigBlob + ); + + EmbeddedObjectFiles = objectFilePaths.ToArray (); var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 56f399913f2..9116cabcd58 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -62,9 +62,10 @@ public static List EmbedBinary ( TaskLoggingHelper log, ICollection supportedAbis, string androidBinUtilsDirectory, - string inputFile, + string? inputFile, EmbedItem embedItem, - string outputDirectory) + string outputDirectory, + bool missingContentOK) { if (supportedAbis.Count < 1) { throw new ArgumentException ("At least one target ABI must be present", nameof (supportedAbis)); @@ -80,7 +81,8 @@ public static List EmbedBinary ( abi, inputFile, outputDirectory, - embedItem + embedItem, + missingContentOK ); } @@ -91,9 +93,10 @@ public static List EmbedBinary ( TaskLoggingHelper log, string abi, string androidBinUtilsDirectory, - string inputFile, + string? inputFile, EmbedItem embedItem, - string outputDirectory) + string outputDirectory, + bool missingContentOK) { if (String.IsNullOrEmpty (abi)) { throw new ArgumentException ("Must be a supported ABI name", nameof (abi)); @@ -107,7 +110,8 @@ public static List EmbedBinary ( abi, inputFile, outputDirectory, - embedItem + embedItem, + missingContentOK ); return ret; } @@ -119,10 +123,11 @@ static void EmbedBinary ( string abi, string inputFile, string outputDirectory, - EmbedItem embedItem) + EmbedItem embedItem, + bool missingContentOK) { string outputFile = Path.Combine (outputDirectory, $"embed_{embedItem.BaseFileName}.{abi.ToLowerInvariant ()}.o"); - DoEmbed (log, MonoAndroidHelper.AbiToTargetArch (abi), llvmMcPath, inputFile, outputFile, embedItem); + DoEmbed (log, MonoAndroidHelper.AbiToTargetArch (abi), llvmMcPath, inputFile, outputFile, embedItem, missingContentOK); if (!File.Exists (outputFile)) { return; } @@ -137,33 +142,54 @@ static void DoEmbed ( TaskLoggingHelper log, AndroidTargetArch arch, string llvmMcPath, - string inputFile, + string? inputFile, string outputFile, - EmbedItem item) + EmbedItem item, + bool missingContentOK) { if (!llvmMcConfigs.TryGetValue (arch, out LlvmMcTargetConfig cfg)) { throw new NotSupportedException ($"Internal error: unsupported target arch '{arch}'"); } - inputFile = Path.GetFullPath (inputFile); + bool haveInputFile = !String.IsNullOrEmpty (inputFile); + if (!haveInputFile) { + if (!missingContentOK) { + throw new InvalidOperationException ("Internal error: input file must be specified"); + } + } else { + inputFile = Path.GetFullPath (inputFile); + } outputFile = Path.GetFullPath (outputFile); - var fi = new FileInfo (inputFile); - long inputFileSize = fi.Length; + long inputFileSize = 0; + string? sanitizedInputFilePath = null; + + if (haveInputFile) { + var fi = new FileInfo (inputFile); + if (fi.Exists) { + inputFileSize = fi.Length; + sanitizedInputFilePath = inputFile.Replace ("\\", "\\\\"); + } else if (!missingContentOK) { + throw new InvalidOperationException ($"Internal error: input file '{inputFile}' does not exist"); + } + } + string asmInputFile = Path.ChangeExtension (outputFile, ".s"); - string sanitizedInputFilePath = inputFile.Replace ("\\", "\\\\"); using var fs = File.Open (asmInputFile, FileMode.Create, FileAccess.Write, FileShare.Read); using var sw = new StreamWriter (fs, asmFileEncoding); string symbolName = item.SymbolName; sw.WriteLine ($".section .rodata,\"a\",{cfg.AssemblerDirectivePrefix}progbits"); - sw.WriteLine (".p2align 3, 0x00"); // Put the data at 4k boundary + sw.WriteLine (".p2align 3, 0x00"); // Put the data at the 4k boundary sw.WriteLine (); sw.WriteLine ($".global {symbolName}"); sw.WriteLine ($".type {symbolName},{cfg.AssemblerDirectivePrefix}object"); sw.WriteLine ($"{symbolName}:"); - sw.WriteLine ($"\t.incbin \"{sanitizedInputFilePath}\""); + + if (!String.IsNullOrEmpty (sanitizedInputFilePath)) { + sw.WriteLine ($"\t.incbin \"{sanitizedInputFilePath}\""); + } sw.WriteLine ($"\t.size {symbolName}, {inputFileSize}"); sw.WriteLine (); From 26bf693904384558f8422aa7c4dcdb5ef4f72e7e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 17 Oct 2024 11:31:59 +0200 Subject: [PATCH 26/49] Don't embed assembly store when fastdev is used --- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 4fbefbffb6f..68901ab617e 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -328,7 +328,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) - <_AndroidEmbedAssemblyStoreInRuntime Condition=" '$(AndroidUseAssemblyStore)' == 'True' And '$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">True + <_AndroidEmbedAssemblyStoreInRuntime Condition=" '$(AndroidUseAssemblyStore)' == 'True' And '$(EmbedAssembliesIntoApk)' == 'true' And '$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">True <_AndroidEmbedAssemblyStoreInRuntime Condition="'$(_AndroidEmbedAssemblyStoreInRuntime)' == '' ">False From ea9d5b2c60f2f78a8ea57bb2f16a18dbcfccc78b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 17 Oct 2024 13:24:11 +0200 Subject: [PATCH 27/49] Let's see if designer tests pass now --- .../Tasks/GeneratePackageManagerJava.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 01caab086ac..47151a70604 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -61,12 +61,11 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } - // This property should be required but it can't, because during design time builds there's no - // value to pass and MSBuild signals an error that a required property wasn't given a value. + // These two properties should be required but they will require modifying `monodroid` first //[Required] public string AndroidBinUtilsDirectory { get; set; } - [Required] + //[Required] public bool AssemblyStoreEmbeddedInRuntime { get; set; } [Output] From 6aebe30db72bda36f26f55442ab53cdec5bf74ac Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 18 Oct 2024 12:40:36 +0200 Subject: [PATCH 28/49] Expect the unexpected, DTB doesn't specify any ABIs --- .../Utilities/ELFEmbeddingHelper.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 9116cabcd58..e58f2b94d5e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -3,6 +3,7 @@ using System.IO; using System.Text; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Android.Tools; @@ -67,12 +68,13 @@ public static List EmbedBinary ( string outputDirectory, bool missingContentOK) { + var ret = new List (); if (supportedAbis.Count < 1) { - throw new ArgumentException ("At least one target ABI must be present", nameof (supportedAbis)); + log.LogDebugMessage ("ELFEmbeddingHelper: at least one target ABI must be specified. Probably a DTB build, skipping generation."); + return ret; } string llvmMcPath = GetLlvmMcPath (androidBinUtilsDirectory); - var ret = new List (); foreach (string abi in supportedAbis) { EmbedBinary ( log, @@ -98,11 +100,12 @@ public static List EmbedBinary ( string outputDirectory, bool missingContentOK) { + var ret = new List (); if (String.IsNullOrEmpty (abi)) { - throw new ArgumentException ("Must be a supported ABI name", nameof (abi)); + log.LogDebugMessage ("ELFEmbeddingHelper: ABI must be specified. Probably a DTB build, skipping generation."); + return ret; } - var ret = new List (); EmbedBinary ( log, ret, From 56f030dccc2ebb01eda6c50b35e2b7d13843d3f6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 18 Oct 2024 19:31:34 +0200 Subject: [PATCH 29/49] [WIP] Builds but doesn't work --- ...crosoft.Android.Sdk.AssemblyStores.targets | 7 +- .../Tasks/CompileNativeAssembly.cs | 21 +- ...teCompressedAssembliesNativeSourceFiles.cs | 2 +- .../Tasks/GeneratePackageManagerJava.cs | 2 +- .../Tasks/PrepareAbiItems.cs | 37 +--- .../Utilities/AssemblyCompression.cs | 2 +- .../Utilities/ELFEmbeddingHelper.cs | 10 +- .../Utilities/MonoAndroidHelper.cs | 7 +- .../Utilities/NativeAssemblerCompilation.cs | 199 ++++++++++++++++++ .../Utilities/NativeAssemblerItemsHelper.cs | 65 ++++++ .../Xamarin.Android.Common.targets | 46 ++-- .../AssemblyStore/StoreReader_V2.cs | 10 +- 12 files changed, 338 insertions(+), 70 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index c7185129668..dee4e5d46be 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -7,13 +7,17 @@ <_EmbeddedAssemblyStoreObjectFile Include="$(_NativeAssemblySourceDir)embed_assembly_store.%(_BuildTargetAbis.Identity).o" /> + + + <_EmbeddedAssemblyStoreSourceFiles Include="@(_EmbeddedAssemblyStoreObjectFile->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.s'))')" /> + + Outputs="@(_EmbeddedAssemblyStoreObjectFile);@(_EmbeddedAssemblyStoreSourceFiles)"> - diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs index b8c112698f9..6206018833e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs @@ -99,9 +99,24 @@ void RunAssembler (Config config) } } + static readonly List llcArguments = new () { + "-O2", + "--debugger-tune=lldb", // NDK uses lldb now + "--debugify-level=location+variables", + "--fatal-warnings", + "--filetype=obj", + "--relocation-model=pic", + }; + + static readonly List llvmMcArguments = new () { + "--assemble", + "--filetype=obj", + "-g", + }; + IEnumerable GetAssemblerConfigs () { - const string assemblerOptions = + const string llcOptions = "-O2 " + "--debugger-tune=lldb " + // NDK uses lldb now "--debugify-level=location+variables " + @@ -113,14 +128,14 @@ IEnumerable GetAssemblerConfigs () foreach (ITaskItem item in Sources) { // We don't need the directory since our WorkingDirectory is where all the sources are string sourceFile = Path.GetFileName (item.ItemSpec); - string outputFile = QuoteFileName (sourceFile.Replace (".ll", ".o")); + string outputFile = QuoteFileName (Path.ChangeExtension (sourceFile, ".o")); string executableDir = Path.GetDirectoryName (llcPath); string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath)); yield return new Config { InputSource = item.ItemSpec, AssemblerPath = Path.Combine (executableDir, executableName), - AssemblerOptions = $"{assemblerOptions} -o={outputFile} {QuoteFileName (sourceFile)}", + AssemblerOptions = $"{llcOptions} -o={outputFile} {QuoteFileName (sourceFile)}", }; } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index 81a11497da4..5961a7c8c8e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -66,7 +66,7 @@ void GenerateCompressedAssemblySources () var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})"); + Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetItemAbi(assembly)})"); continue; } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 47151a70604..181c6b745a7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -248,7 +248,7 @@ void AddEnvironment () uniqueAssemblyNames.Add (assemblyName); } - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + string abi = MonoAndroidHelper.GetItemAbi (assembly); archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); if (!archAssemblyNames.Contains (assemblyName)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index fe494103982..d3d13dab0e2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Collections.Generic; using Microsoft.Build.Framework; @@ -10,13 +8,6 @@ namespace Xamarin.Android.Tasks { public class PrepareAbiItems : AndroidTask { - const string ArmV7a = "armeabi-v7a"; - const string TypeMapBase = "typemaps"; - const string EnvBase = "environment"; - const string CompressedAssembliesBase = "compressed_assemblies"; - const string JniRemappingBase = "jni_remap"; - const string MarshalMethodsBase = "marshal_methods"; - public override string TaskPrefix => "PAI"; [Required] @@ -28,45 +19,23 @@ public class PrepareAbiItems : AndroidTask [Required] public string Mode { get; set; } - [Required] - public bool Debug { get; set; } - [Output] public ITaskItem[] AssemblySources { get; set; } - [Output] - public ITaskItem[] AssemblyIncludes { get; set; } - public override bool RunTask () { var sources = new List (); - var includes = new List (); - string baseName; - - if (String.Compare ("typemap", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = TypeMapBase; - } else if (String.Compare ("environment", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = EnvBase; - } else if (String.Compare ("compressed", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = CompressedAssembliesBase; - } else if (String.Compare ("jniremap", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = JniRemappingBase; - } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = MarshalMethodsBase; - } else { - Log.LogError ($"Unknown mode: {Mode}"); - return false; - } TaskItem item; + NativeAssemblerItemsHelper.KnownMode mode = NativeAssemblerItemsHelper.ToKnownMode (Mode); foreach (string abi in BuildTargetAbis) { - item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}.ll")); + item = new TaskItem (NativeAssemblerItemsHelper.GetSourcePath (Log, mode, NativeSourcesDir, abi)); item.SetMetadata ("abi", abi); + item.SetMetadata ("RuntimeIdentifier", MonoAndroidHelper.AbiToRid (abi)); sources.Add (item); } AssemblySources = sources.ToArray (); - AssemblyIncludes = includes.ToArray (); return !Log.HasLoggedErrors; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index cf9a865b9c4..d2ec77c9f14 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -136,7 +136,7 @@ public static string Compress ( AssemblyData compressedAssembly = new AssemblyData (assembly.ItemSpec, info.DescriptorIndex); string assemblyOutputDir; string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + string abi = MonoAndroidHelper.GetItemAbi (assembly); if (!String.IsNullOrEmpty (subDirectory) && !(subDirectory.EndsWith ($"{abi}/", StringComparison.Ordinal) || subDirectory.EndsWith ($"{abi}\\", StringComparison.Ordinal))) { assemblyOutputDir = Path.Combine (compressedOutputDir, abi, subDirectory); } else { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index e58f2b94d5e..714269b37b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -215,10 +215,12 @@ static void DoEmbed ( MonoAndroidHelper.QuoteFileNameArgument (asmInputFile), }; - int ret = MonoAndroidHelper.RunProcess (llvmMcPath, String.Join (" ", args), log); - if (ret != 0) { - return; - } + // int ret = MonoAndroidHelper.RunProcess (llvmMcPath, String.Join (" ", args), log); + // File.Copy (asmInputFile, $"/tmp/{Path.GetFileName (asmInputFile)}", true); + // File.Copy (outputFile, $"/tmp/{Path.GetFileName (outputFile)}", true); + // if (ret != 0) { + // return; + // } } static string GetLlvmMcPath (string androidBinUtilsDirectory) => MonoAndroidHelper.GetLlvmMcPath (androidBinUtilsDirectory); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 61734c55cb4..fe47d5af794 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -588,17 +588,17 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) } #if MSBUILD - public static string GetAssemblyAbi (ITaskItem asmItem) + public static string GetItemAbi (ITaskItem asmItem) { string? abi = asmItem.GetMetadata ("Abi"); if (String.IsNullOrEmpty (abi)) { - throw new InvalidOperationException ($"Internal error: assembly '{asmItem}' lacks ABI metadata"); + throw new InvalidOperationException ($"Internal error: item '{asmItem}' lacks ABI metadata"); } return abi; } - public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetAssemblyAbi (asmItem)); + public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetItemAbi (asmItem)); #endif // MSBUILD static string GetToolsRootDirectoryRelativePath (string androidBinUtilsDirectory) @@ -797,6 +797,7 @@ public static string QuoteFileNameArgument (string fileName) public static string GetLlvmObjcopyPath (string androidBinUtilsDirectory) => GetBinUtilsToolPath (androidBinUtilsDirectory, "llvm-objcopy"); public static string GetLlvmMcPath (string androidBinUtilsDirectory) => GetBinUtilsToolPath (androidBinUtilsDirectory, "llvm-mc"); + public static string GetLlvmLlcPath (string androidBinUtilsDirectory) => GetBinUtilsToolPath (androidBinUtilsDirectory, "llc"); static string GetBinUtilsToolPath (string androidBinUtilsDirectory, string toolName) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs new file mode 100644 index 00000000000..78a3b33d90e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class NativeAssemblerCompilation +{ + public sealed class AssemblerConfig + { + public readonly string ExecutablePath; + public readonly string Options; + public readonly string InputSource; + + public AssemblerConfig (string executablePath, string options, string inputSource) + { + ExecutablePath = executablePath; + Options = options; + InputSource = inputSource; + } + } + + public sealed class AssemblerRunContext + { + public readonly TaskLoggingHelper Log; + public readonly Action? RegisterForCancellation; + public readonly Action? Cancel; + public readonly string? WorkingDirectory; + + public AssemblerRunContext (TaskLoggingHelper log, string? workingDirectory = null, Action? registerForCancellation = null, Action? cancel = null) + { + Log = log; + RegisterForCancellation = registerForCancellation; + Cancel = cancel; + WorkingDirectory = workingDirectory; + } + } + + public sealed class LlvmMcTargetConfig + { + public readonly string TargetArch; + public readonly string TripleArch; + public readonly string TripleApiPrefix; + public readonly string AssemblerDirectivePrefix; + public readonly string SizeType; + public readonly uint WordSize; + + public LlvmMcTargetConfig (string targetArch, string tripleArch, string tripleApiPrefix, string assemblerDirectivePrefix, string sizeType, uint wordSize) + { + TargetArch = targetArch; + TripleArch = tripleArch; + TripleApiPrefix = tripleApiPrefix; + AssemblerDirectivePrefix = assemblerDirectivePrefix; + SizeType = sizeType; + WordSize = wordSize; + } + } + + static readonly Dictionary llvmMcConfigs = new () { + { AndroidTargetArch.Arm64, new ("aarch64", "aarch64", "android", "@", ".xword", 8) }, + { AndroidTargetArch.Arm, new ("arm", "armv7a", "androideabi", "%", ".long", 4) }, + { AndroidTargetArch.X86_64, new ("x86-64", "x86_64", "android", "@", ".quad", 8) }, + { AndroidTargetArch.X86, new ("x86", "i686", "android", "@", ".long", 4) }, + }; + + static readonly List llcArguments = new () { + "-O2", + "--debugger-tune=lldb", // NDK uses lldb now + "--debugify-level=location+variables", + "--fatal-warnings", + "--filetype=obj", + "--relocation-model=pic", + }; + + static readonly List llvmMcArguments = new () { + "--assemble", + "--filetype=obj", + "-g", + }; + + public static AssemblerConfig GetAssemblerConfig (string androidBinUtilsDir, ITaskItem source, bool stripFilePaths) + { + string sourceFile = stripFilePaths ? Path.GetFileName (source.ItemSpec) : source.ItemSpec; + string sourceExtension = Path.GetExtension (sourceFile); + string executable; + var arguments = new List (); + + if (String.Compare (".ll", sourceExtension, StringComparison.OrdinalIgnoreCase) == 0) { + executable = MonoAndroidHelper.GetLlvmLlcPath (androidBinUtilsDir); + arguments.AddRange (llcArguments); + } else if (String.Compare (".s", sourceExtension, StringComparison.OrdinalIgnoreCase) == 0) { + executable = MonoAndroidHelper.GetLlvmMcPath (androidBinUtilsDir); + arguments.AddRange (llvmMcArguments); + + LlvmMcTargetConfig cfg = GetLlvmMcConfig (MonoAndroidHelper.GetTargetArch (source)); + arguments.Add ($"--arch={cfg.TargetArch}"); + arguments.Add ($"--triple={cfg.TripleArch}-linux-{cfg.TripleApiPrefix}{XABuildConfig.AndroidMinimumDotNetApiLevel}"); + } else { + throw new InvalidOperationException ($"Internal exception: unknown native assembler source {source.ItemSpec}"); + } + + string outputFile = Path.ChangeExtension (sourceFile, ".o"); + arguments.Add ("-o"); + arguments.Add (MonoAndroidHelper.QuoteFileNameArgument (outputFile)); + arguments.Add (MonoAndroidHelper.QuoteFileNameArgument (sourceFile)); + + return new AssemblerConfig (executable, String.Join (" ", arguments), source.ItemSpec); + } + + public static LlvmMcTargetConfig GetLlvmMcConfig (AndroidTargetArch arch) + { + if (!llvmMcConfigs.TryGetValue (arch, out LlvmMcTargetConfig cfg)) { + throw new NotSupportedException ($"Internal error: unsupported target arch '{arch}'"); + } + + return cfg; + } + + public static void RunAssembler (AssemblerRunContext context, AssemblerConfig config) + { + var stdout_completed = new ManualResetEvent (false); + var stderr_completed = new ManualResetEvent (false); + var psi = new ProcessStartInfo () { + FileName = config.ExecutablePath, + Arguments = config.Options, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + WorkingDirectory = context.WorkingDirectory, + }; + + string assemblerName = Path.GetFileName (config.ExecutablePath); + context.Log.LogDebugMessage ($"[{assemblerName}] {psi.FileName} {psi.Arguments}"); + + var stdoutLines = new List (); + var stderrLines = new List (); + + using var proc = new Process (); + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) { + OnOutputData (context, assemblerName, s, e); + stdoutLines.Add (e.Data); + } else + stdout_completed.Set (); + }; + + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) { + OnErrorData (context, assemblerName, s, e); + stderrLines.Add (e.Data); + } else + stderr_completed.Set (); + }; + + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + + if (context.RegisterForCancellation != null) { + context.RegisterForCancellation (proc); + } + + proc.WaitForExit (); + + if (psi.RedirectStandardError) { + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (psi.RedirectStandardOutput) { + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + } + + if (proc.ExitCode != 0) { + var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); + context.Log.LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ()); + context.Cancel?.Invoke (); + } + } + + static void OnOutputData (AssemblerRunContext context, string assemblerName, object sender, DataReceivedEventArgs e) + { + context.Log.LogDebugMessage ($"[{assemblerName} stdout] {e.Data}"); + } + + static void OnErrorData (AssemblerRunContext context, string assemblerName, object sender, DataReceivedEventArgs e) + { + context.Log.LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs new file mode 100644 index 00000000000..6f2df37c30a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks; + +static class NativeAssemblerItemsHelper +{ + public enum KnownMode + { + CompressedAssemblies, + EmbeddedAssemblyStore, + EmbeddedRuntimeConfig, + Environment, + JNIRemap, + MarshalMethods, + TypeMap, + } + + sealed class ModeConfig + { + public readonly string FileNameBase; + public readonly string Extension; + + public ModeConfig (string fileNameBase, string extension) + { + FileNameBase = fileNameBase; + Extension = extension; + } + } + + const string LlvmIrExtension = "ll"; + const string NativeAssemblerExtension = "s"; + + static readonly Dictionary ModeConfigs = new () { + { KnownMode.CompressedAssemblies, new ("compressed_assemblies", LlvmIrExtension) }, + { KnownMode.EmbeddedAssemblyStore, new ("embed_assembly_store", NativeAssemblerExtension) }, + { KnownMode.EmbeddedRuntimeConfig, new ("embed_runtime_config", NativeAssemblerExtension) }, + { KnownMode.Environment, new ("environment", LlvmIrExtension) }, + { KnownMode.JNIRemap, new ("jni_remap", LlvmIrExtension) }, + { KnownMode.MarshalMethods, new ("marshal_methods", LlvmIrExtension) }, + { KnownMode.TypeMap, new ("typemaps", LlvmIrExtension) }, + }; + + public static string? GetSourcePath (TaskLoggingHelper log, KnownMode mode, string nativeSourcesDir, string abi) + { + if (!ModeConfigs.TryGetValue (mode, out ModeConfig config)) { + log.LogError ($"Unknown mode: {mode}"); + return null; + } + + return Path.Combine (nativeSourcesDir, $"{config.FileNameBase}.{abi.ToLowerInvariant ()}.{config.Extension}"); + } + + public static KnownMode ToKnownMode (string mode) + { + if (!Enum.TryParse (mode, ignoreCase: true, out KnownMode result)) { + throw new InvalidOperationException ($"Internal exception: uknown native assembler generator mode '{mode}'"); + } + + return result; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 68901ab617e..4fb677ea771 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1441,10 +1441,8 @@ because xbuild doesn't support framework reference assemblies. + Mode="TypeMap"> - @@ -1527,7 +1525,6 @@ because xbuild doesn't support framework reference assemblies. - @@ -1599,24 +1596,27 @@ because xbuild doesn't support framework reference assemblies. + Mode="Environment"> + Mode="CompressedAssemblies"> + Mode="MarshalMethods"> + + + @@ -1641,8 +1641,7 @@ because xbuild doesn't support framework reference assemblies. + Mode="JNIRemap"> @@ -1751,14 +1750,11 @@ because xbuild doesn't support framework reference assemblies. EnableMarshalMethods="$(_AndroidUseMarshalMethods)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)" - > - - + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)" /> - + @@ -1985,6 +1981,18 @@ because xbuild doesn't support framework reference assemblies. <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AndroidRemapAssemblySource.abi) + <_NativeAssemblyTarget Include="@(_EmbeddedRuntimeConfigAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_EmbeddedRuntimeConfigAssemblySource.abi) + + + + + <_NativeAssemblySource Include="@(_AndroidRemapAssemblySource)" /> + <_NativeAssemblySource Include="@(_CompressedAssembliesAssemblySource)" /> + <_NativeAssemblySource Include="@(_EmbeddedRuntimeConfigAssemblySource)" /> + <_NativeAssemblySource Include="@(_EnvironmentAssemblySource)" /> + <_NativeAssemblySource Include="@(_MarshalMethodsAssemblySource)" /> + <_NativeAssemblySource Include="@(_TypeMapAssemblySource)" /> @@ -2004,10 +2012,10 @@ because xbuild doesn't support framework reference assemblies. 0) { + StoreStream.Seek ((long)elfOffset, SeekOrigin.Begin); + magic = reader.ReadUInt32 (); + } else { + return false; + } } if (magic != Utils.ASSEMBLY_STORE_MAGIC) { From fb0e6e26dcdb46b8e823af989363e21dc91fe171 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 21 Oct 2024 21:14:13 +0200 Subject: [PATCH 30/49] Let's see how it works now --- ...crosoft.Android.Sdk.AssemblyStores.targets | 22 ++- .../Tasks/CompileNativeAssembly.cs | 139 +++--------------- .../Tasks/CreateEmbeddedAssemblyStore.cs | 26 +--- .../Tasks/GeneratePackageManagerJava.cs | 7 +- .../Utilities/ELFEmbeddingHelper.cs | 118 +++------------ .../Utilities/NativeAssemblerCompilation.cs | 11 +- .../Utilities/NativeAssemblerItemsHelper.cs | 6 + .../Xamarin.Android.Common.targets | 8 +- 8 files changed, 65 insertions(+), 272 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index dee4e5d46be..c47516065c0 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -1,23 +1,22 @@ + - - - <_EmbeddedAssemblyStoreObjectFile Include="$(_NativeAssemblySourceDir)embed_assembly_store.%(_BuildTargetAbis.Identity).o" /> - - - - <_EmbeddedAssemblyStoreSourceFiles Include="@(_EmbeddedAssemblyStoreObjectFile->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.s'))')" /> - + + + + Outputs="@(_EmbeddedAssemblyStoreSourceFiles)"> - - - + SupportedAbis="@(_BuildTargetAbis)" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs index 6206018833e..cba3b4bc15e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs @@ -2,12 +2,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Threading; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -using Xamarin.Android.Tools; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -16,13 +12,6 @@ public class CompileNativeAssembly : AsyncTask { public override string TaskPrefix => "CNA"; - sealed class Config - { - public string AssemblerPath; - public string AssemblerOptions; - public string InputSource; - } - [Required] public ITaskItem[] Sources { get; set; } @@ -37,124 +26,34 @@ sealed class Config public override System.Threading.Tasks.Task RunTaskAsync () { - return this.WhenAll (GetAssemblerConfigs (), RunAssembler); + var context = new NativeAssemblerCompilation.AssemblerRunContext ( + Log, + Path.GetFullPath (WorkingDirectory), + registerForCancellation: RegisterForCancellation, + cancel: Cancel + ); + + return this.WhenAll ( + GetAssemblerConfigs (), + (NativeAssemblerCompilation.AssemblerConfig config) => NativeAssemblerCompilation.RunAssembler (context, config) + ); } - void RunAssembler (Config config) + void RegisterForCancellation (Process proc) { - var stdout_completed = new ManualResetEvent (false); - var stderr_completed = new ManualResetEvent (false); - var psi = new ProcessStartInfo () { - FileName = config.AssemblerPath, - Arguments = config.AssemblerOptions, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WindowStyle = ProcessWindowStyle.Hidden, - WorkingDirectory = WorkingDirectory, - }; - - string assemblerName = Path.GetFileName (config.AssemblerPath); - LogDebugMessage ($"[LLVM llc] {psi.FileName} {psi.Arguments}"); - - var stdoutLines = new List (); - var stderrLines = new List (); - - using (var proc = new Process ()) { - proc.OutputDataReceived += (s, e) => { - if (e.Data != null) { - OnOutputData (assemblerName, s, e); - stdoutLines.Add (e.Data); - } else - stdout_completed.Set (); - }; - - proc.ErrorDataReceived += (s, e) => { - if (e.Data != null) { - OnErrorData (assemblerName, s, e); - stderrLines.Add (e.Data); - } else - stderr_completed.Set (); - }; - - proc.StartInfo = psi; - proc.Start (); - proc.BeginOutputReadLine (); - proc.BeginErrorReadLine (); - CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } }); - proc.WaitForExit (); - - if (psi.RedirectStandardError) - stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); - - if (psi.RedirectStandardOutput) - stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); - - if (proc.ExitCode != 0) { - var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); - LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ()); - Cancel (); + CancellationToken.Register (() => { + try { + proc.Kill (); + } catch (Exception) { } - } + }); } - static readonly List llcArguments = new () { - "-O2", - "--debugger-tune=lldb", // NDK uses lldb now - "--debugify-level=location+variables", - "--fatal-warnings", - "--filetype=obj", - "--relocation-model=pic", - }; - - static readonly List llvmMcArguments = new () { - "--assemble", - "--filetype=obj", - "-g", - }; - - IEnumerable GetAssemblerConfigs () + IEnumerable GetAssemblerConfigs () { - const string llcOptions = - "-O2 " + - "--debugger-tune=lldb " + // NDK uses lldb now - "--debugify-level=location+variables " + - "--fatal-warnings " + - "--filetype=obj " + - "--relocation-model=pic"; - string llcPath = Path.Combine (AndroidBinUtilsDirectory, "llc"); - foreach (ITaskItem item in Sources) { - // We don't need the directory since our WorkingDirectory is where all the sources are - string sourceFile = Path.GetFileName (item.ItemSpec); - string outputFile = QuoteFileName (Path.ChangeExtension (sourceFile, ".o")); - string executableDir = Path.GetDirectoryName (llcPath); - string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath)); - - yield return new Config { - InputSource = item.ItemSpec, - AssemblerPath = Path.Combine (executableDir, executableName), - AssemblerOptions = $"{llcOptions} -o={outputFile} {QuoteFileName (sourceFile)}", - }; + yield return NativeAssemblerCompilation.GetAssemblerConfig (AndroidBinUtilsDirectory, item, stripFilePaths: true); } } - - void OnOutputData (string assemblerName, object sender, DataReceivedEventArgs e) - { - LogDebugMessage ($"[{assemblerName} stdout] {e.Data}"); - } - - void OnErrorData (string assemblerName, object sender, DataReceivedEventArgs e) - { - LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High); - } - - static string QuoteFileName (string fileName) - { - var builder = new CommandLineBuilder (); - builder.AppendFileNameIfNotNull (fileName); - return builder.ToString (); - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs index 288f52be3db..2823af2c3e8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -43,12 +43,6 @@ public class CreateEmbeddedAssemblyStore : AndroidTask [Required] public string [] SupportedAbis { get; set; } - [Output] - public ITaskItem[] NativeAssemblySources { get; set; } - - [Output] - public ITaskItem[] EmbeddedObjectFiles { get; set; } - public override bool RunTask () { bool compress = !Debug && EnableCompression; @@ -71,14 +65,12 @@ public override bool RunTask () // Add framework assemblies AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection); - var objectFiles = new List (); - var sourceFiles = new List (); Dictionary assemblyStorePaths = storeBuilder.Generate (Path.Combine (AppSharedLibrariesDir, "embedded")); foreach (var kvp in assemblyStorePaths) { string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); string inputFile = kvp.Value; - List items = ELFEmbeddingHelper.EmbedBinary ( + ELFEmbeddingHelper.EmbedBinary ( Log, abi, AndroidBinUtilsDirectory, @@ -87,24 +79,8 @@ public override bool RunTask () AssemblySourcesDir, missingContentOK: false ); - - if (items.Count == 0) { - continue; - } - - objectFiles.AddRange (items); - foreach (ITaskItem objectItem in items) { - var sourceItem = new TaskItem ( - Path.ChangeExtension (objectItem.ItemSpec, ".s"), - objectItem.CloneCustomMetadata () - ); - sourceFiles.Add (sourceItem); - } } - NativeAssemblySources = sourceFiles.ToArray (); - EmbeddedObjectFiles = objectFiles.ToArray (); - return !Log.HasLoggedErrors; void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 181c6b745a7..432d2c081e3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -68,9 +68,6 @@ public class GeneratePackageManagerJava : AndroidTask //[Required] public bool AssemblyStoreEmbeddedInRuntime { get; set; } - [Output] - public ITaskItem[] EmbeddedObjectFiles { get; set; } - public bool EnableMarshalMethods { get; set; } public string RuntimeConfigBinFilePath { get; set; } public string BoundExceptionType { get; set; } @@ -344,7 +341,7 @@ void AddEnvironment () } bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); - List objectFilePaths = ELFEmbeddingHelper.EmbedBinary ( + ELFEmbeddingHelper.EmbedBinary ( Log, SupportedAbis, AndroidBinUtilsDirectory, @@ -354,8 +351,6 @@ void AddEnvironment () missingContentOK: !haveRuntimeConfigBlob ); - EmbeddedObjectFiles = objectFilePaths.ToArray (); - var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { UsesMonoAOT = usesMonoAOT, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 714269b37b7..2d6a2551765 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -4,7 +4,6 @@ using System.Text; using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Android.Tools; @@ -16,50 +15,25 @@ public sealed class EmbedItem { public readonly string SymbolName; public readonly string BaseFileName; + public readonly NativeAssemblerItemsHelper.KnownMode NativeAssemblerMode; - public EmbedItem (string symbolName, string baseFileName) + public EmbedItem (string symbolName, string baseFileName, NativeAssemblerItemsHelper.KnownMode nativeAssemblerMode) { SymbolName = symbolName; BaseFileName = baseFileName; + NativeAssemblerMode = nativeAssemblerMode; } } public static class KnownEmbedItems { - public static readonly EmbedItem RuntimeConfig = new ("embedded_runtime_config", "runtime_config"); - public static readonly EmbedItem AssemblyStore = new ("embedded_assembly_store", "assembly_store"); + public static readonly EmbedItem RuntimeConfig = new ("embedded_runtime_config", "runtime_config", NativeAssemblerItemsHelper.KnownMode.EmbeddedRuntimeConfig); + public static readonly EmbedItem AssemblyStore = new ("embedded_assembly_store", "assembly_store", NativeAssemblerItemsHelper.KnownMode.EmbeddedAssemblyStore); } - sealed class LlvmMcTargetConfig - { - public readonly string TargetArch; - public readonly string TripleArch; - public readonly string TripleApiPrefix; - public readonly string AssemblerDirectivePrefix; - public readonly string SizeType; - public readonly uint WordSize; - - public LlvmMcTargetConfig (string targetArch, string tripleArch, string tripleApiPrefix, string assemblerDirectivePrefix, string sizeType, uint wordSize) - { - TargetArch = targetArch; - TripleArch = tripleArch; - TripleApiPrefix = tripleApiPrefix; - AssemblerDirectivePrefix = assemblerDirectivePrefix; - SizeType = sizeType; - WordSize = wordSize; - } - } - - static readonly Dictionary llvmMcConfigs = new () { - { AndroidTargetArch.Arm64, new ("aarch64", "aarch64", "android", "@", ".xword", 8) }, - { AndroidTargetArch.Arm, new ("arm", "armv7a", "androideabi", "%", ".long", 4) }, - { AndroidTargetArch.X86_64, new ("x86-64", "x86_64", "android", "@", ".quad", 8) }, - { AndroidTargetArch.X86, new ("x86", "i686", "android", "@", ".long", 4) }, - }; - static readonly Encoding asmFileEncoding = new UTF8Encoding (false); - public static List EmbedBinary ( + public static void EmbedBinary ( TaskLoggingHelper log, ICollection supportedAbis, string androidBinUtilsDirectory, @@ -68,30 +42,23 @@ public static List EmbedBinary ( string outputDirectory, bool missingContentOK) { - var ret = new List (); if (supportedAbis.Count < 1) { log.LogDebugMessage ("ELFEmbeddingHelper: at least one target ABI must be specified. Probably a DTB build, skipping generation."); - return ret; + return; } - string llvmMcPath = GetLlvmMcPath (androidBinUtilsDirectory); foreach (string abi in supportedAbis) { - EmbedBinary ( + DoEmbed ( log, - ret, - llvmMcPath, - abi, - inputFile, - outputDirectory, + MonoAndroidHelper.AbiToTargetArch (abi), + inputFile, outputDirectory, embedItem, missingContentOK ); } - - return ret; } - public static List EmbedBinary ( + public static void EmbedBinary ( TaskLoggingHelper log, string abi, string androidBinUtilsDirectory, @@ -100,59 +67,30 @@ public static List EmbedBinary ( string outputDirectory, bool missingContentOK) { - var ret = new List (); if (String.IsNullOrEmpty (abi)) { log.LogDebugMessage ("ELFEmbeddingHelper: ABI must be specified. Probably a DTB build, skipping generation."); - return ret; + return; } - EmbedBinary ( + DoEmbed ( log, - ret, - GetLlvmMcPath (androidBinUtilsDirectory), - abi, + MonoAndroidHelper.AbiToTargetArch (abi), inputFile, outputDirectory, embedItem, missingContentOK ); - return ret; - } - - static void EmbedBinary ( - TaskLoggingHelper log, - List resultItems, - string llvmMcPath, - string abi, - string inputFile, - string outputDirectory, - EmbedItem embedItem, - bool missingContentOK) - { - string outputFile = Path.Combine (outputDirectory, $"embed_{embedItem.BaseFileName}.{abi.ToLowerInvariant ()}.o"); - DoEmbed (log, MonoAndroidHelper.AbiToTargetArch (abi), llvmMcPath, inputFile, outputFile, embedItem, missingContentOK); - if (!File.Exists (outputFile)) { - return; - } - - var taskItem = new TaskItem (outputFile); - taskItem.SetMetadata ("Abi", abi); - taskItem.SetMetadata ("RuntimeIdentifier", MonoAndroidHelper.AbiToRid (abi)); - resultItems.Add (taskItem); } static void DoEmbed ( TaskLoggingHelper log, AndroidTargetArch arch, - string llvmMcPath, string? inputFile, - string outputFile, + string outputDirectory, EmbedItem item, bool missingContentOK) { - if (!llvmMcConfigs.TryGetValue (arch, out LlvmMcTargetConfig cfg)) { - throw new NotSupportedException ($"Internal error: unsupported target arch '{arch}'"); - } + NativeAssemblerCompilation.LlvmMcTargetConfig cfg = NativeAssemblerCompilation.GetLlvmMcConfig (arch); bool haveInputFile = !String.IsNullOrEmpty (inputFile); if (!haveInputFile) { @@ -162,7 +100,6 @@ static void DoEmbed ( } else { inputFile = Path.GetFullPath (inputFile); } - outputFile = Path.GetFullPath (outputFile); long inputFileSize = 0; string? sanitizedInputFilePath = null; @@ -177,9 +114,9 @@ static void DoEmbed ( } } - string asmInputFile = Path.ChangeExtension (outputFile, ".s"); + string asmSourceFile = NativeAssemblerItemsHelper.GetSourcePath (log, item.NativeAssemblerMode, outputDirectory, arch); - using var fs = File.Open (asmInputFile, FileMode.Create, FileAccess.Write, FileShare.Read); + using var fs = File.Open (asmSourceFile, FileMode.Create, FileAccess.Write, FileShare.Read); using var sw = new StreamWriter (fs, asmFileEncoding); string symbolName = item.SymbolName; @@ -204,24 +141,5 @@ static void DoEmbed ( sw.Flush (); sw.Close (); - - var args = new List { - $"--arch={cfg.TargetArch}", - "--assemble", - "--filetype=obj", - "-g", - $"--triple={cfg.TripleArch}-linux-{cfg.TripleApiPrefix}{XABuildConfig.AndroidMinimumDotNetApiLevel}", - "-o", MonoAndroidHelper.QuoteFileNameArgument (outputFile), - MonoAndroidHelper.QuoteFileNameArgument (asmInputFile), - }; - - // int ret = MonoAndroidHelper.RunProcess (llvmMcPath, String.Join (" ", args), log); - // File.Copy (asmInputFile, $"/tmp/{Path.GetFileName (asmInputFile)}", true); - // File.Copy (outputFile, $"/tmp/{Path.GetFileName (outputFile)}", true); - // if (ret != 0) { - // return; - // } } - - static string GetLlvmMcPath (string androidBinUtilsDirectory) => MonoAndroidHelper.GetLlvmMcPath (androidBinUtilsDirectory); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs index 78a3b33d90e..6e4c08478ce 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerCompilation.cs @@ -149,26 +149,25 @@ public static void RunAssembler (AssemblerRunContext context, AssemblerConfig co if (e.Data != null) { OnOutputData (context, assemblerName, s, e); stdoutLines.Add (e.Data); - } else + } else { stdout_completed.Set (); + } }; proc.ErrorDataReceived += (s, e) => { if (e.Data != null) { OnErrorData (context, assemblerName, s, e); stderrLines.Add (e.Data); - } else + } else { stderr_completed.Set (); + } }; proc.StartInfo = psi; proc.Start (); proc.BeginOutputReadLine (); proc.BeginErrorReadLine (); - - if (context.RegisterForCancellation != null) { - context.RegisterForCancellation (proc); - } + context.RegisterForCancellation?.Invoke (proc); proc.WaitForExit (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs index 6f2df37c30a..6781f2461eb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerItemsHelper.cs @@ -3,6 +3,7 @@ using System.IO; using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -44,6 +45,11 @@ public ModeConfig (string fileNameBase, string extension) { KnownMode.TypeMap, new ("typemaps", LlvmIrExtension) }, }; + public static string? GetSourcePath (TaskLoggingHelper log, KnownMode mode, string nativeSourcesDir, AndroidTargetArch arch) + { + return GetSourcePath (log, mode, nativeSourcesDir, MonoAndroidHelper.ArchToAbi (arch)); + } + public static string? GetSourcePath (TaskLoggingHelper log, KnownMode mode, string nativeSourcesDir, string abi) { if (!ModeConfigs.TryGetValue (mode, out ModeConfig config)) { diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 4fb677ea771..c837867f055 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1961,7 +1961,7 @@ because xbuild doesn't support framework reference assemblies. - + <_NativeAssemblyTarget Include="@(_TypeMapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_TypeMapAssemblySource.abi) @@ -1981,6 +1981,9 @@ because xbuild doesn't support framework reference assemblies. <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AndroidRemapAssemblySource.abi) + <_NativeAssemblyTarget Include="@(_EmbeddedAssemblyStoreSourceFiles->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_EmbeddedAssemblyStoreSourceFiles.abi) + <_NativeAssemblyTarget Include="@(_EmbeddedRuntimeConfigAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_EmbeddedRuntimeConfigAssemblySource.abi) @@ -1989,6 +1992,7 @@ because xbuild doesn't support framework reference assemblies. <_NativeAssemblySource Include="@(_AndroidRemapAssemblySource)" /> <_NativeAssemblySource Include="@(_CompressedAssembliesAssemblySource)" /> + <_NativeAssemblySource Include="@(_EmbeddedAssemblyStoreSourceFiles)" /> <_NativeAssemblySource Include="@(_EmbeddedRuntimeConfigAssemblySource)" /> <_NativeAssemblySource Include="@(_EnvironmentAssemblySource)" /> <_NativeAssemblySource Include="@(_MarshalMethodsAssemblySource)" /> @@ -2011,7 +2015,7 @@ because xbuild doesn't support framework reference assemblies. Date: Tue, 22 Oct 2024 11:46:32 +0200 Subject: [PATCH 31/49] Always generate embedded store sources in CreateEmbeddedAssemblyStore --- ...crosoft.Android.Sdk.AssemblyStores.targets | 2 +- .../Tasks/CreateEmbeddedAssemblyStore.cs | 25 +++++++++++++++++++ .../Tasks/GeneratePackageManagerJava.cs | 6 +---- ...pplicationConfigNativeAssemblyGenerator.cs | 19 -------------- .../Xamarin.Android.Common.targets | 3 +-- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index c47516065c0..89da58cc4f1 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -13,7 +13,6 @@ @@ -21,6 +20,7 @@ AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)" AssemblySourcesDir="$(IntermediateOutputPath)android" + AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)" CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)\test\" Debug="$(AndroidIncludeDebugSymbols)" EnableCompression="$(AndroidEnableAssemblyCompression)" diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs index 2823af2c3e8..d6eda689509 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs @@ -25,6 +25,9 @@ public class CreateEmbeddedAssemblyStore : AndroidTask [Required] public string CompressedAssembliesDir { get; set; } + [Required] + public bool AssemblyStoreEmbeddedInRuntime { get; set; } + [Required] public bool Debug { get; set; } @@ -44,6 +47,28 @@ public class CreateEmbeddedAssemblyStore : AndroidTask public string [] SupportedAbis { get; set; } public override bool RunTask () + { + if (AssemblyStoreEmbeddedInRuntime) { + return EmbedAssemblyStore (); + } + + // Generate sources to satisfy libmonodroid's ABI requirements + foreach (string abi in SupportedAbis) { + ELFEmbeddingHelper.EmbedBinary ( + Log, + abi, + AndroidBinUtilsDirectory, + inputFile: null, + ELFEmbeddingHelper.KnownEmbedItems.AssemblyStore, + AssemblySourcesDir, + missingContentOK: true + ); + } + + return !Log.HasLoggedErrors; + } + + bool EmbedAssemblyStore () { bool compress = !Debug && EnableCompression; IDictionary>? compressedAssembliesInfo = null; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 432d2c081e3..ebb9c77c080 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -61,13 +61,10 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } - // These two properties should be required but they will require modifying `monodroid` first + // This property should be required but it will require modifying `monodroid` first //[Required] public string AndroidBinUtilsDirectory { get; set; } - //[Required] - public bool AssemblyStoreEmbeddedInRuntime { get; set; } - public bool EnableMarshalMethods { get; set; } public string RuntimeConfigBinFilePath { get; set; } public string BoundExceptionType { get; set; } @@ -377,7 +374,6 @@ void AddEnvironment () JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, MarshalMethodsEnabled = EnableMarshalMethods, IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), - AssemblyStoreEmbeddedInRuntime = UseAssemblyStore && AssemblyStoreEmbeddedInRuntime, }; LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 0280d7fd1ea..d06a5f69b79 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -189,7 +189,6 @@ sealed class XamarinAndroidBundledAssembly public uint PackagedNativeLibrariesCount { get; set; } public bool MarshalMethodsEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } - public bool AssemblyStoreEmbeddedInRuntime { get; set; } public ApplicationConfigNativeAssemblyGenerator (IDictionary environmentVariables, IDictionary systemProperties, TaskLoggingHelper log) : base (log) @@ -296,24 +295,6 @@ protected override void Construct (LlvmIrModule module) module.Add (bundled_assemblies); AddAssemblyStores (module); - - if (AssemblyStoreEmbeddedInRuntime) { - return; - } - - // Need these to keep ABI compatibility with `libxamarin-app.so` used at the runtime's build time - var embedded_assembly_store_size = new LlvmIrGlobalVariable ( - (ulong)0, - "embedded_assembly_store_size", - LlvmIrVariableOptions.GlobalConstant - ); - module.Add (embedded_assembly_store_size); - - var embedded_assembly_store = new LlvmIrGlobalVariable (typeof (byte[]), "embedded_assembly_store", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = 0, - }; - module.Add (embedded_assembly_store); } void AddAssemblyStores (LlvmIrModule module) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index c837867f055..5f374dc948b 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1749,8 +1749,7 @@ because xbuild doesn't support framework reference assemblies. UseAssemblyStore="$(AndroidUseAssemblyStore)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" - AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" - AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)" /> + AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" /> From 581991eb2c21ae384e8ef5b27a17f7b6aebf23e7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 22 Oct 2024 16:46:39 +0200 Subject: [PATCH 32/49] Let's see... --- .../Microsoft.Android.Sdk.AssemblyResolution.targets | 4 ++-- .../Microsoft.Android.Sdk.AssemblyStores.targets | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index 10382ed3cc4..488582a12d4 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -185,7 +185,7 @@ _ResolveAssemblies MSBuild target. - + <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> @@ -194,7 +194,7 @@ _ResolveAssemblies MSBuild target. <_ShrunkUserAssemblies Include="@(_ResolvedUserAssemblies)" /> <_ShrunkFrameworkAssemblies Include="@(_ResolvedFrameworkAssemblies)" /> - + <_ResolvedAssemblies Include="@(ResolvedAssemblies)" /> <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies)" /> <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies)" /> diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets index 89da58cc4f1..22fa9dca278 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets @@ -8,13 +8,18 @@ BuildTargetAbis="@(_BuildTargetAbis)" NativeSourcesDir="$(_NativeAssemblySourceDir)" Mode="EmbeddedAssemblyStore"> - - + + + + + <_CreateEmbeddedAssemblyStoreAssembly Include="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths);@(_ShrunkFrameworkAssemblies)"/> + <_CreateEmbeddedAssemblyStoreAssembly Condition=" '@(_CreateEmbeddedAssemblyStoreAssembly->Count())' == '0' " Include="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" /> + Date: Wed, 23 Oct 2024 13:09:41 +0200 Subject: [PATCH 33/49] Don't assume assemblies exist --- .../Mono.Android/ApplicationAttribute.Partial.cs | 6 ++++++ .../Mono.Android/InstrumentationAttribute.Partial.cs | 8 +++++++- .../Mono.Android/PermissionAttribute.Partial.cs | 8 +++++++- .../Mono.Android/PermissionGroupAttribute.Partial.cs | 8 +++++++- .../Mono.Android/PermissionTreeAttribute.Partial.cs | 8 +++++++- .../Mono.Android/SupportsGLTextureAttribute.Partial.cs | 9 +++++++-- .../Mono.Android/UsesFeatureAttribute.Partial.cs | 9 +++++++-- .../Mono.Android/UsesPermissionAttribute.Partial.cs | 6 ++++++ .../Tasks/GeneratePackageManagerJava.cs | 9 ++++++++- .../Utilities/ManifestDocument.cs | 4 ++++ 10 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs index 4507424b7a5..b72c2ce4d9d 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs @@ -57,6 +57,12 @@ static partial void AddManualMapping () public static ApplicationAttribute FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + return null; + } + CustomAttribute attr = provider.GetCustomAttributes ("Android.App.ApplicationAttribute") .SingleOrDefault (); if (attr == null) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs index ae3cc8d31b9..ef1f5833ff2 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs @@ -11,11 +11,17 @@ namespace Android.App { partial class InstrumentationAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + foreach (CustomAttribute attr in provider.GetCustomAttributes ("Android.App.InstrumentationAttribute")) { InstrumentationAttribute self = new InstrumentationAttribute (); self.specified = mapping.Load (self, attr, cache); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs index 1ca8b28ae39..16d32e0bf5b 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs @@ -16,11 +16,17 @@ namespace Android.App { partial class PermissionAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.PermissionAttribute"); foreach (var attr in attrs) { PermissionAttribute self = new PermissionAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs index a1e00e4b68e..1cda1eefe1b 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs @@ -16,11 +16,17 @@ namespace Android.App { partial class PermissionGroupAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.PermissionGroupAttribute"); foreach (var attr in attrs) { PermissionGroupAttribute self = new PermissionGroupAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs index 0d388e440d3..5447164294c 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs @@ -16,11 +16,17 @@ namespace Android.App { partial class PermissionTreeAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.PermissionTreeAttribute"); foreach (var attr in attrs) { PermissionTreeAttribute self = new PermissionTreeAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs index 41d970eb8cb..1f5cc0c65ec 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs @@ -29,16 +29,21 @@ internal XElement ToElement (string packageName, TypeDefinitionCache cache) public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.SupportsGLTextureAttribute"); foreach (var attr in attrs) { if (attr.HasConstructorArguments && attr.ConstructorArguments.Count == 1) { SupportsGLTextureAttribute self = new SupportsGLTextureAttribute((string)attr.ConstructorArguments[0].Value); self.specified = mapping.Load (self, attr, cache); self.specified.Add("Name"); - yield return self; + yield return self; } } } } } - diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs index 58fc679741a..ca62e532f93 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs @@ -35,12 +35,18 @@ internal XElement ToElement (string packageName, TypeDefinitionCache cache) public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.UsesFeatureAttribute"); foreach (var attr in attrs) { UsesFeatureAttribute self = new UsesFeatureAttribute (); - if (attr.HasProperties) { + if (attr.HasProperties) { // handle the case where the user sets additional properties self.specified = mapping.Load (self, attr, cache); if (self.specified.Contains("GLESVersion") && self.GLESVersion==0) { @@ -64,4 +70,3 @@ public static IEnumerable FromCustomAttributeProvider (ICu } } } - diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs index 5ecad9d4fdd..2a850fe120e 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs @@ -16,6 +16,12 @@ partial class UsesPermissionAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { + // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of + // tests which check these situations. + if (provider == null) { + yield break; + } + var attrs = provider.GetCustomAttributes ("Android.App.UsesPermissionAttribute"); foreach (var attr in attrs) { UsesPermissionAttribute self; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index ebb9c77c080..046057a40bd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -468,8 +468,15 @@ bool ShouldIgnoreSplitConfigs () void GetRequiredTokens (string assemblyFilePath, out int android_runtime_jnienv_class_token, out int jnienv_initialize_method_token, out int jnienv_registerjninatives_method_token) { - using (var pe = new PEReader (File.OpenRead (assemblyFilePath))) { + if (File.Exists (assemblyFilePath)) { + using var pe = new PEReader (File.OpenRead (assemblyFilePath)); GetRequiredTokens (pe.GetMetadataReader (), out android_runtime_jnienv_class_token, out jnienv_initialize_method_token, out jnienv_registerjninatives_method_token); + } else { + android_runtime_jnienv_class_token = -1; + jnienv_initialize_method_token = -1; + jnienv_registerjninatives_method_token = -1; + Log.LogDebugMessage ($"Assembly '{assemblyFilePath}' does not exist, unable to read required tokens from it"); + return; } if (android_runtime_jnienv_class_token == -1 || jnienv_initialize_method_token == -1 || jnienv_registerjninatives_method_token == -1) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 50e1e71e3f9..4419ad33407 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -578,6 +578,10 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List usesConfigurationAttr = []; foreach (var assemblyPath in Assemblies) { var assembly = Resolver.GetAssembly (assemblyPath); + if (assembly == null) { + continue; + } + if (ApplicationAttribute.FromCustomAttributeProvider (assembly, cache) is ApplicationAttribute a) { assemblyAttr.Add (a); } From 81726be048f3afd0085941fce9665a6267258b45 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 13:15:22 +0200 Subject: [PATCH 34/49] better --- .../Tasks/GeneratePackageManagerJava.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 046057a40bd..c3baa52f288 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -468,10 +468,7 @@ bool ShouldIgnoreSplitConfigs () void GetRequiredTokens (string assemblyFilePath, out int android_runtime_jnienv_class_token, out int jnienv_initialize_method_token, out int jnienv_registerjninatives_method_token) { - if (File.Exists (assemblyFilePath)) { - using var pe = new PEReader (File.OpenRead (assemblyFilePath)); - GetRequiredTokens (pe.GetMetadataReader (), out android_runtime_jnienv_class_token, out jnienv_initialize_method_token, out jnienv_registerjninatives_method_token); - } else { + if (!File.Exists (assemblyFilePath)) { android_runtime_jnienv_class_token = -1; jnienv_initialize_method_token = -1; jnienv_registerjninatives_method_token = -1; @@ -479,6 +476,9 @@ void GetRequiredTokens (string assemblyFilePath, out int android_runtime_jnienv_ return; } + using var pe = new PEReader (File.OpenRead (assemblyFilePath)); + GetRequiredTokens (pe.GetMetadataReader (), out android_runtime_jnienv_class_token, out jnienv_initialize_method_token, out jnienv_registerjninatives_method_token); + if (android_runtime_jnienv_class_token == -1 || jnienv_initialize_method_token == -1 || jnienv_registerjninatives_method_token == -1) { throw new InvalidOperationException ($"Unable to find the required Android.Runtime.JNIEnvInit method tokens for {assemblyFilePath}"); } From a84b663897cd6b80c61c2e131b5f43306df43dd2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 13:18:47 +0200 Subject: [PATCH 35/49] Update apkdesc files --- .../BuildReleaseArm64SimpleDotNet.apkdesc | 33 ++++---- .../BuildReleaseArm64XFormsDotNet.apkdesc | 77 +++++++++---------- 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 7245c6f39a5..cdb54a8d1b5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -5,49 +5,46 @@ "Size": 3036 }, "classes.dex": { - "Size": 389672 + "Size": 389676 }, "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { - "Size": 18208 + "Size": 18296 }, "lib/arm64-v8a/lib_Java.Interop.dll.so": { "Size": 86256 }, "lib/arm64-v8a/lib_Mono.Android.dll.so": { - "Size": 115344 + "Size": 116080 }, "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { - "Size": 22400 + "Size": 22408 }, "lib/arm64-v8a/lib_System.Console.dll.so": { - "Size": 24384 + "Size": 24376 }, "lib/arm64-v8a/lib_System.Linq.dll.so": { "Size": 26480 }, "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { - "Size": 633920 + "Size": 634344 }, "lib/arm64-v8a/lib_System.Runtime.dll.so": { - "Size": 20048 + "Size": 20040 }, "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { - "Size": 21592 + "Size": 21584 }, "lib/arm64-v8a/lib_UnnamedProject.dll.so": { "Size": 20024 }, - "lib/arm64-v8a/libarc.bin.so": { - "Size": 18776 - }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 485400 + "Size": 493384 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3196336 + "Size": 3196512 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 67248 @@ -62,16 +59,16 @@ "Size": 160232 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 12648 + "Size": 15144 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1223 + "Size": 1221 }, "META-INF/BNDLTOOL.SF": { - "Size": 3266 + "Size": 3167 }, "META-INF/MANIFEST.MF": { - "Size": 3139 + "Size": 3040 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -98,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2865685 + "PackageSize": 2865612 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 3fdac7611d2..a48f617a89a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -32,28 +32,28 @@ "Size": 2396 }, "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { - "Size": 19456 + "Size": 19544 }, "lib/arm64-v8a/lib_FormsViewGroup.dll.so": { "Size": 25184 }, "lib/arm64-v8a/lib_Java.Interop.dll.so": { - "Size": 94640 + "Size": 94648 }, "lib/arm64-v8a/lib_Mono.Android.dll.so": { - "Size": 521824 + "Size": 522008 }, "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { - "Size": 22400 + "Size": 22408 }, "lib/arm64-v8a/lib_mscorlib.dll.so": { - "Size": 21440 + "Size": 21432 }, "lib/arm64-v8a/lib_netstandard.dll.so": { - "Size": 23080 + "Size": 23072 }, "lib/arm64-v8a/lib_System.Collections.Concurrent.dll.so": { - "Size": 29800 + "Size": 29792 }, "lib/arm64-v8a/lib_System.Collections.dll.so": { "Size": 36288 @@ -62,22 +62,22 @@ "Size": 25760 }, "lib/arm64-v8a/lib_System.Collections.Specialized.dll.so": { - "Size": 23848 + "Size": 23840 }, "lib/arm64-v8a/lib_System.ComponentModel.dll.so": { "Size": 19584 }, "lib/arm64-v8a/lib_System.ComponentModel.Primitives.dll.so": { - "Size": 21296 + "Size": 21304 }, "lib/arm64-v8a/lib_System.ComponentModel.TypeConverter.dll.so": { - "Size": 42448 + "Size": 42456 }, "lib/arm64-v8a/lib_System.Console.dll.so": { "Size": 24416 }, "lib/arm64-v8a/lib_System.Core.dll.so": { - "Size": 19456 + "Size": 19448 }, "lib/arm64-v8a/lib_System.Diagnostics.DiagnosticSource.dll.so": { "Size": 28440 @@ -86,19 +86,19 @@ "Size": 24688 }, "lib/arm64-v8a/lib_System.dll.so": { - "Size": 19856 + "Size": 19848 }, "lib/arm64-v8a/lib_System.Drawing.dll.so": { - "Size": 19432 + "Size": 19424 }, "lib/arm64-v8a/lib_System.Drawing.Primitives.dll.so": { - "Size": 30048 + "Size": 30040 }, "lib/arm64-v8a/lib_System.Formats.Asn1.dll.so": { - "Size": 49936 + "Size": 49928 }, "lib/arm64-v8a/lib_System.IO.Compression.Brotli.dll.so": { - "Size": 29480 + "Size": 29472 }, "lib/arm64-v8a/lib_System.IO.Compression.dll.so": { "Size": 33784 @@ -110,31 +110,31 @@ "Size": 38736 }, "lib/arm64-v8a/lib_System.Linq.Expressions.dll.so": { - "Size": 185808 + "Size": 185800 }, "lib/arm64-v8a/lib_System.Net.Http.dll.so": { - "Size": 89496 + "Size": 89488 }, "lib/arm64-v8a/lib_System.Net.Primitives.dll.so": { - "Size": 41120 + "Size": 41112 }, "lib/arm64-v8a/lib_System.Net.Requests.dll.so": { - "Size": 21552 + "Size": 21544 }, "lib/arm64-v8a/lib_System.ObjectModel.dll.so": { - "Size": 27072 + "Size": 27064 }, "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { - "Size": 956408 + "Size": 956568 }, "lib/arm64-v8a/lib_System.Private.DataContractSerialization.dll.so": { - "Size": 216688 + "Size": 216720 }, "lib/arm64-v8a/lib_System.Private.Uri.dll.so": { - "Size": 62192 + "Size": 62184 }, "lib/arm64-v8a/lib_System.Private.Xml.dll.so": { - "Size": 237104 + "Size": 237120 }, "lib/arm64-v8a/lib_System.Private.Xml.Linq.dll.so": { "Size": 35584 @@ -143,7 +143,7 @@ "Size": 20200 }, "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { - "Size": 21592 + "Size": 21584 }, "lib/arm64-v8a/lib_System.Runtime.Numerics.dll.so": { "Size": 54408 @@ -152,22 +152,22 @@ "Size": 19352 }, "lib/arm64-v8a/lib_System.Runtime.Serialization.Formatters.dll.so": { - "Size": 20336 + "Size": 20328 }, "lib/arm64-v8a/lib_System.Runtime.Serialization.Primitives.dll.so": { "Size": 21448 }, "lib/arm64-v8a/lib_System.Security.Cryptography.dll.so": { - "Size": 80504 + "Size": 80496 }, "lib/arm64-v8a/lib_System.Text.RegularExpressions.dll.so": { - "Size": 183592 + "Size": 183584 }, "lib/arm64-v8a/lib_System.Xml.dll.so": { - "Size": 19256 + "Size": 19248 }, "lib/arm64-v8a/lib_System.Xml.Linq.dll.so": { - "Size": 19272 + "Size": 19264 }, "lib/arm64-v8a/lib_UnnamedProject.dll.so": { "Size": 22096 @@ -241,17 +241,14 @@ "lib/arm64-v8a/lib_Xamarin.Google.Android.Material.dll.so": { "Size": 84400 }, - "lib/arm64-v8a/libarc.bin.so": { - "Size": 18776 - }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87432 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 485400 + "Size": 493384 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3196336 + "Size": 3196512 }, "lib/arm64-v8a/libSystem.Globalization.Native.so": { "Size": 67248 @@ -266,7 +263,7 @@ "Size": 160232 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 119928 + "Size": 122144 }, "META-INF/androidx.activity_activity.version": { "Size": 6 @@ -422,7 +419,7 @@ "Size": 1221 }, "META-INF/BNDLTOOL.SF": { - "Size": 98661 + "Size": 98562 }, "META-INF/com.android.tools/proguard/coroutines.pro": { "Size": 1345 @@ -449,7 +446,7 @@ "Size": 5 }, "META-INF/MANIFEST.MF": { - "Size": 98534 + "Size": 98435 }, "META-INF/maven/com.google.guava/listenablefuture/pom.properties": { "Size": 96 @@ -2489,5 +2486,5 @@ "Size": 812848 } }, - "PackageSize": 10673477 + "PackageSize": 10677500 } \ No newline at end of file From 1035cddb78112a67a58235312a7aa74a186cd981 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 18:23:47 +0200 Subject: [PATCH 36/49] Load embedded assembly store also in filesystem mode --- .../monodroid/embedded-assemblies-zip.cc | 2 +- src/native/monodroid/embedded-assemblies.cc | 24 ++++++++++++------- src/native/monodroid/embedded-assemblies.hh | 6 +++++ src/native/runtime-base/monodroid-dl.hh | 4 ++-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index d2888cd97f0..9958560fbc3 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -256,7 +256,7 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::span const& b dynamic_local_string entry_name; bool assembly_store_found = embedded_assembly_store_size != 0; if (assembly_store_found) { - verify_assembly_store_and_set_info (embedded_assembly_store, "embedded"); + load_embedded_assembly_store (); log_debug (LOG_ASSEMBLY, "Looking for DSOs in APK"); } else { log_debug (LOG_ASSEMBLY, "Looking for assembly store ('%s') and DSOs in APK", assembly_store_file_path.data ()); diff --git a/src/native/monodroid/embedded-assemblies.cc b/src/native/monodroid/embedded-assemblies.cc index 8db5e4c0ee0..ba751fa96d6 100644 --- a/src/native/monodroid/embedded-assemblies.cc +++ b/src/native/monodroid/embedded-assemblies.cc @@ -1310,13 +1310,21 @@ EmbeddedAssemblies::register_from_filesystem (const char *lib_dir_path,bool look size_t EmbeddedAssemblies::register_from_filesystem (monodroid_should_register should_register) noexcept { - log_debug (LOG_ASSEMBLY, "Registering assemblies from the filesystem"); - constexpr bool LookForMangledNames = true; - size_t assembly_count = register_from_filesystem ( - AndroidSystem::app_lib_directories[0], - LookForMangledNames, - should_register - ); + size_t assembly_count; + + if (embedded_assembly_store_size > 0) { + log_debug (LOG_ASSEMBLY, "Filesystem mode, but registering assemblies from the embedded assembly store"); + load_embedded_assembly_store (); + assembly_count = assembly_store.assembly_count; + } else { + log_debug (LOG_ASSEMBLY, "Registering assemblies from the filesystem"); + constexpr bool LookForMangledNames = true; + assembly_count = register_from_filesystem ( + AndroidSystem::app_lib_directories[0], + LookForMangledNames, + should_register + ); + } #if defined(DEBUG) constexpr bool DoNotLookForMangledNames = false; @@ -1328,6 +1336,6 @@ EmbeddedAssemblies::register_from_filesystem (monodroid_should_register should_r ); #endif - log_debug (LOG_ASSEMBLY, "Found %zu assemblies on the filesystem", assembly_count); + log_debug (LOG_ASSEMBLY, "Found %zu assemblies", assembly_count); return assembly_count; } diff --git a/src/native/monodroid/embedded-assemblies.hh b/src/native/monodroid/embedded-assemblies.hh index 4e67985c3da..8d05f911fba 100644 --- a/src/native/monodroid/embedded-assemblies.hh +++ b/src/native/monodroid/embedded-assemblies.hh @@ -393,6 +393,12 @@ namespace xamarin::android::internal { void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; void verify_assembly_store_and_set_info (void *data_start, const char *name) noexcept; + + void load_embedded_assembly_store () noexcept + { + verify_assembly_store_and_set_info (embedded_assembly_store, "embedded"); + } + void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; const AssemblyStoreIndexEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept; void store_individual_assembly_data (dynamic_local_string const& entry_name, ZipEntryLoadState const& state, monodroid_should_register should_register) noexcept; diff --git a/src/native/runtime-base/monodroid-dl.hh b/src/native/runtime-base/monodroid-dl.hh index 14455578f0e..9ffd7a3c0f5 100644 --- a/src/native/runtime-base/monodroid-dl.hh +++ b/src/native/runtime-base/monodroid-dl.hh @@ -48,11 +48,11 @@ namespace xamarin::android::internal size_t arr_size; if constexpr (WhichCache == CacheKind::AOT) { - log_debug (LOG_ASSEMBLY, "Looking for hash 0x%x in AOT cache", hash); + log_debug (LOG_ASSEMBLY, "Looking for hash 0x%zx in AOT cache", hash); arr = aot_dso_cache; arr_size = application_config.number_of_aot_cache_entries; } else if constexpr (WhichCache == CacheKind::DSO) { - log_debug (LOG_ASSEMBLY, "Looking for hash 0x%x in DSO cache", hash); + log_debug (LOG_ASSEMBLY, "Looking for hash 0x%zx in DSO cache", hash); arr = dso_cache; arr_size = application_config.number_of_dso_cache_entries; } From d46e20fc2b14502cc03e9aa93b8bf6501922bbc8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 18:29:51 +0200 Subject: [PATCH 37/49] Log it when an assembly is missing in ManifestDocument --- .../Utilities/ManifestDocument.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 4419ad33407..684f93e5174 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -287,7 +287,7 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li string.IsNullOrEmpty (VersionName) ? "1.0" : VersionName); } - app = CreateApplicationElement (manifest, applicationClass, subclasses, cache); + app = CreateApplicationElement (log, manifest, applicationClass, subclasses, cache); if (app.Attribute (androidNs + "label") == null && !string.IsNullOrEmpty (ApplicationLabel)) app.SetAttributeValue (androidNs + "label", ApplicationLabel); @@ -567,7 +567,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (TaskLoggingHelper log, XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -579,6 +579,7 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L foreach (var assemblyPath in Assemblies) { var assembly = Resolver.GetAssembly (assemblyPath); if (assembly == null) { + log.LogDebugMessage ($"Assembly '{assemblyPath}' not found."); continue; } From d14811ca27cd81cbf63c5e3f64fb8f67a4247e20 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 18:38:06 +0200 Subject: [PATCH 38/49] Add some debug logging --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 0e92abe4c0a..b1e09eff3a3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -408,11 +408,14 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { + Log.LogDebugMessage ("g#: in AddAssemblies"); string sourcePath; AssemblyStoreBuilder? storeBuilder = null; if (UseAssemblyStore) { + Log.LogDebugMessage ("g#: assembly store used"); if (AssemblyStoreEmbeddedInRuntime) { + Log.LogDebugMessage ("g#: assembly store embedded in the runtime"); // We don't need to do anything here, the store is in `libxamarin-app.so` return; } @@ -420,16 +423,20 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap storeBuilder = new AssemblyStoreBuilder (Log); } + Log.LogDebugMessage ("g#: adding user assemblies"); // Add user assemblies AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, DoAddAssembliesFromArchCollection); + Log.LogDebugMessage ("g#: adding framework assemblies"); // Add framework assemblies AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection); if (!UseAssemblyStore) { + Log.LogDebugMessage ("g#: assembly store not used, returning"); return; } + Log.LogDebugMessage ("g#: generating assembly stores"); Dictionary assemblyStorePaths = storeBuilder.Generate (AppSharedLibrariesDir); if (assemblyStorePaths.Count == 0) { @@ -440,6 +447,7 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); } + Log.LogDebugMessage ($"g#: {assemblyStorePaths.Count} assembly stores added"); string inArchivePath; foreach (var kvp in assemblyStorePaths) { string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); @@ -457,6 +465,7 @@ void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch // or not we're supposed to compress .so files. sourcePath = CompressAssembly (assembly); if (UseAssemblyStore) { + Log.LogDebugMessage ($"g#: adding '{assembly}' to the store"); storeBuilder.AddAssembly (sourcePath, assembly, includeDebugSymbols: debug); return; } From 3648ab1de086a52ab91ef4787c76bedb78e2f49e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 23 Oct 2024 21:02:49 +0200 Subject: [PATCH 39/49] Don't throw --- .../AssemblyStore/AssemblyStoreExplorer.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs index 52e6f20fbf8..fdf331d3396 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs @@ -140,7 +140,19 @@ public static (IList? explorers, string? errorMessage) Op ZipEntry entry = zip.ReadEntry (path); var stream = new MemoryStream (); entry.Extract (stream); - ret.Add (new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}")); + AssemblyStoreExplorer? explorer = null; + try { + // It may throw when opening an apk without any assembly stores, in which case the v2 store reader would + // always find `libxamarin-app.so` and try to read the embedded store and fail, throwing from the explorer + // constructor. + explorer = new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}"); + } catch (NotSupportedException) { + // Ignore + } + + if (explorer != null) { + ret.Add (explorer); + } } if (ret.Count == 0) { From 4acc5a27a678657b0851b6130b6cc771c58f192f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 24 Oct 2024 15:49:21 +0200 Subject: [PATCH 40/49] Always create the destination directory --- src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs index 2d6a2551765..52704354a63 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFEmbeddingHelper.cs @@ -116,6 +116,7 @@ static void DoEmbed ( string asmSourceFile = NativeAssemblerItemsHelper.GetSourcePath (log, item.NativeAssemblerMode, outputDirectory, arch); + Directory.CreateDirectory (Path.GetDirectoryName (asmSourceFile)); using var fs = File.Open (asmSourceFile, FileMode.Create, FileAccess.Write, FileShare.Read); using var sw = new StreamWriter (fs, asmFileEncoding); From a7095d2de7c84aa4b32596574137e435f33faec4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 28 Oct 2024 12:26:17 +0100 Subject: [PATCH 41/49] Post-rebase fixups --- src/native/monodroid/embedded-assemblies-zip.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/native/monodroid/embedded-assemblies-zip.cc b/src/native/monodroid/embedded-assemblies-zip.cc index 9958560fbc3..848f23a5ad2 100644 --- a/src/native/monodroid/embedded-assemblies-zip.cc +++ b/src/native/monodroid/embedded-assemblies-zip.cc @@ -353,8 +353,10 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus .max_assembly_file_name_size = 0u, }; + std::unique_ptr raw_data (new uint8_t[cd_size]); + std::span buf (raw_data.get (), cd_size); ssize_t nread = read (fd, buf.data (), buf.size ()); - if (static_cast(nread) != cd_size) { + if (static_cast(nread) != cd_size) [[unlikely]] { Helpers::abort_application ( LOG_ASSEMBLY, Util::monodroid_strdup_printf ( From 4031e9b30107dae0c9f2782194719bee54456bb3 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 28 Oct 2024 14:29:50 +0100 Subject: [PATCH 42/49] Let's see what's in the override dir listing --- tests/MSBuildDeviceIntegration/Tests/InstallTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index 2e2960dcd03..2a9af46cabc 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -201,6 +201,7 @@ public void InstallWithoutSharedRuntime () //Assert.AreEqual (0, RunAdbCommand ("shell pm list packages Mono.Android.DebugRuntime").Trim ().Length, // "The Shared Runtime should not have been installed."); var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); + Console.WriteLine ($"InstallWithoutSharedRuntime: directoryList == '{directorylist}'"); StringAssert.Contains ($"{proj.ProjectName}.dll", directorylist, $"{proj.ProjectName}.dll should exist in the .__override__/{DeviceAbi} directory."); StringAssert.Contains ($"System.Private.CoreLib.dll", directorylist, $"System.Private.CoreLib.dll should exist in the .__override__/{DeviceAbi} directory."); StringAssert.Contains ($"Mono.Android.dll", directorylist, $"Mono.Android.dll should exist in the .__override__/{DeviceAbi} directory."); @@ -546,7 +547,7 @@ public void IncrementalFastDeployment (string packageFormat) } long lib1FirstBuildSize = new FileInfo (Path.Combine (rootPath, lib1.ProjectName, lib1.OutputPath, "Library1.dll")).Length; - + using (var builder = CreateApkBuilder (Path.Combine (rootPath, app.ProjectName))) { builder.Verbosity = LoggerVerbosity.Detailed; builder.ThrowOnBuildFailure = false; @@ -653,7 +654,7 @@ public void AppWithAndroidJavaSource () public class TestJavaClass2 { public String test(){ - + return ""Java is called""; } }", @@ -671,7 +672,7 @@ public String test(){ public class TestJavaClass { public String test(){ - + return ""Java is called""; } }", From 836c4b4c115dc52bd44f43ea31c27b2b3309c139 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 28 Oct 2024 20:13:04 +0100 Subject: [PATCH 43/49] Let's see --- .../targets/Microsoft.Android.Sdk.AssemblyResolution.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index 488582a12d4..10382ed3cc4 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -185,7 +185,7 @@ _ResolveAssemblies MSBuild target. - + <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> @@ -194,7 +194,7 @@ _ResolveAssemblies MSBuild target. <_ShrunkUserAssemblies Include="@(_ResolvedUserAssemblies)" /> <_ShrunkFrameworkAssemblies Include="@(_ResolvedFrameworkAssemblies)" /> - + <_ResolvedAssemblies Include="@(ResolvedAssemblies)" /> <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies)" /> <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies)" /> From f4f15029118670b9a9665125b4d1ed833270001b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 29 Oct 2024 10:29:17 +0100 Subject: [PATCH 44/49] Cleanup --- .../Mono.Android/InstrumentationAttribute.Partial.cs | 2 +- .../Mono.Android/PermissionAttribute.Partial.cs | 2 +- .../Mono.Android/PermissionGroupAttribute.Partial.cs | 2 +- .../Mono.Android/PermissionTreeAttribute.Partial.cs | 2 +- .../Mono.Android/SupportsGLTextureAttribute.Partial.cs | 3 ++- .../Mono.Android/UsesFeatureAttribute.Partial.cs | 3 ++- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 9 --------- tests/MSBuildDeviceIntegration/Tests/InstallTests.cs | 7 +++---- 8 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs index ef1f5833ff2..0facaf53e23 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs @@ -11,7 +11,7 @@ namespace Android.App { partial class InstrumentationAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs index 16d32e0bf5b..ebabfbc9562 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs @@ -16,7 +16,7 @@ namespace Android.App { partial class PermissionAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs index 1cda1eefe1b..99cedace7d6 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs @@ -16,7 +16,7 @@ namespace Android.App { partial class PermissionGroupAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs index 5447164294c..79ca7af4999 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs @@ -16,7 +16,7 @@ namespace Android.App { partial class PermissionTreeAttribute { - + ICollection specified; public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs index 1f5cc0c65ec..45f4df2e91e 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs @@ -41,9 +41,10 @@ public static IEnumerable FromCustomAttributeProvide SupportsGLTextureAttribute self = new SupportsGLTextureAttribute((string)attr.ConstructorArguments[0].Value); self.specified = mapping.Load (self, attr, cache); self.specified.Add("Name"); - yield return self; + yield return self; } } } } } + diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs index ca62e532f93..d4d832d34f8 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs @@ -46,7 +46,7 @@ public static IEnumerable FromCustomAttributeProvider (ICu UsesFeatureAttribute self = new UsesFeatureAttribute (); - if (attr.HasProperties) { + if (attr.HasProperties) { // handle the case where the user sets additional properties self.specified = mapping.Load (self, attr, cache); if (self.specified.Contains("GLESVersion") && self.GLESVersion==0) { @@ -70,3 +70,4 @@ public static IEnumerable FromCustomAttributeProvider (ICu } } } + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index b1e09eff3a3..0e92abe4c0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -408,14 +408,11 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { - Log.LogDebugMessage ("g#: in AddAssemblies"); string sourcePath; AssemblyStoreBuilder? storeBuilder = null; if (UseAssemblyStore) { - Log.LogDebugMessage ("g#: assembly store used"); if (AssemblyStoreEmbeddedInRuntime) { - Log.LogDebugMessage ("g#: assembly store embedded in the runtime"); // We don't need to do anything here, the store is in `libxamarin-app.so` return; } @@ -423,20 +420,16 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap storeBuilder = new AssemblyStoreBuilder (Log); } - Log.LogDebugMessage ("g#: adding user assemblies"); // Add user assemblies AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, DoAddAssembliesFromArchCollection); - Log.LogDebugMessage ("g#: adding framework assemblies"); // Add framework assemblies AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection); if (!UseAssemblyStore) { - Log.LogDebugMessage ("g#: assembly store not used, returning"); return; } - Log.LogDebugMessage ("g#: generating assembly stores"); Dictionary assemblyStorePaths = storeBuilder.Generate (AppSharedLibrariesDir); if (assemblyStorePaths.Count == 0) { @@ -447,7 +440,6 @@ void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx ap throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); } - Log.LogDebugMessage ($"g#: {assemblyStorePaths.Count} assembly stores added"); string inArchivePath; foreach (var kvp in assemblyStorePaths) { string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); @@ -465,7 +457,6 @@ void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch // or not we're supposed to compress .so files. sourcePath = CompressAssembly (assembly); if (UseAssemblyStore) { - Log.LogDebugMessage ($"g#: adding '{assembly}' to the store"); storeBuilder.AddAssembly (sourcePath, assembly, includeDebugSymbols: debug); return; } diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index 2a9af46cabc..2e2960dcd03 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -201,7 +201,6 @@ public void InstallWithoutSharedRuntime () //Assert.AreEqual (0, RunAdbCommand ("shell pm list packages Mono.Android.DebugRuntime").Trim ().Length, // "The Shared Runtime should not have been installed."); var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); - Console.WriteLine ($"InstallWithoutSharedRuntime: directoryList == '{directorylist}'"); StringAssert.Contains ($"{proj.ProjectName}.dll", directorylist, $"{proj.ProjectName}.dll should exist in the .__override__/{DeviceAbi} directory."); StringAssert.Contains ($"System.Private.CoreLib.dll", directorylist, $"System.Private.CoreLib.dll should exist in the .__override__/{DeviceAbi} directory."); StringAssert.Contains ($"Mono.Android.dll", directorylist, $"Mono.Android.dll should exist in the .__override__/{DeviceAbi} directory."); @@ -547,7 +546,7 @@ public void IncrementalFastDeployment (string packageFormat) } long lib1FirstBuildSize = new FileInfo (Path.Combine (rootPath, lib1.ProjectName, lib1.OutputPath, "Library1.dll")).Length; - + using (var builder = CreateApkBuilder (Path.Combine (rootPath, app.ProjectName))) { builder.Verbosity = LoggerVerbosity.Detailed; builder.ThrowOnBuildFailure = false; @@ -654,7 +653,7 @@ public void AppWithAndroidJavaSource () public class TestJavaClass2 { public String test(){ - + return ""Java is called""; } }", @@ -672,7 +671,7 @@ public String test(){ public class TestJavaClass { public String test(){ - + return ""Java is called""; } }", From 8e41bddd7f34e694e9090dfb58926ad8ee5fc884 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 31 Oct 2024 09:10:59 +0100 Subject: [PATCH 45/49] Address feedback --- .../Mono.Android/ApplicationAttribute.Partial.cs | 6 ------ .../Mono.Android/InstrumentationAttribute.Partial.cs | 6 ------ .../Mono.Android/PermissionAttribute.Partial.cs | 6 ------ .../Mono.Android/PermissionGroupAttribute.Partial.cs | 6 ------ .../Mono.Android/PermissionTreeAttribute.Partial.cs | 6 ------ .../Mono.Android/SupportsGLTextureAttribute.Partial.cs | 6 ------ .../Mono.Android/UsesFeatureAttribute.Partial.cs | 6 ------ .../Mono.Android/UsesPermissionAttribute.Partial.cs | 6 ------ 8 files changed, 48 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs index b72c2ce4d9d..4507424b7a5 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/ApplicationAttribute.Partial.cs @@ -57,12 +57,6 @@ static partial void AddManualMapping () public static ApplicationAttribute FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - return null; - } - CustomAttribute attr = provider.GetCustomAttributes ("Android.App.ApplicationAttribute") .SingleOrDefault (); if (attr == null) diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs index 0facaf53e23..ae3cc8d31b9 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/InstrumentationAttribute.Partial.cs @@ -16,12 +16,6 @@ partial class InstrumentationAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - foreach (CustomAttribute attr in provider.GetCustomAttributes ("Android.App.InstrumentationAttribute")) { InstrumentationAttribute self = new InstrumentationAttribute (); self.specified = mapping.Load (self, attr, cache); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs index ebabfbc9562..1ca8b28ae39 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionAttribute.Partial.cs @@ -21,12 +21,6 @@ partial class PermissionAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.PermissionAttribute"); foreach (var attr in attrs) { PermissionAttribute self = new PermissionAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs index 99cedace7d6..a1e00e4b68e 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionGroupAttribute.Partial.cs @@ -21,12 +21,6 @@ partial class PermissionGroupAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.PermissionGroupAttribute"); foreach (var attr in attrs) { PermissionGroupAttribute self = new PermissionGroupAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs index 79ca7af4999..0d388e440d3 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/PermissionTreeAttribute.Partial.cs @@ -21,12 +21,6 @@ partial class PermissionTreeAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.PermissionTreeAttribute"); foreach (var attr in attrs) { PermissionTreeAttribute self = new PermissionTreeAttribute (); diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs index 45f4df2e91e..41d970eb8cb 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/SupportsGLTextureAttribute.Partial.cs @@ -29,12 +29,6 @@ internal XElement ToElement (string packageName, TypeDefinitionCache cache) public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.SupportsGLTextureAttribute"); foreach (var attr in attrs) { if (attr.HasConstructorArguments && attr.ConstructorArguments.Count == 1) { diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs index d4d832d34f8..58fc679741a 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesFeatureAttribute.Partial.cs @@ -35,12 +35,6 @@ internal XElement ToElement (string packageName, TypeDefinitionCache cache) public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.UsesFeatureAttribute"); foreach (var attr in attrs) { diff --git a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs index 2a850fe120e..5ecad9d4fdd 100644 --- a/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs +++ b/src/Xamarin.Android.Build.Tasks/Mono.Android/UsesPermissionAttribute.Partial.cs @@ -16,12 +16,6 @@ partial class UsesPermissionAttribute { public static IEnumerable FromCustomAttributeProvider (ICustomAttributeProvider provider, TypeDefinitionCache cache) { - // `provider` might be null in situations when application configuration is broken, and it surfaces in a number of - // tests which check these situations. - if (provider == null) { - yield break; - } - var attrs = provider.GetCustomAttributes ("Android.App.UsesPermissionAttribute"); foreach (var attr in attrs) { UsesPermissionAttribute self; From bc478708dd1afa809c4197a867ba018d0c6395e7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 4 Nov 2024 19:38:39 +0100 Subject: [PATCH 46/49] Let's see what breaks --- .../targets/Microsoft.Android.Sdk.BuildOrder.targets | 1 + src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index d937c238c6f..ee627e65d03 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -62,6 +62,7 @@ properties that determine build ordering. <_PrepareBuildApkDependsOnTargets> _SetLatestTargetFrameworkVersion; _GetLibraryImports; + _RemoveRegisterAttribute; _ResolveAssemblies; _ResolveSatellitePaths; _CreatePackageWorkspace; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 5f374dc948b..5d5033b65b4 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1867,7 +1867,6 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; - _RemoveRegisterAttribute; _CreateApplicationSharedLibraries; _GetMonoPlatformJarPath; _GetLibraryImports; From 8dfab2e743c0e9c5c67decf8de909ffcc9f37d0f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Nov 2024 09:32:05 +0100 Subject: [PATCH 47/49] Revert "Let's see what breaks" This reverts commit bc478708dd1afa809c4197a867ba018d0c6395e7. It broke 481 tests --- .../targets/Microsoft.Android.Sdk.BuildOrder.targets | 1 - src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index ee627e65d03..d937c238c6f 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -62,7 +62,6 @@ properties that determine build ordering. <_PrepareBuildApkDependsOnTargets> _SetLatestTargetFrameworkVersion; _GetLibraryImports; - _RemoveRegisterAttribute; _ResolveAssemblies; _ResolveSatellitePaths; _CreatePackageWorkspace; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 5d5033b65b4..5f374dc948b 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1867,6 +1867,7 @@ because xbuild doesn't support framework reference assemblies. <_CompileToDalvikDependsOnTargets> _CompileJava; + _RemoveRegisterAttribute; _CreateApplicationSharedLibraries; _GetMonoPlatformJarPath; _GetLibraryImports; From 66179bc90e7399c54fd6c2fce06c0dae500131e2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Nov 2024 13:33:26 +0100 Subject: [PATCH 48/49] Restore [Require] --- .../Tasks/GeneratePackageManagerJava.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index c3baa52f288..3c26d933e5f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -62,7 +62,7 @@ public class GeneratePackageManagerJava : AndroidTask public bool EnablePreloadAssembliesDefault { get; set; } // This property should be required but it will require modifying `monodroid` first - //[Required] + [Required] public string AndroidBinUtilsDirectory { get; set; } public bool EnableMarshalMethods { get; set; } From c584dbd83fbdc2bdb7b93be28546820715f10c3b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Nov 2024 16:41:18 +0100 Subject: [PATCH 49/49] Remove comment, no longer valid --- .../Tasks/GeneratePackageManagerJava.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 3c26d933e5f..58402e57167 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -61,7 +61,6 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } - // This property should be required but it will require modifying `monodroid` first [Required] public string AndroidBinUtilsDirectory { get; set; }