Embed some data files in libxamarin-app.so

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets

@@ -29,4 +29,5 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets.
   <Import Project="Microsoft.Android.Sdk.Publish.targets" />
   <Import Project="Microsoft.Android.Sdk.RuntimeConfig.targets" />
   <Import Project="Microsoft.Android.Sdk.Tooling.targets" />
+  <Import Project="Microsoft.Android.Sdk.AssemblyStores.targets" />
 </Project>

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyStores.targets

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <UsingTask TaskName="Xamarin.Android.Tasks.CreateEmbeddedAssemblyStore" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+  <UsingTask TaskName="Xamarin.Android.Tasks.PrepareAbiItems" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+
+  <Target Name="_PrepareCreateEmbeddedAssemblyStoreOutputItems">
+    <PrepareAbiItems
+      BuildTargetAbis="@(_BuildTargetAbis)"
+      NativeSourcesDir="$(_NativeAssemblySourceDir)"
+      Mode="EmbeddedAssemblyStore">
+      <Output TaskParameter="AssemblySources" ItemName="_EmbeddedAssemblyStoreSourceFiles" />
+    </PrepareAbiItems>
+
+    <ItemGroup>
+      <_CreateEmbeddedAssemblyStoreAssembly Include="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths);@(_ShrunkFrameworkAssemblies)"/>
+      <_CreateEmbeddedAssemblyStoreAssembly Condition=" '@(_CreateEmbeddedAssemblyStoreAssembly->Count())' == '0' " Include="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="_CreateEmbeddedAssemblyStore"
+          DependsOnTargets="_PrepareCreateEmbeddedAssemblyStoreOutputItems"
+          Inputs="@(_CreateEmbeddedAssemblyStoreAssembly)"
+          Outputs="@(_EmbeddedAssemblyStoreSourceFiles)">
+    <CreateEmbeddedAssemblyStore
+        AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)"
+        AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)"
+        AssemblySourcesDir="$(IntermediateOutputPath)android"
+        AssemblyStoreEmbeddedInRuntime="$(_AndroidEmbedAssemblyStoreInRuntime)"
+        CompressedAssembliesDir="$(_AndroidCompressedAssembliesDir)\test\"
+        Debug="$(AndroidIncludeDebugSymbols)"
+        EnableCompression="$(AndroidEnableAssemblyCompression)"
+        ProjectFullPath="$(MSBuildProjectFullPath)"
+        ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)"
+        ResolvedFrameworkAssemblies="@(_ShrunkFrameworkAssemblies)"
+        SupportedAbis="@(_BuildTargetAbis)" />
+
+    <ItemGroup>
+      <FileWrites Include="@(_EmbeddedAssemblyStoreSourceFiles)" />
+    </ItemGroup>
+  </Target>
+</Project>

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets

@@ -62,7 +62,6 @@ properties that determine build ordering.
     <_PrepareBuildApkDependsOnTargets>
       _SetLatestTargetFrameworkVersion;
       _GetLibraryImports;
-      _RemoveRegisterAttribute;
       _ResolveAssemblies;
       _ResolveSatellitePaths;
       _CreatePackageWorkspace;

src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs

@@ -2,12 +2,8 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Threading;
 
 using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-using Xamarin.Android.Tools;
 using Microsoft.Android.Build.Tasks;
 
 namespace Xamarin.Android.Tasks
@@ -37,109 +33,34 @@ sealed class Config
 
 		public override System.Threading.Tasks.Task RunTaskAsync ()
 		{
-			return this.WhenAll (GetAssemblerConfigs (), RunAssembler);
+			var context = new NativeAssemblerCompilation.AssemblerRunContext (
+				Log,
+				Path.GetFullPath (WorkingDirectory),
+				registerForCancellation: RegisterForCancellation,
+				cancel: Cancel
+			);
+
+			return this.WhenAll (
+				GetAssemblerConfigs (),
+				(NativeAssemblerCompilation.AssemblerConfig config) => NativeAssemblerCompilation.RunAssembler (context, config)
+			);
 		}
 
-		void RunAssembler (Config config)
+		void RegisterForCancellation (Process proc)
 		{
-			var stdout_completed = new ManualResetEvent (false);
-			var stderr_completed = new ManualResetEvent (false);
-			var psi = new ProcessStartInfo () {
-				FileName = config.AssemblerPath,
-				Arguments = config.AssemblerOptions,
-				UseShellExecute = false,
-				RedirectStandardOutput = true,
-				RedirectStandardError = true,
-				CreateNoWindow = true,
-				WindowStyle = ProcessWindowStyle.Hidden,
-				WorkingDirectory = WorkingDirectory,
-			};
-
-			string assemblerName = Path.GetFileName (config.AssemblerPath);
-			LogDebugMessage ($"[LLVM llc] {psi.FileName} {psi.Arguments}");
-
-			var stdoutLines = new List<string> ();
-			var stderrLines = new List<string> ();
-
-			using (var proc = new Process ()) {
-				proc.OutputDataReceived += (s, e) => {
-					if (e.Data != null) {
-						OnOutputData (assemblerName, s, e);
-						stdoutLines.Add (e.Data);
-					} else
-						stdout_completed.Set ();
-				};
-
-				proc.ErrorDataReceived += (s, e) => {
-					if (e.Data != null) {
-						OnErrorData (assemblerName, s, e);
-						stderrLines.Add (e.Data);
-					} else
-						stderr_completed.Set ();
-				};
-
-				proc.StartInfo = psi;
-				proc.Start ();
-				proc.BeginOutputReadLine ();
-				proc.BeginErrorReadLine ();
-				CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } });
-				proc.WaitForExit ();
-
-				if (psi.RedirectStandardError)
-					stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
-
-				if (psi.RedirectStandardOutput)
-					stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
-
-				if (proc.ExitCode != 0) {
-					var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines);
-					LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ());
-					Cancel ();
+			CancellationToken.Register (() => {
+				try {
+					proc.Kill ();
+				} catch (Exception) {
 				}
-			}
+			});
 		}
 
-		IEnumerable<Config> GetAssemblerConfigs ()
+		IEnumerable<NativeAssemblerCompilation.AssemblerConfig> GetAssemblerConfigs ()
 		{
-			const string assemblerOptions =
-				"-O2 " +
-				"--debugger-tune=lldb " + // NDK uses lldb now
-				"--debugify-level=location+variables " +
-				"--fatal-warnings " +
-				"--filetype=obj " +
-				"--relocation-model=pic";
-			string llcPath = Path.Combine (AndroidBinUtilsDirectory, "llc");
-
 			foreach (ITaskItem item in Sources) {
-				// We don't need the directory since our WorkingDirectory is where all the sources are
-				string sourceFile = Path.GetFileName (item.ItemSpec);
-				string outputFile = QuoteFileName (sourceFile.Replace (".ll", ".o"));
-				string executableDir = Path.GetDirectoryName (llcPath);
-				string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath));
-
-				yield return new Config {
-					InputSource = item.ItemSpec,
-					AssemblerPath = Path.Combine (executableDir, executableName),
-					AssemblerOptions = $"{assemblerOptions} -o={outputFile} {QuoteFileName (sourceFile)}",
-				};
+				yield return NativeAssemblerCompilation.GetAssemblerConfig (AndroidBinUtilsDirectory, item, stripFilePaths: true);
 			}
 		}
-
-		void OnOutputData (string assemblerName, object sender, DataReceivedEventArgs e)
-		{
-			LogDebugMessage ($"[{assemblerName} stdout] {e.Data}");
-		}
-
-		void OnErrorData (string assemblerName, object sender, DataReceivedEventArgs e)
-		{
-			LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High);
-		}
-
-		static string QuoteFileName (string fileName)
-		{
-			var builder = new CommandLineBuilder ();
-			builder.AppendFileNameIfNotNull (fileName);
-			return builder.ToString ();
-		}
 	}
 }

src/Xamarin.Android.Build.Tasks/Tasks/CreateEmbeddedAssemblyStore.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+using Microsoft.Android.Build.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Xamarin.Android.Tools;
+
+namespace Xamarin.Android.Tasks;
+
+public class CreateEmbeddedAssemblyStore : AndroidTask
+{
+	public override string TaskPrefix => "CEAS";
+
+	[Required]
+	public string AndroidBinUtilsDirectory { get; set; }
+
+	[Required]
+	public string AppSharedLibrariesDir { get; set; }
+
+	[Required]
+	public string AssemblySourcesDir { get; set; }
+
+	[Required]
+	public string CompressedAssembliesDir { get; set; }
+
+	[Required]
+	public bool AssemblyStoreEmbeddedInRuntime { get; set; }
+
+	[Required]
+	public bool Debug { get; set; }
+
+	[Required]
+	public bool EnableCompression { get; set; }
+
+	[Required]
+	public string ProjectFullPath { get; set; }
+
+	[Required]
+	public ITaskItem[] ResolvedUserAssemblies { get; set; }
+
+	[Required]
+	public ITaskItem[] ResolvedFrameworkAssemblies { get; set; }
+
+	[Required]
+	public string [] SupportedAbis { get; set; }
+
+	public override bool RunTask ()
+	{
+		if (AssemblyStoreEmbeddedInRuntime) {
+			return EmbedAssemblyStore ();
+		}
+
+		// Generate sources to satisfy libmonodroid's ABI requirements
+		foreach (string abi in SupportedAbis) {
+			ELFEmbeddingHelper.EmbedBinary (
+				Log,
+				abi,
+				AndroidBinUtilsDirectory,
+				inputFile: null,
+				ELFEmbeddingHelper.KnownEmbedItems.AssemblyStore,
+				AssemblySourcesDir,
+				missingContentOK: true
+			);
+		}
+
+		return !Log.HasLoggedErrors;
+	}
+
+	bool EmbedAssemblyStore ()
+	{
+		bool compress = !Debug && EnableCompression;
+		IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>>? compressedAssembliesInfo = null;
+
+		if (compress) {
+			string key = CompressedAssemblyInfo.GetKey (ProjectFullPath);
+			Log.LogDebugMessage ($"[{TaskPrefix}] Retrieving assembly compression info with key '{key}'");
+			compressedAssembliesInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>>> (key, RegisteredTaskObjectLifetime.Build);
+			if (compressedAssembliesInfo == null) {
+				throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed.");
+			}
+		}
+
+		var storeBuilder = new AssemblyStoreBuilder (Log);
+
+		// Add user assemblies
+		AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, DoAddAssembliesFromArchCollection);
+
+		// Add framework assemblies
+		AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection);
+
+		Dictionary<AndroidTargetArch, string> assemblyStorePaths = storeBuilder.Generate (Path.Combine (AppSharedLibrariesDir, "embedded"));
+		foreach (var kvp in assemblyStorePaths) {
+			string abi = MonoAndroidHelper.ArchToAbi (kvp.Key);
+			string inputFile = kvp.Value;
+
+			ELFEmbeddingHelper.EmbedBinary (
+				Log,
+				abi,
+				AndroidBinUtilsDirectory,
+				inputFile,
+				ELFEmbeddingHelper.KnownEmbedItems.AssemblyStore,
+				AssemblySourcesDir,
+				missingContentOK: false
+			);
+		}
+
+		return !Log.HasLoggedErrors;
+
+		void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly)
+		{
+			string sourcePath = CompressAssembly (assembly);
+			storeBuilder.AddAssembly (sourcePath, assembly, includeDebugSymbols: Debug);
+			return;
+		}
+
+		string CompressAssembly (ITaskItem assembly)
+		{
+			if (!compress) {
+				return assembly.ItemSpec;
+			}
+
+			return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, CompressedAssembliesDir);
+		}
+	}
+}

src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs

@@ -66,7 +66,7 @@ void GenerateCompressedAssemblySources ()
 
 					var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly);
 					if (assemblies.ContainsKey (assemblyKey)) {
-						Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})");
+						Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetItemAbi(assembly)})");
 						continue;
 					}