diff --git a/Documentation/building/configuration.md b/Documentation/building/configuration.md index 9e409096049..c605a3dc277 100644 --- a/Documentation/building/configuration.md +++ b/Documentation/building/configuration.md @@ -118,6 +118,12 @@ Overridable MSBuild properties include: assemblies placed in the APK will be compressed in `Release` builds. `Debug` builds are not affected. + * `$(AndroidEnableAssemblyStoreDecompressionCache)`: Defaults to `False`. When + enabled for a CoreCLR `Release` build, decompressed assemblies are cached in + the app's Android code-cache directory and mapped from there on subsequent + launches. The cache consumes additional on-device storage and is rebuilt + after app or platform updates. + ## Options suitable for local development ### Native runtime (`src/native`) diff --git a/Documentation/project-docs/AssemblyStores.md b/Documentation/project-docs/AssemblyStores.md index f2eb29aec9d..8893d249677 100644 --- a/Documentation/project-docs/AssemblyStores.md +++ b/Documentation/project-docs/AssemblyStores.md @@ -97,6 +97,7 @@ The header is a fixed-size structure at the beginning of each assembly store fil - **ENTRY_COUNT** (`uint32_t`) - Number of assemblies in the store - **INDEX_ENTRY_COUNT** (`uint32_t`) - Number of entries in the index (typically `ENTRY_COUNT * 2`) - **INDEX_SIZE** (`uint32_t`) - Index size in bytes +- **CONTENT_ID** (`uint64_t`) - Deterministic xxHash3 of everything after the header ## [INDEX] @@ -154,6 +155,7 @@ All kinds of stores share the following header format: uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; Individual fields have the following meanings: @@ -165,6 +167,7 @@ Individual fields have the following meanings: table, see below) - `index_entry_count`: number of entries in the index - `index_size`: index size in bytes + - `content_id`: deterministic xxHash3 of the index, descriptors, names, and assembly data ## Assembly descriptor table @@ -252,6 +255,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; ``` diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs index 52404559199..d9aa4171736 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs @@ -59,6 +59,7 @@ public class GenerateNativeApplicationConfigSources : AndroidTask public bool EnableMarshalMethods { get; set; } public bool EnableManagedMarshalMethodsLookup { get; set; } + public bool AndroidEnableAssemblyStoreDecompressionCache { get; set; } public string? RuntimeConfigBinFilePath { get; set; } public string ProjectRuntimeConfigFilePath { get; set; } = String.Empty; public string? BoundExceptionType { get; set; } @@ -285,6 +286,7 @@ static bool ShouldSkipAssembly (ITaskItem assembly) MarshalMethodsEnabled = EnableMarshalMethods, ManagedMarshalMethodsLookupEnabled = EnableManagedMarshalMethodsLookup, IgnoreSplitConfigs = ShouldIgnoreSplitConfigs (), + AssemblyStoreDecompressionCacheEnabled = AndroidEnableAssemblyStoreDecompressionCache, }; } else { appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (envBuilder.EnvironmentVariables, envBuilder.SystemProperties, Log) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs new file mode 100644 index 00000000000..466f52e15df --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/CreateAssemblyStoreTests.cs @@ -0,0 +1,57 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Hashing; +using System.Linq; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NUnit.Framework; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Build.Tests.Tasks; + +[TestFixture] +public class CreateAssemblyStoreTests : BaseTest +{ + [Test] + public void ContentIdMatchesStoreContents () + { + string testDirectory = Path.Combine (Root, "temp", nameof (ContentIdMatchesStoreContents)); + Directory.CreateDirectory (testDirectory); + + string assemblyPath = Path.Combine (testDirectory, "Example.dll.zst"); + byte [] assemblyData = [1, 3, 3, 7, 9, 11, 17, 23]; + File.WriteAllBytes (assemblyPath, assemblyData); + + var metadata = new Dictionary { + ["Abi"] = "arm64-v8a", + }; + var task = new CreateAssemblyStore { + BuildEngine = new MockBuildEngine (TestContext.Out), + AppSharedLibrariesDir = Path.Combine (testDirectory, "stores"), + ResolvedFrameworkAssemblies = [], + ResolvedUserAssemblies = [new TaskItem (assemblyPath, metadata)], + SupportedAbis = ["arm64-v8a"], + TargetRuntime = "CoreCLR", + UseAssemblyStore = true, + }; + + Assert.IsTrue (task.Execute (), "CreateAssemblyStore should succeed."); + + string storePath = task.AssembliesToAddToArchive.Single ().ItemSpec; + byte [] store = File.ReadAllBytes (storePath); + using var reader = new BinaryReader (new MemoryStream (store)); + Assert.AreEqual (0x41424158u, reader.ReadUInt32 (), "Unexpected assembly store magic."); + Assert.AreEqual (0x80010004u, reader.ReadUInt32 (), "Unexpected arm64 assembly store version."); + reader.BaseStream.Seek (3 * sizeof (uint), SeekOrigin.Current); + ulong contentId = reader.ReadUInt64 (); + + Assert.AreEqual ( + XxHash3.HashToUInt64 (store.AsSpan (5 * sizeof (uint) + sizeof (ulong))), + contentId, + "The content ID should hash everything after the assembly store header." + ); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs new file mode 100644 index 00000000000..bb8e295f48d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GenerateNativeApplicationConfigSourcesTests.cs @@ -0,0 +1,46 @@ +#nullable enable +using System.IO; + +using Microsoft.Build.Utilities; +using NUnit.Framework; +using Xamarin.Android.Tasks; +using Xamarin.ProjectTools; + +namespace Xamarin.Android.Build.Tests.Tasks; + +[TestFixture] +public class GenerateNativeApplicationConfigSourcesTests : BaseTest +{ + [TestCase (false)] + [TestCase (true)] + public void AssemblyStoreDecompressionCacheSettingIsEmitted (bool enabled) + { + string outputRoot = Path.Combine (Root, "temp", $"{nameof (AssemblyStoreDecompressionCacheSettingIsEmitted)}-{enabled}"); + string monoAndroidPath = Path.Combine (TestEnvironment.MonoAndroidFrameworkDirectory, "Mono.Android.dll"); + FileAssert.Exists (monoAndroidPath); + + var task = new GenerateNativeApplicationConfigSources { + BuildEngine = new MockBuildEngine (TestContext.Out), + ResolvedAssemblies = [new TaskItem (monoAndroidPath)], + EnvironmentOutputDirectory = Path.Combine (outputRoot, "android"), + SupportedAbis = ["arm64-v8a"], + AndroidPackageName = "com.microsoft.android.cachetest", + EnablePreloadAssembliesDefault = false, + TargetsCLR = true, + AndroidRuntime = "CoreCLR", + UseAssemblyStore = true, + AndroidEnableAssemblyStoreDecompressionCache = enabled, + }; + + Assert.IsTrue (task.Execute (), "GenerateNativeApplicationConfigSources should succeed."); + + var environmentFiles = EnvironmentHelper.GatherEnvironmentFiles ( + outputRoot, + "arm64-v8a", + required: true, + runtime: AndroidRuntime.CoreCLR + ); + var config = (EnvironmentHelper.ApplicationConfig_CoreCLR)EnvironmentHelper.ReadApplicationConfig (environmentFiles, AndroidRuntime.CoreCLR); + Assert.AreEqual (enabled, config.assembly_store_decompression_cache_enabled); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 927e4ccad70..0c84196f7ff 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -62,9 +62,10 @@ public sealed class ApplicationConfig_CoreCLR : IApplicationConfig public uint jni_remapping_replacement_method_index_entry_count; public string android_package_name = String.Empty; public bool managed_marshal_methods_lookup_enabled; + public bool assembly_store_decompression_cache_enabled; } - const uint ApplicationConfigFieldCount_CoreCLR = 20; + const uint ApplicationConfigFieldCount_CoreCLR = 21; // This must be identical to the ApplicationConfig structure in src/native/mono/xamarin-app-stub/xamarin-app.hh public sealed class ApplicationConfig_MonoVM : IApplicationConfig @@ -407,6 +408,11 @@ static IApplicationConfig ReadApplicationConfig_CoreCLR (EnvironmentFile envFile AssertFieldType (envFile.Path, parser.SourceFilePath, ".byte", field [0], item.LineNumber); ret.managed_marshal_methods_lookup_enabled = ConvertFieldToBool ("managed_marshal_methods_lookup_enabled", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; + + case 20: // assembly_store_decompression_cache_enabled: bool / .byte + AssertFieldType (envFile.Path, parser.SourceFilePath, ".byte", field [0], item.LineNumber); + ret.assembly_store_decompression_cache_enabled = ConvertFieldToBool ("assembly_store_decompression_cache_enabled", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + break; } fieldCount++; } @@ -771,6 +777,7 @@ static void AssertApplicationConfigIsIdentical (ApplicationConfig_CoreCLR firstA Assert.AreEqual (firstAppConfig.environment_variable_count, secondAppConfig.environment_variable_count, $"Field 'environment_variable_count' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); Assert.AreEqual (firstAppConfig.system_property_count, secondAppConfig.system_property_count, $"Field 'system_property_count' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); Assert.AreEqual (firstAppConfig.android_package_name, secondAppConfig.android_package_name, $"Field 'android_package_name' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); + Assert.AreEqual (firstAppConfig.assembly_store_decompression_cache_enabled, secondAppConfig.assembly_store_decompression_cache_enabled, $"Field 'assembly_store_decompression_cache_enabled' has different value in environment file '{secondEnvFile}' than in environment file '{firstEnvFile}'"); } static void AssertApplicationConfigIsIdentical (ApplicationConfig_MonoVM firstAppConfig, string firstEnvFile, ApplicationConfig_MonoVM secondAppConfig, string secondEnvFile) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs index b48178218f1..4f61bfb7316 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigCLR.cs @@ -51,4 +51,5 @@ sealed class ApplicationConfigCLR public uint jni_remapping_replacement_method_index_entry_count; public string android_package_name = String.Empty; public bool managed_marshal_methods_lookup_enabled; + public bool assembly_store_decompression_cache_enabled; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs index f4ef94d75b1..ee9064bd1d0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs @@ -287,6 +287,7 @@ sealed class DsoCacheState public bool MarshalMethodsEnabled { get; set; } public bool ManagedMarshalMethodsLookupEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } + public bool AssemblyStoreDecompressionCacheEnabled { get; set; } public ApplicationConfigNativeAssemblyGeneratorCLR (IDictionary environmentVariables, IDictionary systemProperties, IDictionary? runtimeProperties, TaskLoggingHelper log) @@ -373,6 +374,7 @@ protected override void Construct (LlvmIrModule module) jni_remapping_replacement_type_count = (uint)JniRemappingReplacementTypeCount, jni_remapping_replacement_method_index_entry_count = (uint)JniRemappingReplacementMethodIndexEntryCount, android_package_name = AndroidPackageName, + assembly_store_decompression_cache_enabled = AssemblyStoreDecompressionCacheEnabled, }; application_config = new StructureInstance (applicationConfigStructureInfo, app_cfg); module.AddGlobalVariable ("application_config", application_config); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs index 9df30db6303..7a30ec231b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -5,7 +5,7 @@ partial class AssemblyStoreGenerator { sealed class AssemblyStoreHeader { - public const uint NativeSize = 5 * sizeof (uint); + public const uint NativeSize = 5 * sizeof (uint) + sizeof (ulong); public readonly uint magic = ASSEMBLY_STORE_MAGIC; public readonly uint version; @@ -14,17 +14,19 @@ sealed class AssemblyStoreHeader // Index size in bytes public readonly uint index_size; + public readonly ulong content_id; - public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size) + public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) { this.version = version; this.entry_count = entry_count; this.index_entry_count = index_entry_count; this.index_size = index_size; + this.content_id = content_id; } #if XABT_TESTS - public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) - : this (version, entry_count, index_entry_count, index_size) + public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) + : this (version, entry_count, index_entry_count, index_size, content_id) { this.magic = magic; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index bad4cacb3f2..0a959f67460 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Hashing; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; @@ -28,6 +29,7 @@ namespace Xamarin.Android.Tasks; // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name @@ -53,8 +55,8 @@ partial class AssemblyStoreGenerator const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones - const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000003; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant - const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000003; + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000004; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000004; const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; @@ -122,7 +124,7 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List is64Bit ? AssemblyStoreIndexEntry.NativeSize64 : AssemblyStoreIndexEntry.NativeSize32; } + static ulong ComputeContentId (Stream stream) + { + stream.Seek (AssemblyStoreHeader.NativeSize, SeekOrigin.Begin); + + var hash = new XxHash3 (); + byte [] buffer = new byte [64 * 1024]; + int bytesRead; + while ((bytesRead = stream.Read (buffer, 0, buffer.Length)) > 0) { + hash.Append (buffer.AsSpan (0, bytesRead)); + } + + return hash.GetCurrentHashAsUInt64 (); + } + void CopyData (FileInfo? src, Stream dest, string storePath) { if (src == null) { @@ -239,6 +263,7 @@ void WriteHeader (BinaryWriter writer, AssemblyStoreHeader header) writer.Write (header.entry_count); writer.Write (header.index_entry_count); writer.Write (header.index_size); + writer.Write (header.content_id); } #if XABT_TESTS AssemblyStoreHeader ReadHeader (BinaryReader reader) @@ -249,8 +274,9 @@ AssemblyStoreHeader ReadHeader (BinaryReader reader) uint entry_count = reader.ReadUInt32 (); uint index_entry_count = reader.ReadUInt32 (); uint index_size = reader.ReadUInt32 (); + ulong content_id = reader.ReadUInt64 (); - return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size); + return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size, content_id); } #endif diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 03084bd6c37..bed73a09f8c 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -166,6 +166,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. False Android.App.Fragment True + False False <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> @@ -1802,6 +1803,7 @@ because xbuild doesn't support framework reference assemblies. BoundExceptionType="$(AndroidBoundExceptionType)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UseAssemblyStore="$(_AndroidUseAssemblyStore)" + AndroidEnableAssemblyStoreDecompressionCache="$(AndroidEnableAssemblyStoreDecompressionCache)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" EnableManagedMarshalMethodsLookup="$(_AndroidUseManagedMarshalMethodsLookup)" CustomBundleConfigFile="$(AndroidBundleConfigurationFile)" diff --git a/src/java-runtime/java/mono/android/clr/MonoPackageManager.java b/src/java-runtime/java/mono/android/clr/MonoPackageManager.java index 4cfa0ea2169..5592cc2b68e 100644 --- a/src/java-runtime/java/mono/android/clr/MonoPackageManager.java +++ b/src/java-runtime/java/mono/android/clr/MonoPackageManager.java @@ -52,6 +52,7 @@ public static void LoadApplication (Context context) String language = locale.getLanguage () + "-" + locale.getCountry (); String filesDir = context.getFilesDir ().getAbsolutePath (); String cacheDir = context.getCacheDir ().getAbsolutePath (); + String codeCacheDir = context.getCodeCacheDir ().getAbsolutePath (); String dataDir = getNativeLibraryPath (context); ClassLoader loader = context.getClassLoader (); String runtimeDir = getNativeLibraryPath (runtimePackage); @@ -66,7 +67,7 @@ public static void LoadApplication (Context context) // // Should the order change here, src/native/clr/include/constants.hh must be updated accordingly // - String[] appDirs = new String[] {filesDir, cacheDir, dataDir}; + String[] appDirs = new String[] {filesDir, cacheDir, dataDir, codeCacheDir}; boolean haveSplitApks = runtimePackage.splitSourceDirs != null && runtimePackage.splitSourceDirs.length > 0; System.loadLibrary("monodroid"); diff --git a/src/native/clr/host/assembly-store.cc b/src/native/clr/host/assembly-store.cc index 2fdc7119d15..c95b4ff9c77 100644 --- a/src/native/clr/host/assembly-store.cc +++ b/src/native/clr/host/assembly-store.cc @@ -1,7 +1,20 @@ +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include + #include #include +#include #include #include #include @@ -10,6 +23,441 @@ using namespace xamarin::android; +namespace { + namespace asm_cache { + constexpr std::string_view CACHE_DIR_NAME = "decompressed-assembly-cache-v1"sv; + constexpr uint32_t CACHE_FILE_MAGIC = 0x43434158; // 'XACC', little-endian + constexpr uint32_t CACHE_FILE_FORMAT_VERSION = 1; + constexpr size_t MAX_QUEUED_BYTES = 32uz * 1024uz * 1024uz; + + struct [[gnu::packed]] CacheFileFooter final + { + uint32_t magic; + uint32_t version; + uint64_t store_id; + uint64_t payload_hash; + uint32_t descriptor_index; + uint32_t payload_size; + }; + + static_assert (sizeof (CacheFileFooter) == 32uz); + + struct WriteRequest final + { + std::string path; + std::unique_ptr data; + size_t size; + }; + + enum class WriteResult + { + Succeeded, + Skipped, + Failed, + }; + + std::mutex state_lock; + std::deque write_queue; + std::string cache_dir; + std::unique_ptr tracking; + size_t queued_bytes = 0; + uint64_t store_id = 0; + bool initialized = false; + bool enabled = false; + bool writes_enabled = false; + bool writer_running = false; + + auto hash_payload (const uint8_t *data, size_t size) noexcept -> uint64_t + { + return static_cast(XXH3_64bits (data, size)); + } + + bool write_fully (int fd, const uint8_t *buf, size_t len) noexcept + { + size_t off = 0; + while (off < len) { + ssize_t n = write (fd, buf + off, len - off); + if (n < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (n == 0) { + errno = EIO; + return false; + } + off += static_cast(n); + } + return true; + } + + void log_file_error (std::string_view operation, std::string const& path, int error) noexcept + { + log_debug (LOG_ASSEMBLY, "Decompressed-assembly cache {} failed for '{}': {}"sv, operation, path, std::strerror (error)); + } + + auto write_cache_file (WriteRequest const& req) noexcept -> WriteResult + { + std::string tmp_path = req.path; + tmp_path.append (".tmp"sv); + + int fd; + do { + fd = open (tmp_path.c_str (), O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0600); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + log_file_error ("temporary-file creation"sv, req.path, errno); + return WriteResult::Failed; + } + + bool ok = write_fully (fd, req.data.get (), req.size); + int error = ok ? 0 : errno; + if (close (fd) != 0 && ok) { + ok = false; + error = errno; + } + + if (!ok) { + log_file_error ("write"sv, req.path, error); + unlink (tmp_path.c_str ()); + return WriteResult::Failed; + } + + int rename_result; + do { + rename_result = rename (tmp_path.c_str (), req.path.c_str ()); + } while (rename_result != 0 && errno == EINTR); + + if (rename_result != 0) { + error = errno; + // Another process can publish the shared, content-identical temp + // file first. Its rename consumes the temp path, so this writer + // has nothing left to publish. + if (error == ENOENT) { + return WriteResult::Skipped; + } + + log_file_error ("publish"sv, req.path, error); + unlink (tmp_path.c_str ()); + return WriteResult::Failed; + } + + return WriteResult::Succeeded; + } + + void clear_write_queue_locked () noexcept + { + for (WriteRequest const& request : write_queue) { + queued_bytes -= request.size; + } + write_queue.clear (); + } + + [[gnu::cold]] + auto writer_loop ([[maybe_unused]] void *arg) noexcept -> void* + { + while (true) { + WriteRequest request; + { + std::lock_guard lock (state_lock); + if (write_queue.empty ()) { + writer_running = false; + return nullptr; + } + + request = std::move (write_queue.front ()); + write_queue.pop_front (); + } + + size_t request_size = request.size; + WriteResult write_result = write_cache_file (request); + request.data.reset (); + + { + std::lock_guard lock (state_lock); + queued_bytes -= request_size; + if (write_result == WriteResult::Failed) { + writes_enabled = false; + clear_write_queue_locked (); + writer_running = false; + log_debug (LOG_ASSEMBLY, "Disabling decompressed-assembly cache writes after a persistence failure"sv); + return nullptr; + } + } + } + } + + bool start_writer_locked () noexcept + { + pthread_attr_t attributes; + int result = pthread_attr_init (&attributes); + bool attributes_initialized = result == 0; + if (result == 0) { + result = pthread_attr_setdetachstate (&attributes, PTHREAD_CREATE_DETACHED); + } + + pthread_t writer_thread; + if (result == 0) { + result = pthread_create (&writer_thread, &attributes, writer_loop, nullptr); + } + + if (attributes_initialized) { + pthread_attr_destroy (&attributes); + } + if (result != 0) { + log_debug (LOG_ASSEMBLY, "Failed to start decompressed-assembly cache writer: {}"sv, std::strerror (result)); + return false; + } + + return true; + } + + bool ensure_directory (std::string const& path) noexcept + { + if (mkdir (path.c_str (), 0700) == 0) { + return true; + } + + int error = errno; + if (error != EEXIST) { + log_file_error ("directory creation"sv, path, error); + return false; + } + + struct stat st {}; + if (lstat (path.c_str (), &st) != 0) { + log_file_error ("directory validation"sv, path, errno); + return false; + } + if (!S_ISDIR (st.st_mode)) { + log_file_error ("directory validation"sv, path, ENOTDIR); + return false; + } + + return true; + } + + void ensure_initialized (uint64_t assembly_store_id) noexcept + { + if (initialized) { + return; + } + initialized = true; + + bool cache_requested = application_config.assembly_store_decompression_cache_enabled; + + // Allow overriding the build setting at runtime for A/B benchmarking: + // adb shell setprop debug.net.asmcache 0 # off + // adb shell setprop debug.net.asmcache 1 # on + if (getenv ("XA_DISABLE_ASSEMBLY_CACHE") != nullptr) { + return; + } + { + dynamic_local_property_string prop_value; + if (AndroidSystem::monodroid_get_system_property ("debug.net.asmcache"sv, prop_value) > 0 && prop_value.get () != nullptr) { + if (prop_value.get ()[0] == '0') { + cache_requested = false; + } else if (prop_value.get ()[0] == '1') { + cache_requested = true; + } + } + } + + if (!cache_requested) { + return; + } + + std::string const& code_cache_dir = AndroidSystem::get_app_code_cache_dir (); + if (code_cache_dir.empty ()) { + return; + } + + cache_dir.assign (code_cache_dir); + cache_dir.append ("/"); + cache_dir.append (CACHE_DIR_NAME); + if (!ensure_directory (cache_dir)) { + return; + } + + store_id = assembly_store_id; + cache_dir.append ("/"); + cache_dir.append (std::format ("{:x}", store_id)); + if (!ensure_directory (cache_dir)) { + return; + } + + if (compressed_assembly_count > 0) { + tracking.reset (new (std::nothrow) uint8_t*[compressed_assembly_count]()); + } + + enabled = (tracking != nullptr); + if (!enabled) { + return; + } + + { + std::lock_guard lock (state_lock); + writes_enabled = true; + } + + log_debug ( + LOG_ASSEMBLY, + "Enabled decompressed-assembly cache at '{}'; store ID 0x{:x}; write queue limit {} bytes"sv, + cache_dir, + store_id, + MAX_QUEUED_BYTES + ); + } + + auto build_path (uint32_t descriptor_index) noexcept -> std::string + { + std::string path = cache_dir; + path.append ("/"); + path.append (std::to_string (descriptor_index)); + path.append (".bin"sv); + return path; + } + + auto try_load (uint32_t descriptor_index, std::string_view name, uint32_t expected_size) noexcept -> uint8_t* + { + if (!enabled) { + return nullptr; + } + + std::string path = build_path (descriptor_index); + int fd = open (path.c_str (), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) { + return nullptr; + } + + struct stat st {}; + if (fstat (fd, &st) != 0 || + !S_ISREG (st.st_mode) || + static_cast(st.st_size) != static_cast(expected_size) + sizeof (CacheFileFooter)) { + close (fd); + return nullptr; + } + + size_t map_size = static_cast(expected_size) + sizeof (CacheFileFooter); + // The runtime may modify the image, so keep those changes private while + // retaining clean file-backed pages until they are actually written. + void *mapped = mmap (nullptr, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + close (fd); + if (mapped == MAP_FAILED) { + return nullptr; + } + + CacheFileFooter footer {}; + memcpy (&footer, static_cast(mapped) + expected_size, sizeof (footer)); + if (footer.magic != CACHE_FILE_MAGIC || + footer.version != CACHE_FILE_FORMAT_VERSION || + footer.store_id != store_id || + footer.descriptor_index != descriptor_index || + footer.payload_size != expected_size || + footer.payload_hash != hash_payload (static_cast(mapped), expected_size)) { + munmap (mapped, map_size); + log_debug (LOG_ASSEMBLY, "Ignoring invalid decompressed-assembly cache entry for '{}'"sv, name); + return nullptr; + } + + return static_cast(mapped); + } + + void enqueue_write (uint32_t descriptor_index, std::string_view name, const uint8_t *data, size_t size) noexcept + { + if (!enabled) { + return; + } + + if (size > SIZE_MAX - sizeof (CacheFileFooter)) { + return; + } + size_t total = size + sizeof (CacheFileFooter); + + size_t bytes_queued = 0; + bool queue_full = false; + { + std::lock_guard lock (state_lock); + if (!writes_enabled) { + return; + } + if (total > MAX_QUEUED_BYTES || queued_bytes > MAX_QUEUED_BYTES - total) { + queue_full = true; + bytes_queued = queued_bytes; + } else { + queued_bytes += total; + } + } + + if (queue_full) { + if (total > MAX_QUEUED_BYTES) { + log_debug ( + LOG_ASSEMBLY, + "Skipping decompressed-assembly cache write for '{}': {} bytes exceed the {}-byte queue limit"sv, + name, + total, + MAX_QUEUED_BYTES + ); + } else { + log_debug ( + LOG_ASSEMBLY, + "Skipping decompressed-assembly cache write for '{}': {} of {} queue bytes are in use"sv, + name, + bytes_queued, + MAX_QUEUED_BYTES + ); + } + return; + } + + auto snapshot = std::unique_ptr (new (std::nothrow) uint8_t[total]); + if (snapshot == nullptr) { + std::lock_guard lock (state_lock); + queued_bytes -= total; + return; + } + // The runtime can modify the shared decompression buffer after this + // method returns, so the background writer needs an immutable copy. + memcpy (snapshot.get (), data, size); + + CacheFileFooter footer { + .magic = CACHE_FILE_MAGIC, + .version = CACHE_FILE_FORMAT_VERSION, + .store_id = store_id, + .payload_hash = hash_payload (snapshot.get (), size), + .descriptor_index = descriptor_index, + .payload_size = static_cast(size), + }; + memcpy (snapshot.get () + size, &footer, sizeof (footer)); + + WriteRequest req { + .path = build_path (descriptor_index), + .data = std::move (snapshot), + .size = total, + }; + + { + std::lock_guard lock (state_lock); + if (!writes_enabled) { + queued_bytes -= total; + return; + } + + write_queue.push_back (std::move (req)); + if (!writer_running) { + writer_running = true; + if (!start_writer_locked ()) { + writer_running = false; + writes_enabled = false; + clear_write_queue_locked (); + } + } + } + } + } // namespace asm_cache +} // anonymous namespace + [[gnu::always_inline]] void AssemblyStore::set_assembly_data_and_size (uint8_t* source_assembly_data, uint32_t source_assembly_data_size, uint8_t*& dest_assembly_data, uint32_t& dest_assembly_data_size) noexcept { @@ -26,7 +474,7 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co #if defined (RELEASE) auto header = reinterpret_cast(e.image_data); if (header->magic == COMPRESSED_DATA_MAGIC) { - log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); + log_debug (LOG_ASSEMBLY, "Resolving compressed assembly '{}' from the assembly store"sv, name); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.start_event (TimingEventKind::AssemblyDecompression); @@ -76,11 +524,25 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } uint8_t *data_buffer = uncompressed_assemblies_data_buffer + cad.buffer_offset; - if (!cad.loaded) { + uint32_t const descriptor_index = header->descriptor_index; + auto is_loaded = [&cad]() noexcept -> bool { + return __atomic_load_n (&cad.loaded, __ATOMIC_ACQUIRE); + }; + + // Resolves to the mmap'd cache file when this assembly was loaded from + // the on-device cache, otherwise to the shared decompression buffer. + auto resolve_data = [descriptor_index, data_buffer]() noexcept -> uint8_t* { + if (asm_cache::tracking != nullptr && asm_cache::tracking[descriptor_index] != nullptr) { + return asm_cache::tracking[descriptor_index]; + } + return data_buffer; + }; + + if (!is_loaded ()) { StartupAwareLock decompress_lock (assembly_decompress_mutex); - if (cad.loaded) { - set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); + if (is_loaded ()) { + set_assembly_data_and_size (resolve_data (), cad.uncompressed_file_size, assembly_data, assembly_data_size); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.end_event (true /* uses_more_info */); @@ -93,6 +555,8 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co return {assembly_data, assembly_data_size}; } + asm_cache::ensure_initialized (assembly_store_content_id); + if (header->uncompressed_length != cad.uncompressed_file_size) { if (header->uncompressed_length > cad.uncompressed_file_size) { Helpers::abort_application ( @@ -111,38 +575,59 @@ auto AssemblyStore::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData co } const char *data_start = pointer_add(e.image_data, sizeof(CompressedAssemblyHeader)); - size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ZSTD_isError (ret)) { - Helpers::abort_application ( - LOG_ASSEMBLY, - std::format ( - "Decompression of assembly {} failed: {}"sv, - name, - ZSTD_getErrorName (ret) - ) - ); - } + bool loaded_from_cache = false; + uint8_t *cached = asm_cache::try_load (descriptor_index, name, cad.uncompressed_file_size); + if (cached != nullptr) { + loaded_from_cache = true; + log_debug (LOG_ASSEMBLY, "Loaded decompressed assembly '{}' from the on-device cache"sv, name); + if (asm_cache::tracking != nullptr) { + asm_cache::tracking[descriptor_index] = cached; + } + } else { + log_debug (LOG_ASSEMBLY, "Decompressing assembly '{}' from the assembly store"sv, name); + size_t ret = ZSTD_decompress (data_buffer, cad.uncompressed_file_size, data_start, assembly_data_size); - if (ret != cad.uncompressed_file_size) { - Helpers::abort_application ( - LOG_ASSEMBLY, - std::format ( - "Decompression of assembly {} yielded a different size (expected {}, got {})"sv, - name, - cad.uncompressed_file_size, - static_cast(ret) - ) - ); + if (ZSTD_isError (ret)) { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Decompression of assembly {} failed: {}"sv, + name, + ZSTD_getErrorName (ret) + ) + ); + } + + if (ret != cad.uncompressed_file_size) { + Helpers::abort_application ( + LOG_ASSEMBLY, + std::format ( + "Decompression of assembly {} yielded a different size (expected {}, got {})"sv, + name, + cad.uncompressed_file_size, + static_cast(ret) + ) + ); + } + + asm_cache::enqueue_write (descriptor_index, name, data_buffer, cad.uncompressed_file_size); } - cad.loaded = true; + + __atomic_store_n (&cad.loaded, true, __ATOMIC_RELEASE); if (FastTiming::enabled ()) [[unlikely]] { internal_timing.end_event (true /* uses_more_info */); - internal_timing.add_more_info (name); + + dynamic_local_string msg; + msg.append (name); + if (loaded_from_cache) { + msg.append (" (decompressed cache hit)"sv); + } + internal_timing.add_more_info (msg); } } - set_assembly_data_and_size (data_buffer, cad.uncompressed_file_size, assembly_data, assembly_data_size); + set_assembly_data_and_size (resolve_data (), cad.uncompressed_file_size, assembly_data, assembly_data_size); } else #endif // def RELEASE { @@ -303,11 +788,12 @@ void AssemblyStore::map (int fd, std::string_view const& apk_path, std::string_v constexpr size_t header_size = sizeof(AssemblyStoreHeader); + assembly_store_content_id = header->content_id; assembly_store.data_start = static_cast(payload_start); assembly_store.assembly_count = header->entry_count; assembly_store.index_entry_count = header->index_entry_count; assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); - log_debug (LOG_ASSEMBLY, "Mapped assembly store {}"sv, get_full_store_path ()); + log_debug (LOG_ASSEMBLY, "Mapped assembly store {}; content ID 0x{:x}"sv, get_full_store_path (), assembly_store_content_id); } diff --git a/src/native/clr/host/host.cc b/src/native/clr/host/host.cc index 3e84ef930d8..fd614335e73 100644 --- a/src/native/clr/host/host.cc +++ b/src/native/clr/host/host.cc @@ -430,6 +430,7 @@ void Host::Java_mono_android_Runtime_initInternal ( AndroidSystem::detect_embedded_dso_mode (applicationDirs); AndroidSystem::set_running_in_emulator (isEmulator); AndroidSystem::set_primary_override_dir (files_dir); + AndroidSystem::set_app_code_cache_dir (applicationDirs[Constants::APP_DIRS_CODE_CACHE_DIR_INDEX]); AndroidSystem::create_update_dir (AndroidSystem::get_primary_override_dir ()); AndroidSystem::setup_environment (); Logger::init_reference_logging (AndroidSystem::get_primary_override_dir ()); diff --git a/src/native/clr/include/constants.hh b/src/native/clr/include/constants.hh index d9764c9cefb..5de37b8cfd8 100644 --- a/src/native/clr/include/constants.hh +++ b/src/native/clr/include/constants.hh @@ -114,6 +114,7 @@ namespace xamarin::android { static constexpr size_t APP_DIRS_FILES_DIR_INDEX = 0uz; static constexpr size_t APP_DIRS_CACHE_DIR_INDEX = 1uz; static constexpr size_t APP_DIRS_DATA_DIR_INDEX = 2uz; + static constexpr size_t APP_DIRS_CODE_CACHE_DIR_INDEX = 3uz; static inline constexpr size_t PROPERTY_VALUE_BUFFER_LEN = PROP_VALUE_MAX + 1uz; diff --git a/src/native/clr/include/host/assembly-store.hh b/src/native/clr/include/host/assembly-store.hh index eac5a30f5f1..a07eceaee7b 100644 --- a/src/native/clr/include/host/assembly-store.hh +++ b/src/native/clr/include/host/assembly-store.hh @@ -31,6 +31,7 @@ namespace xamarin::android { private: static inline AssemblyStoreIndexEntry *assembly_store_hashes = nullptr; + static inline uint64_t assembly_store_content_id = 0; static inline std::mutex assembly_decompress_mutex {}; }; } diff --git a/src/native/clr/include/runtime-base/android-system.hh b/src/native/clr/include/runtime-base/android-system.hh index b60871a93c4..3ddaee861b6 100644 --- a/src/native/clr/include/runtime-base/android-system.hh +++ b/src/native/clr/include/runtime-base/android-system.hh @@ -70,6 +70,16 @@ namespace xamarin::android { primary_override_dir = determine_primary_override_dir (home); } + static auto get_app_code_cache_dir () noexcept -> std::string const& + { + return app_code_cache_dir; + } + + static void set_app_code_cache_dir (jstring_wrapper& code_cache_dir) noexcept + { + app_code_cache_dir.assign (code_cache_dir.get_cstr ()); + } + static auto get_native_libraries_dir () noexcept -> std::string const& { return native_libraries_dir; @@ -146,6 +156,7 @@ namespace xamarin::android { static inline bool embedded_dso_mode_enabled = false; static inline std::string primary_override_dir; static inline std::string native_libraries_dir; + static inline std::string app_code_cache_dir; #if defined (DEBUG) static inline std::unordered_map bundled_properties; diff --git a/src/native/clr/include/xamarin-app.hh b/src/native/clr/include/xamarin-app.hh index 39fbb97f822..79f98a067ee 100644 --- a/src/native/clr/include/xamarin-app.hh +++ b/src/native/clr/include/xamarin-app.hh @@ -31,7 +31,7 @@ static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000; #endif // Increase whenever an incompatible change is made to the assembly store format -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 3 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 4 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian @@ -136,6 +136,7 @@ struct CompressedAssemblyDescriptor // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name @@ -167,6 +168,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; struct [[gnu::packed]] AssemblyStoreIndexEntry final @@ -231,6 +233,7 @@ struct ApplicationConfig uint32_t jni_remapping_replacement_method_index_entry_count; const char *android_package_name; bool managed_marshal_methods_lookup_enabled; + bool assembly_store_decompression_cache_enabled; }; struct RuntimeProperty diff --git a/src/native/clr/xamarin-app-stub/application_dso_stub.cc b/src/native/clr/xamarin-app-stub/application_dso_stub.cc index 14f33be4c29..9c4cc6502ee 100644 --- a/src/native/clr/xamarin-app-stub/application_dso_stub.cc +++ b/src/native/clr/xamarin-app-stub/application_dso_stub.cc @@ -71,6 +71,7 @@ const ApplicationConfig application_config = { .jni_remapping_replacement_method_index_entry_count = 2, .android_package_name = android_package_name, .managed_marshal_methods_lookup_enabled = false, + .assembly_store_decompression_cache_enabled = false, }; // TODO: migrate to std::string_view for these two diff --git a/src/native/mono/xamarin-app-stub/xamarin-app.hh b/src/native/mono/xamarin-app-stub/xamarin-app.hh index 2a21c1e827b..1a912d272d5 100644 --- a/src/native/mono/xamarin-app-stub/xamarin-app.hh +++ b/src/native/mono/xamarin-app-stub/xamarin-app.hh @@ -34,7 +34,7 @@ static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000; #endif // Increase whenever an incompatible change is made to the assembly store format -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 3 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 4 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian @@ -145,6 +145,7 @@ struct XamarinAndroidBundledAssembly // [ENTRY_COUNT] uint; number of entries in the store // [INDEX_ENTRY_COUNT] uint; number of entries in the index // [INDEX_SIZE] uint; index size in bytes +// [CONTENT_ID] ulong: deterministic hash of everything after the header // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name @@ -176,6 +177,7 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t entry_count; uint32_t index_entry_count; uint32_t index_size; // index size in bytes + uint64_t content_id; }; struct [[gnu::packed]] AssemblyStoreIndexEntry final diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 241f116a183..6028a7e862a 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -660,6 +660,76 @@ public void DeployToDevice ([Values] bool isRelease, [Values (AndroidRuntime.Cor Assert.IsTrue (didLaunch, "Activity should have started."); } + [Test] + public void AssemblyStoreDecompressionCacheMapsPersistedAssemblies () + { + if (IgnoreUnsupportedConfiguration (AndroidRuntime.CoreCLR, release: true)) { + return; + } + + var app = new XamarinAndroidApplicationProject (packageName: PackageUtils.MakePackageName (AndroidRuntime.CoreCLR, "assemblycache")) { + IsRelease = true, + }; + app.SetRuntime (AndroidRuntime.CoreCLR); + app.SetRuntimeIdentifiers (new [] { DeviceAbi }); + app.SetProperty ("AndroidEnableAssemblyStoreDecompressionCache", "true"); + app.AndroidManifest = app.AndroidManifest.Replace (" (uint)(5 * sizeof (uint) + ((version & ASSEMBLY_STORE_FORMAT_NUMBER_MASK) >= 4 ? sizeof (ulong) : 0)); - public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size, ulong content_id) { this.magic = magic; this.version = version; this.entry_count = entry_count; this.index_entry_count = index_entry_count; this.index_size = index_size; + this.content_id = content_id; } } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index e10d9a99ab4..d18b9404bd5 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -11,9 +11,12 @@ namespace Xamarin.Android.AssemblyStore; partial class StoreReader_V2 : AssemblyStoreReader { // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones - const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000003; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant - const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000003; + const uint ASSEMBLY_STORE_FORMAT_VERSION_3_64BIT = 0x80000003; + const uint ASSEMBLY_STORE_FORMAT_VERSION_3_32BIT = 0x00000003; + const uint ASSEMBLY_STORE_FORMAT_VERSION_4_64BIT = 0x80000004; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_4_32BIT = 0x00000004; const uint ASSEMBLY_STORE_FORMAT_VERSION_MASK = 0xF0000000; + const uint ASSEMBLY_STORE_FORMAT_NUMBER_MASK = 0x0000FFFF; const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; @@ -75,10 +78,14 @@ public StoreReader_V2 (Stream store, string path) : base (store, path) { supportedVersions = new HashSet { - ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_AARCH64, - ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_X64, - ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_ARM, - ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_X86, + ASSEMBLY_STORE_FORMAT_VERSION_3_64BIT | ASSEMBLY_STORE_ABI_AARCH64, + ASSEMBLY_STORE_FORMAT_VERSION_3_64BIT | ASSEMBLY_STORE_ABI_X64, + ASSEMBLY_STORE_FORMAT_VERSION_3_32BIT | ASSEMBLY_STORE_ABI_ARM, + ASSEMBLY_STORE_FORMAT_VERSION_3_32BIT | ASSEMBLY_STORE_ABI_X86, + ASSEMBLY_STORE_FORMAT_VERSION_4_64BIT | ASSEMBLY_STORE_ABI_AARCH64, + ASSEMBLY_STORE_FORMAT_VERSION_4_64BIT | ASSEMBLY_STORE_ABI_X64, + ASSEMBLY_STORE_FORMAT_VERSION_4_32BIT | ASSEMBLY_STORE_ABI_ARM, + ASSEMBLY_STORE_FORMAT_VERSION_4_32BIT | ASSEMBLY_STORE_ABI_X86, }; } @@ -124,8 +131,9 @@ protected override bool IsSupported () uint entry_count = reader.ReadUInt32 (); uint index_entry_count = reader.ReadUInt32 (); uint index_size = reader.ReadUInt32 (); + ulong content_id = (version & ASSEMBLY_STORE_FORMAT_NUMBER_MASK) >= 4 ? reader.ReadUInt64 () : 0; - header = new Header (magic, version, entry_count, index_entry_count, index_size); + header = new Header (magic, version, entry_count, index_entry_count, index_size, content_id); return true; } @@ -147,7 +155,7 @@ protected override void Prepare () AssemblyCount = header.entry_count; IndexEntryCount = header.index_entry_count; - StoreStream.Seek ((long)elfOffset + Header.NativeSize, SeekOrigin.Begin); + StoreStream.Seek ((long)elfOffset + header.NativeSize, SeekOrigin.Begin); using var reader = CreateReader (); var index = new List ();