misc: Implement address space size workarounds (#5191)

* misc: Implement address space size workarounds

This adds code to support userland with less than 39 bits of address
space available by testing reserving multiple sizes and reducing
guess address space when needed.

This is required for ARM64 support when the kernel is
configured to use 63..39 bits for kernel space.(meaning only 38 bits is available to userland)

* Address comments

* Fix 32 bits address space support and address more comments

src/Ryujinx.Cpu/AddressSpace.cs

@@ -5,7 +5,7 @@
 
 namespace Ryujinx.Cpu
 {
-    class AddressSpace : IDisposable
+    public class AddressSpace : IDisposable
     {
         private const ulong PageSize = 0x1000;
 
@@ -154,7 +154,9 @@ public int CompareTo(PrivateMapping other)
         public MemoryBlock Base { get; }
         public MemoryBlock Mirror { get; }
 
-        public AddressSpace(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages)
+        public ulong AddressSpaceSize { get; }
+
+        public AddressSpace(MemoryBlock backingMemory, MemoryBlock baseMemory, MemoryBlock mirrorMemory, ulong addressSpaceSize, bool supports4KBPages)
         {
             if (!supports4KBPages)
             {
@@ -163,17 +165,48 @@ public AddressSpace(MemoryBlock backingMemory, ulong asSize, bool supports4KBPag
                 _privateTree = new IntrusiveRedBlackTree<PrivateMapping>();
                 _treeLock = new object();
 
-                _mappingTree.Add(new Mapping(0UL, asSize, MappingType.None));
-                _privateTree.Add(new PrivateMapping(0UL, asSize, default));
+                _mappingTree.Add(new Mapping(0UL, addressSpaceSize, MappingType.None));
+                _privateTree.Add(new PrivateMapping(0UL, addressSpaceSize, default));
             }
 
             _backingMemory = backingMemory;
             _supports4KBPages = supports4KBPages;
 
+            Base = baseMemory;
+            Mirror = mirrorMemory;
+            AddressSpaceSize = addressSpaceSize;
+        }
+
+        public static bool TryCreate(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages, out AddressSpace addressSpace)
+        {
+            addressSpace = null;
+
             MemoryAllocationFlags asFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible;
 
-            Base = new MemoryBlock(asSize, asFlags);
-            Mirror = new MemoryBlock(asSize, asFlags);
+            ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36);
+
+            // Attempt to create the address space with expected size or try to reduce it until it succeed.
+            for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize >>= 1)
+            {
+                MemoryBlock baseMemory = null;
+                MemoryBlock mirrorMemory = null;
+
+                try
+                {
+                    baseMemory = new MemoryBlock(addressSpaceSize, asFlags);
+                    mirrorMemory = new MemoryBlock(addressSpaceSize, asFlags);
+                    addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize, supports4KBPages);
+
+                    break;
+                }
+                catch (OutOfMemoryException)
+                {
+                    baseMemory?.Dispose();
+                    mirrorMemory?.Dispose();
+                }
+            }
+
+            return addressSpace != null;
         }
 
         public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)

src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs

@@ -38,7 +38,8 @@ private enum HostMappedPtBits : ulong
         private readonly bool _unsafeMode;
 
         private readonly AddressSpace _addressSpace;
-        private readonly ulong _addressSpaceSize;
+
+        public ulong AddressSpaceSize { get; }
 
         private readonly PageTable<ulong> _pageTable;
 
@@ -62,21 +63,21 @@ private enum HostMappedPtBits : ulong
         /// <summary>
         /// Creates a new instance of the host mapped memory manager.
         /// </summary>
-        /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
-        /// <param name="addressSpaceSize">Size of the address space</param>
+        /// <param name="addressSpace">Address space instance to use</param>
         /// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param>
         /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
-        public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler = null)
+        public MemoryManagerHostMapped(AddressSpace addressSpace, bool unsafeMode, InvalidAccessHandler invalidAccessHandler)
         {
+            _addressSpace = addressSpace;
             _pageTable = new PageTable<ulong>();
             _invalidAccessHandler = invalidAccessHandler;
             _unsafeMode = unsafeMode;
-            _addressSpaceSize = addressSpaceSize;
+            AddressSpaceSize = addressSpace.AddressSpaceSize;
 
             ulong asSize = PageSize;
             int asBits = PageBits;
 
-            while (asSize < addressSpaceSize)
+            while (asSize < AddressSpaceSize)
             {
                 asSize <<= 1;
                 asBits++;
@@ -86,8 +87,6 @@ public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize
 
             _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
 
-            _addressSpace = new AddressSpace(backingMemory, asSize, Supports4KBPages);
-
             Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
             _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking);
         }
@@ -99,7 +98,7 @@ public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize
         /// <returns>True if the virtual address is part of the addressable space</returns>
         private bool ValidateAddress(ulong va)
         {
-            return va < _addressSpaceSize;
+            return va < AddressSpaceSize;
         }
 
         /// <summary>
@@ -111,7 +110,7 @@ private bool ValidateAddress(ulong va)
         private bool ValidateAddressAndSize(ulong va, ulong size)
         {
             ulong endVa = va + size;
-            return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
+            return endVa >= va && endVa >= size && endVa <= AddressSpaceSize;
         }
 
         /// <summary>

src/Ryujinx.HLE/HOS/ArmProcessContext.cs

@@ -25,7 +25,15 @@ class ArmProcessContext<T> : IArmProcessContext where T : class, IVirtualMemoryM
 
         public IVirtualMemoryManager AddressSpace => _memoryManager;
 
-        public ArmProcessContext(ulong pid, ICpuEngine cpuEngine, GpuContext gpuContext, T memoryManager, bool for64Bit)
+        public ulong AddressSpaceSize { get; }
+
+        public ArmProcessContext(
+            ulong pid,
+            ICpuEngine cpuEngine,
+            GpuContext gpuContext,
+            T memoryManager,
+            ulong addressSpaceSize,
+            bool for64Bit)
         {
             if (memoryManager is IRefCounted rc)
             {
@@ -38,6 +46,8 @@ public ArmProcessContext(ulong pid, ICpuEngine cpuEngine, GpuContext gpuContext,
             _gpuContext = gpuContext;
             _cpuContext = cpuEngine.CreateCpuContext(memoryManager, for64Bit);
             _memoryManager = memoryManager;
+
+            AddressSpaceSize = addressSpaceSize;
         }
 
         public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)

src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs

@@ -1,4 +1,5 @@
 using Ryujinx.Common.Configuration;
+using Ryujinx.Common.Logging;
 using Ryujinx.Cpu;
 using Ryujinx.Cpu.AppleHv;
 using Ryujinx.Cpu.Jit;
@@ -49,31 +50,49 @@ public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpa
             {
                 var cpuEngine = new HvEngine(_tickSource);
                 var memoryManager = new HvMemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler);
-                processContext = new ArmProcessContext<HvMemoryManager>(pid, cpuEngine, _gpu, memoryManager, for64Bit);
+                processContext = new ArmProcessContext<HvMemoryManager>(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit);
             }
             else
             {
                 MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode;
 
                 if (!MemoryBlock.SupportsFlags(MemoryAllocationFlags.ViewCompatible))
                 {
+                    Logger.Warning?.Print(LogClass.Cpu, "Host system doesn't support views, falling back to software page table");
+
                     mode = MemoryManagerMode.SoftwarePageTable;
                 }
 
                 var cpuEngine = new JitEngine(_tickSource);
 
+                AddressSpace addressSpace = null;
+
+                if (mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe)
+                {
+                    if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, MemoryBlock.GetPageSize() == MemoryManagerHostMapped.PageSize, out addressSpace))
+                    {
+                        Logger.Warning?.Print(LogClass.Cpu, "Address space creation failed, falling back to software page table");
+
+                        mode = MemoryManagerMode.SoftwarePageTable;
+                    }
+                }
+
                 switch (mode)
                 {
                     case MemoryManagerMode.SoftwarePageTable:
                         var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler);
-                        processContext = new ArmProcessContext<MemoryManager>(pid, cpuEngine, _gpu, memoryManager, for64Bit);
+                        processContext = new ArmProcessContext<MemoryManager>(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit);
                         break;
 
                     case MemoryManagerMode.HostMapped:
                     case MemoryManagerMode.HostMappedUnsafe:
-                        bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe;
-                        var memoryManagerHostMapped = new MemoryManagerHostMapped(context.Memory, addressSpaceSize, unsafeMode, invalidAccessHandler);
-                        processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, memoryManagerHostMapped, for64Bit);
+                        if (addressSpaceSize != addressSpace.AddressSpaceSize)
+                        {
+                            Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})");
+                        }
+
+                        var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler);
+                        processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit);
                         break;
 
                     default:

src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs

@@ -13,7 +13,7 @@ class KPageTable : KPageTableBase
 
         protected override bool Supports4KBPages => _cpuMemory.Supports4KBPages;
 
-        public KPageTable(KernelContext context, IVirtualMemoryManager cpuMemory) : base(context)
+        public KPageTable(KernelContext context, IVirtualMemoryManager cpuMemory, ulong reservedAddressSpaceSize) : base(context, reservedAddressSpaceSize)
         {
             _cpuMemory = cpuMemory;
         }