More flexible memory manager (#307)

* Keep track mapped buffers with fixed offsets

* Started rewriting the memory manager

* Initial support for MapPhysicalMemory and UnmapPhysicalMemory, other tweaks

* MapPhysicalMemory/UnmapPhysicalMemory support, other tweaks

* Rebased

* Optimize the map/unmap physical memory svcs

* Integrate shared font support

* Fix address space reserve alignment

* Some fixes related to gpu memory mapping

* Some cleanup

* Only try uploading const buffers that are really used

* Check if memory region is contiguous

* Rebased

* Add missing count increment on IsRegionModified

* Check for reads/writes outside of the address space, optimize translation with a tail call

AOptimizations.cs

@@ -2,8 +2,6 @@ using System.Runtime.Intrinsics.X86;
 
 public static class AOptimizations
 {
-    public static bool DisableMemoryChecks = false;
-
     public static bool GenerateCallStack = true;
 
     private static bool UseAllSseIfAvailable = true;

Exceptions/VmmAccessViolationException.cs

@@ -1,14 +0,0 @@
-using ChocolArm64.Memory;
-using System;
-
-namespace ChocolArm64.Exceptions
-{
-    public class VmmAccessViolationException : Exception
-    {
-        private const string ExMsg = "Address 0x{0:x16} does not have \"{1}\" permission!";
-
-        public VmmAccessViolationException() { }
-
-        public VmmAccessViolationException(long Position, AMemoryPerm Perm) : base(string.Format(ExMsg, Position, Perm)) { }
-    }
-}

Exceptions/VmmOutOfMemoryException.cs

@@ -2,12 +2,12 @@ using System;
 
 namespace ChocolArm64.Exceptions
 {
-    public class VmmOutOfMemoryException : Exception
+    public class VmmAccessException : Exception
     {
-        private const string ExMsg = "Failed to allocate {0} bytes of memory!";
+        private const string ExMsg = "Memory region at 0x{0} with size 0x{1} is not contiguous!";
 
-        public VmmOutOfMemoryException() { }
+        public VmmAccessException() { }
 
-        public VmmOutOfMemoryException(long Size) : base(string.Format(ExMsg, Size)) { }
+        public VmmAccessException(long Position, long Size) : base(string.Format(ExMsg, Position, Size)) { }
     }
 }

Instruction/AInstEmitMemoryHelper.cs

@@ -45,46 +45,21 @@ namespace ChocolArm64.Instruction
             {
                 switch (Size)
                 {
-                    case 0: Name = AOptimizations.DisableMemoryChecks
+                    case 0: Name = nameof(AMemory.ReadVector8);   break;
-                        ? nameof(AMemory.ReadVector8Unchecked)
+                    case 1: Name = nameof(AMemory.ReadVector16);  break;
-                        : nameof(AMemory.ReadVector8); break;
+                    case 2: Name = nameof(AMemory.ReadVector32);  break;
-
+                    case 3: Name = nameof(AMemory.ReadVector64);  break;
-                    case 1: Name = AOptimizations.DisableMemoryChecks
+                    case 4: Name = nameof(AMemory.ReadVector128); break;
-                        ? nameof(AMemory.ReadVector16Unchecked)
-                        : nameof(AMemory.ReadVector16); break;
-
-                    case 2: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadVector32Unchecked)
-                        : nameof(AMemory.ReadVector32); break;
-
-                    case 3: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadVector64Unchecked)
-                        : nameof(AMemory.ReadVector64); break;
-
-                    case 4: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadVector128Unchecked)
-                        : nameof(AMemory.ReadVector128); break;
                 }
             }
             else
             {
                 switch (Size)
                 {
-                    case 0: Name = AOptimizations.DisableMemoryChecks
+                    case 0: Name = nameof(AMemory.ReadByte);   break;
-                        ? nameof(AMemory.ReadByteUnchecked)
+                    case 1: Name = nameof(AMemory.ReadUInt16); break;
-                        : nameof(AMemory.ReadByte); break;
+                    case 2: Name = nameof(AMemory.ReadUInt32); break;
-
+                    case 3: Name = nameof(AMemory.ReadUInt64); break;
-                    case 1: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadUInt16Unchecked)
-                        : nameof(AMemory.ReadUInt16); break;
-
-                    case 2: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadUInt32Unchecked)
-                        : nameof(AMemory.ReadUInt32); break;
-
-                    case 3: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.ReadUInt64Unchecked)
-                        : nameof(AMemory.ReadUInt64); break;
                 }
             }
 
@@ -132,46 +107,21 @@ namespace ChocolArm64.Instruction
             {
                 switch (Size)
                 {
-                    case 0: Name = AOptimizations.DisableMemoryChecks
+                    case 0: Name = nameof(AMemory.WriteVector8);   break;
-                        ? nameof(AMemory.WriteVector8Unchecked)
+                    case 1: Name = nameof(AMemory.WriteVector16);  break;
-                        : nameof(AMemory.WriteVector8); break;
+                    case 2: Name = nameof(AMemory.WriteVector32);  break;
-
+                    case 3: Name = nameof(AMemory.WriteVector64);  break;
-                    case 1: Name = AOptimizations.DisableMemoryChecks
+                    case 4: Name = nameof(AMemory.WriteVector128); break;
-                        ? nameof(AMemory.WriteVector16Unchecked)
-                        : nameof(AMemory.WriteVector16); break;
-
-                    case 2: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteVector32Unchecked)
-                        : nameof(AMemory.WriteVector32); break;
-
-                    case 3: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteVector64Unchecked)
-                        : nameof(AMemory.WriteVector64); break;
-
-                    case 4: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteVector128Unchecked)
-                        : nameof(AMemory.WriteVector128); break;
                 }
             }
             else
             {
                 switch (Size)
                 {
-                    case 0: Name = AOptimizations.DisableMemoryChecks
+                    case 0: Name = nameof(AMemory.WriteByte);   break;
-                        ? nameof(AMemory.WriteByteUnchecked)
+                    case 1: Name = nameof(AMemory.WriteUInt16); break;
-                        : nameof(AMemory.WriteByte); break;
+                    case 2: Name = nameof(AMemory.WriteUInt32); break;
-
+                    case 3: Name = nameof(AMemory.WriteUInt64); break;
-                    case 1: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteUInt16Unchecked)
-                        : nameof(AMemory.WriteUInt16); break;
-
-                    case 2: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteUInt32Unchecked)
-                        : nameof(AMemory.WriteUInt32); break;
-
-                    case 3: Name = AOptimizations.DisableMemoryChecks
-                        ? nameof(AMemory.WriteUInt64Unchecked)
-                        : nameof(AMemory.WriteUInt64); break;
                 }
             }
 

Memory/AMemory.cs

@@ -1,6 +1,7 @@
 using ChocolArm64.Exceptions;
 using ChocolArm64.State;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -12,9 +13,22 @@ namespace ChocolArm64.Memory
 {
     public unsafe class AMemory : IAMemory, IDisposable
     {
-        private const long ErgMask = (4 << AThreadState.ErgSizeLog2) - 1;
+        private const int PTLvl0Bits = 13;
+        private const int PTLvl1Bits = 14;
+        private const int PTPageBits = 12;
 
-        public AMemoryMgr Manager { get; private set; }
+        private const int PTLvl0Size = 1 << PTLvl0Bits;
+        private const int PTLvl1Size = 1 << PTLvl1Bits;
+        public  const int PageSize   = 1 << PTPageBits;
+
+        private const int PTLvl0Mask = PTLvl0Size - 1;
+        private const int PTLvl1Mask = PTLvl1Size - 1;
+        public  const int PageMask   = PageSize   - 1;
+
+        private const int PTLvl0Bit = PTPageBits + PTLvl1Bits;
+        private const int PTLvl1Bit = PTPageBits;
+
+        private const long ErgMask = (4 << AThreadState.ErgSizeLog2) - 1;
 
         private class ArmMonitor
         {
@@ -29,32 +43,30 @@ namespace ChocolArm64.Memory
 
         private Dictionary<int, ArmMonitor> Monitors;
 
+        private ConcurrentDictionary<long, IntPtr> ObservedPages;
+
         public IntPtr Ram { get; private set; }
 
         private byte* RamPtr;
 
-        private int HostPageSize;
+        private byte*** PageTable;
 
-        public AMemory()
+        public AMemory(IntPtr Ram)
         {
-            Manager = new AMemoryMgr();
-
             Monitors = new Dictionary<int, ArmMonitor>();
 
-            IntPtr Size = (IntPtr)AMemoryMgr.RamSize + AMemoryMgr.PageSize;
+            ObservedPages = new ConcurrentDictionary<long, IntPtr>();
 
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            this.Ram = Ram;
-            {
-                Ram = AMemoryWin32.Allocate(Size);
-
-                HostPageSize = AMemoryWin32.GetPageSize(Ram, Size);
-            }
-            else
-            {
-                Ram = Marshal.AllocHGlobal(Size);
-            }
 
             RamPtr = (byte*)Ram;
+
+            PageTable = (byte***)Marshal.AllocHGlobal(PTLvl0Size * IntPtr.Size);
+
+            for (int L0 = 0; L0 < PTLvl0Size; L0++)
+            {
+                PageTable[L0] = null;
+            }
         }
 
         public void RemoveMonitor(AThreadState State)
@@ -155,62 +167,6 @@ namespace ChocolArm64.Memory
             }
         }
 
-        public int GetHostPageSize()
-        {
-            return HostPageSize;
-        }
-
-        public (bool[], long) IsRegionModified(long Position, long Size)
-        {
-            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-            {
-                return (null, 0);
-            }
-
-            long EndPos = Position + Size;
-
-            if ((ulong)EndPos < (ulong)Position)
-            {
-                return (null, 0);
-            }
-
-            if ((ulong)EndPos > AMemoryMgr.RamSize)
-            {
-                return (null, 0);
-            }
-
-            IntPtr MemAddress = new IntPtr(RamPtr + Position);
-            IntPtr MemSize    = new IntPtr(Size);
-
-            int HostPageMask = HostPageSize - 1;
-
-            Position &= ~HostPageMask;
-
-            Size = EndPos - Position;
-
-            IntPtr[] Addresses  = new IntPtr[(Size + HostPageMask) / HostPageSize];
-
-            AMemoryWin32.IsRegionModified(MemAddress, MemSize, Addresses, out int Count);
-
-            bool[] Modified = new bool[Addresses.Length];
-
-            for (int Index = 0; Index < Count; Index++)
-            {
-                long VA = Addresses[Index].ToInt64() - Ram.ToInt64();
-
-                Modified[(VA - Position) / HostPageSize] = true;
-            }
-
-            return (Modified, Count);
-        }
-
-        public IntPtr GetHostAddress(long Position, long Size)
-        {
-            EnsureRangeIsValid(Position, Size, AMemoryPerm.Read);
-
-            return (IntPtr)(RamPtr + (ulong)Position);
-        }
-
         public sbyte ReadSByte(long Position)
         {
             return (sbyte)ReadByte(Position);
@@ -233,33 +189,22 @@ namespace ChocolArm64.Memory
 
         public byte ReadByte(long Position)
         {
-            EnsureAccessIsValid(Position, AMemoryPerm.Read);
+            return *((byte*)Translate(Position));
-
-            return ReadByteUnchecked(Position);
         }
 
         public ushort ReadUInt16(long Position)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Read);
+            return *((ushort*)Translate(Position));
-            EnsureAccessIsValid(Position + 1, AMemoryPerm.Read);
-
-            return ReadUInt16Unchecked(Position);
         }
 
         public uint ReadUInt32(long Position)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Read);
+            return *((uint*)Translate(Position));
-            EnsureAccessIsValid(Position + 3, AMemoryPerm.Read);
-
-            return ReadUInt32Unchecked(Position);
         }
 
         public ulong ReadUInt64(long Position)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Read);
+            return *((ulong*)Translate(Position));
-            EnsureAccessIsValid(Position + 7, AMemoryPerm.Read);
-
-            return ReadUInt64Unchecked(Position);
         }
 
         public Vector128<float> ReadVector8(long Position)
@@ -274,6 +219,7 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public Vector128<float> ReadVector16(long Position)
         {
             if (Sse2.IsSupported)
@@ -286,14 +232,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
         public Vector128<float> ReadVector32(long Position)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Read);
-            EnsureAccessIsValid(Position + 3, AMemoryPerm.Read);
-
             if (Sse.IsSupported)
             {
-                return Sse.LoadScalarVector128((float*)(RamPtr + (uint)Position));
+                return Sse.LoadScalarVector128((float*)Translate(Position));
             }
             else
             {
@@ -301,14 +245,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
         public Vector128<float> ReadVector64(long Position)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Read);
-            EnsureAccessIsValid(Position + 7, AMemoryPerm.Read);
-
             if (Sse2.IsSupported)
             {
-                return Sse.StaticCast<double, float>(Sse2.LoadScalarVector128((double*)(RamPtr + (uint)Position)));
+                return Sse.StaticCast<double, float>(Sse2.LoadScalarVector128((double*)Translate(Position)));
             }
             else
             {
@@ -316,118 +258,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public Vector128<float> ReadVector128(long Position)
         {
-            EnsureAccessIsValid(Position + 0,  AMemoryPerm.Read);
-            EnsureAccessIsValid(Position + 15, AMemoryPerm.Read);
-
             if (Sse.IsSupported)
             {
-                return Sse.LoadVector128((float*)(RamPtr + (uint)Position));
+                return Sse.LoadVector128((float*)Translate(Position));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        public sbyte ReadSByteUnchecked(long Position)
-        {
-            return (sbyte)ReadByteUnchecked(Position);
-        }
-
-        public short ReadInt16Unchecked(long Position)
-        {
-            return (short)ReadUInt16Unchecked(Position);
-        }
-
-        public int ReadInt32Unchecked(long Position)
-        {
-            return (int)ReadUInt32Unchecked(Position);
-        }
-
-        public long ReadInt64Unchecked(long Position)
-        {
-            return (long)ReadUInt64Unchecked(Position);
-        }
-
-        public byte ReadByteUnchecked(long Position)
-        {
-            return *((byte*)(RamPtr + (uint)Position));
-        }
-
-        public ushort ReadUInt16Unchecked(long Position)
-        {
-            return *((ushort*)(RamPtr + (uint)Position));
-        }
-
-        public uint ReadUInt32Unchecked(long Position)
-        {
-            return *((uint*)(RamPtr + (uint)Position));
-        }
-
-        public ulong ReadUInt64Unchecked(long Position)
-        {
-            return *((ulong*)(RamPtr + (uint)Position));
-        }
-
-        public Vector128<float> ReadVector8Unchecked(long Position)
-        {
-            if (Sse2.IsSupported)
-            {
-                return Sse.StaticCast<byte, float>(Sse2.SetVector128(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ReadByte(Position)));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public Vector128<float> ReadVector16Unchecked(long Position)
-        {
-            if (Sse2.IsSupported)
-            {
-                return Sse.StaticCast<ushort, float>(Sse2.Insert(Sse2.SetZeroVector128<ushort>(), ReadUInt16Unchecked(Position), 0));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public Vector128<float> ReadVector32Unchecked(long Position)
-        {
-            if (Sse.IsSupported)
-            {
-                return Sse.LoadScalarVector128((float*)(RamPtr + (uint)Position));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public Vector128<float> ReadVector64Unchecked(long Position)
-        {
-            if (Sse2.IsSupported)
-            {
-                return Sse.StaticCast<double, float>(Sse2.LoadScalarVector128((double*)(RamPtr + (uint)Position)));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public Vector128<float> ReadVector128Unchecked(long Position)
-        {
-            if (Sse.IsSupported)
-            {
-                return Sse.LoadVector128((float*)(RamPtr + (uint)Position));
             }
             else
             {
@@ -442,11 +278,11 @@ namespace ChocolArm64.Memory
                 throw new ArgumentOutOfRangeException(nameof(Size));
             }
 
-            EnsureRangeIsValid(Position, Size, AMemoryPerm.Read);
+            EnsureRangeIsValid(Position, Size);
 
             byte[] Data = new byte[Size];
 
-            Marshal.Copy((IntPtr)(RamPtr + (uint)Position), Data, 0, (int)Size);
+            Marshal.Copy((IntPtr)Translate(Position), Data, 0, (int)Size);
 
             return Data;
         }
@@ -473,35 +309,25 @@ namespace ChocolArm64.Memory
 
         public void WriteByte(long Position, byte Value)
         {
-            EnsureAccessIsValid(Position, AMemoryPerm.Write);
+            *((byte*)TranslateWrite(Position)) = Value;
-
-            WriteByteUnchecked(Position, Value);
         }
 
         public void WriteUInt16(long Position, ushort Value)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Write);
+            *((ushort*)TranslateWrite(Position)) = Value;
-            EnsureAccessIsValid(Position + 1, AMemoryPerm.Write);
-
-            WriteUInt16Unchecked(Position, Value);
         }
 
         public void WriteUInt32(long Position, uint Value)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Write);
+            *((uint*)TranslateWrite(Position)) = Value;
-            EnsureAccessIsValid(Position + 3, AMemoryPerm.Write);
-
-            WriteUInt32Unchecked(Position, Value);
         }
 
         public void WriteUInt64(long Position, ulong Value)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Write);
+            *((ulong*)TranslateWrite(Position)) = Value;
-            EnsureAccessIsValid(Position + 7, AMemoryPerm.Write);
-
-            WriteUInt64Unchecked(Position, Value);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void WriteVector8(long Position, Vector128<float> Value)
         {
             if (Sse41.IsSupported)
@@ -518,6 +344,7 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void WriteVector16(long Position, Vector128<float> Value)
         {
             if (Sse2.IsSupported)
@@ -530,14 +357,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
         public void WriteVector32(long Position, Vector128<float> Value)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Write);
-            EnsureAccessIsValid(Position + 3, AMemoryPerm.Write);
-
             if (Sse.IsSupported)
             {
-                Sse.StoreScalar((float*)(RamPtr + (uint)Position), Value);
+                Sse.StoreScalar((float*)TranslateWrite(Position), Value);
             }
             else
             {
@@ -545,14 +370,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.NoInlining)]
         public void WriteVector64(long Position, Vector128<float> Value)
         {
-            EnsureAccessIsValid(Position + 0, AMemoryPerm.Write);
-            EnsureAccessIsValid(Position + 7, AMemoryPerm.Write);
-
             if (Sse2.IsSupported)
             {
-                Sse2.StoreScalar((double*)(RamPtr + (uint)Position), Sse.StaticCast<float, double>(Value));
+                Sse2.StoreScalar((double*)TranslateWrite(Position), Sse.StaticCast<float, double>(Value));
             }
             else
             {
@@ -560,123 +383,12 @@ namespace ChocolArm64.Memory
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void WriteVector128(long Position, Vector128<float> Value)
         {
-            EnsureAccessIsValid(Position + 0,  AMemoryPerm.Write);
-            EnsureAccessIsValid(Position + 15, AMemoryPerm.Write);
-
             if (Sse.IsSupported)
             {
-                Sse.Store((float*)(RamPtr + (uint)Position), Value);
+                Sse.Store((float*)TranslateWrite(Position), Value);
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        public void WriteSByteUnchecked(long Position, sbyte Value)
-        {
-            WriteByteUnchecked(Position, (byte)Value);
-        }
-
-        public void WriteInt16Unchecked(long Position, short Value)
-        {
-            WriteUInt16Unchecked(Position, (ushort)Value);
-        }
-
-        public void WriteInt32Unchecked(long Position, int Value)
-        {
-            WriteUInt32Unchecked(Position, (uint)Value);
-        }
-
-        public void WriteInt64Unchecked(long Position, long Value)
-        {
-            WriteUInt64Unchecked(Position, (ulong)Value);
-        }
-
-        public void WriteByteUnchecked(long Position, byte Value)
-        {
-            *((byte*)(RamPtr + (uint)Position)) = Value;
-        }
-
-        public void WriteUInt16Unchecked(long Position, ushort Value)
-        {
-            *((ushort*)(RamPtr + (uint)Position)) = Value;
-        }
-
-        public void WriteUInt32Unchecked(long Position, uint Value)
-        {
-            *((uint*)(RamPtr + (uint)Position)) = Value;
-        }
-
-        public void WriteUInt64Unchecked(long Position, ulong Value)
-        {
-            *((ulong*)(RamPtr + (uint)Position)) = Value;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void WriteVector8Unchecked(long Position, Vector128<float> Value)
-        {
-            if (Sse41.IsSupported)
-            {
-                WriteByteUnchecked(Position, Sse41.Extract(Sse.StaticCast<float, byte>(Value), 0));
-            }
-            else if (Sse2.IsSupported)
-            {
-                WriteByteUnchecked(Position, (byte)Sse2.Extract(Sse.StaticCast<float, ushort>(Value), 0));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void WriteVector16Unchecked(long Position, Vector128<float> Value)
-        {
-            if (Sse2.IsSupported)
-            {
-                WriteUInt16Unchecked(Position, Sse2.Extract(Sse.StaticCast<float, ushort>(Value), 0));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public void WriteVector32Unchecked(long Position, Vector128<float> Value)
-        {
-            if (Sse.IsSupported)
-            {
-                Sse.StoreScalar((float*)(RamPtr + (uint)Position), Value);
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        public void WriteVector64Unchecked(long Position, Vector128<float> Value)
-        {
-            if (Sse2.IsSupported)
-            {
-                Sse2.StoreScalar((double*)(RamPtr + (uint)Position), Sse.StaticCast<float, double>(Value));
-            }
-            else
-            {
-                throw new PlatformNotSupportedException();
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public void WriteVector128Unchecked(long Position, Vector128<float> Value)
-        {
-            if (Sse.IsSupported)
-            {
-                Sse.Store((float*)(RamPtr + (uint)Position), Value);
             }
             else
             {
@@ -686,36 +398,285 @@ namespace ChocolArm64.Memory
 
         public void WriteBytes(long Position, byte[] Data)
         {
-            EnsureRangeIsValid(Position, (uint)Data.Length, AMemoryPerm.Write);
+            EnsureRangeIsValid(Position, (uint)Data.Length);
 
-            Marshal.Copy(Data, 0, (IntPtr)(RamPtr + (uint)Position), Data.Length);
+            Marshal.Copy(Data, 0, (IntPtr)TranslateWrite(Position), Data.Length);
         }
 
-        private void EnsureRangeIsValid(long Position, long Size, AMemoryPerm Perm)
+        public void Map(long VA, long PA, long Size)
         {
-            long EndPos = Position + Size;
+            SetPTEntries(VA, RamPtr + PA, Size);
-
-            Position &= ~AMemoryMgr.PageMask;
-
-            while ((ulong)Position < (ulong)EndPos)
-            {
-                EnsureAccessIsValid(Position, Perm);
-
-                Position += AMemoryMgr.PageSize;
-            }
         }
 
-        private void EnsureAccessIsValid(long Position, AMemoryPerm Perm)
+        public void Unmap(long Position, long Size)
         {
-            if (!Manager.IsMapped(Position))
+            SetPTEntries(Position, null, Size);
+
+            StopObservingRegion(Position, Size);
+        }
+
+        public bool IsMapped(long Position)
         {
+            if (!(IsValidPosition(Position)))
+            {
+                return false;
+            }
+
+            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
+            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
+
+            if (PageTable[L0] == null)
+            {
+                return false;
+            }
+
+            return PageTable[L0][L1] != null || ObservedPages.ContainsKey(Position >> PTPageBits);
+        }
+
+        public long GetPhysicalAddress(long VirtualAddress)
+        {
+            byte* Ptr = Translate(VirtualAddress);
+
+            return (long)(Ptr - RamPtr);
+        }
+
+        internal byte* Translate(long Position)
+        {
+            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
+            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
+
+            long Old = Position;
+
+            byte** Lvl1 = PageTable[L0];
+
+            if ((Position >> (PTLvl0Bit + PTLvl0Bits)) != 0)
+            {
+                goto Unmapped;
+            }
+
+            if (Lvl1 == null)
+            {
+                goto Unmapped;
+            }
+
+            Position &= PageMask;
+
+            byte* Ptr = Lvl1[L1];
+
+            if (Ptr == null)
+            {
+                goto Unmapped;
+            }
+
+            return Ptr + Position;
+
+Unmapped:
+            return HandleNullPte(Old);
+        }
+
+        private byte* HandleNullPte(long Position)
+        {
+            long Key = Position >> PTPageBits;
+
+            if (ObservedPages.TryGetValue(Key, out IntPtr Ptr))
+            {
+                return (byte*)Ptr + (Position & PageMask);
+            }
+
             throw new VmmPageFaultException(Position);
         }
 
-            if (!Manager.HasPermission(Position, Perm))
+        internal byte* TranslateWrite(long Position)
         {
-                throw new VmmAccessViolationException(Position, Perm);
+            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
+            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
+
+            long Old = Position;
+
+            byte** Lvl1 = PageTable[L0];
+
+            if ((Position >> (PTLvl0Bit + PTLvl0Bits)) != 0)
+            {
+                goto Unmapped;
             }
+
+            if (Lvl1 == null)
+            {
+                goto Unmapped;
+            }
+
+            Position &= PageMask;
+
+            byte* Ptr = Lvl1[L1];
+
+            if (Ptr == null)
+            {
+                goto Unmapped;
+            }
+
+            return Ptr + Position;
+
+Unmapped:
+            return HandleNullPteWrite(Old);
+        }
+
+        private byte* HandleNullPteWrite(long Position)
+        {
+            long Key = Position >> PTPageBits;
+
+            if (ObservedPages.TryGetValue(Key, out IntPtr Ptr))
+            {
+                SetPTEntry(Position, (byte*)Ptr);
+
+                return (byte*)Ptr + (Position & PageMask);
+            }
+
+            throw new VmmPageFaultException(Position);
+        }
+
+        private void SetPTEntries(long VA, byte* Ptr, long Size)
+        {
+            long EndPosition = (VA + Size + PageMask) & ~PageMask;
+
+            while ((ulong)VA < (ulong)EndPosition)
+            {
+                SetPTEntry(VA, Ptr);
+
+                VA += PageSize;
+
+                if (Ptr != null)
+                {
+                    Ptr += PageSize;
+                }
+            }
+        }
+
+        private void SetPTEntry(long Position, byte* Ptr)
+        {
+            if (!IsValidPosition(Position))
+            {
+                throw new ArgumentOutOfRangeException(nameof(Position));
+            }
+
+            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
+            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
+
+            if (PageTable[L0] == null)
+            {
+                byte** Lvl1 = (byte**)Marshal.AllocHGlobal(PTLvl1Size * IntPtr.Size);
+
+                for (int ZL1 = 0; ZL1 < PTLvl1Size; ZL1++)
+                {
+                    Lvl1[ZL1] = null;
+                }
+
+                Thread.MemoryBarrier();
+
+                PageTable[L0] = Lvl1;
+            }
+
+            PageTable[L0][L1] = Ptr;
+        }
+
+        public (bool[], int) IsRegionModified(long Position, long Size)
+        {
+            long EndPosition = (Position + Size + PageMask) & ~PageMask;
+
+            Position &= ~PageMask;
+
+            Size = EndPosition - Position;
+
+            bool[] Modified = new bool[Size >> PTPageBits];
+
+            int Count = 0;
+
+            lock (ObservedPages)
+            {
+                for (int Page = 0; Page < Modified.Length; Page++)
+                {
+                    byte* Ptr = Translate(Position);
+
+                    if (ObservedPages.TryAdd(Position >> PTPageBits, (IntPtr)Ptr))
+                    {
+                        Modified[Page] = true;
+
+                        Count++;
+                    }
+                    else
+                    {
+                        long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
+                        long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
+
+                        byte** Lvl1 = PageTable[L0];
+
+                        if (Lvl1 != null)
+                        {
+                            if (Modified[Page] = Lvl1[L1] != null)
+                            {
+                                Count++;
+                            }
+                        }
+                    }
+
+                    SetPTEntry(Position, null);
+
+                    Position += PageSize;
+                }
+            }
+
+            return (Modified, Count);
+        }
+
+        public void StopObservingRegion(long Position, long Size)
+        {
+            long EndPosition = (Position + Size + PageMask) & ~PageMask;
+
+            while (Position < EndPosition)
+            {
+                lock (ObservedPages)
+                {
+                    if (ObservedPages.TryRemove(Position >> PTPageBits, out IntPtr Ptr))
+                    {
+                        SetPTEntry(Position, (byte*)Ptr);
+                    }
+                }
+
+                Position += PageSize;
+            }
+        }
+
+        public IntPtr GetHostAddress(long Position, long Size)
+        {
+            EnsureRangeIsValid(Position, Size);
+
+            return (IntPtr)Translate(Position);
+        }
+
+        internal void EnsureRangeIsValid(long Position, long Size)
+        {
+            long EndPos = Position + Size;
+
+            Position &= ~PageMask;
+
+            long ExpectedPA = GetPhysicalAddress(Position);
+
+            while ((ulong)Position < (ulong)EndPos)
+            {
+                long PA = GetPhysicalAddress(Position);
+
+                if (PA != ExpectedPA)
+                {
+                    throw new VmmAccessException(Position, Size);
+                }
+
+                Position   += PageSize;
+                ExpectedPA += PageSize;
+            }
+        }
+
+        public bool IsValidPosition(long Position)
+        {
+            return Position >> (PTLvl0Bits + PTLvl1Bits + PTPageBits) == 0;
         }
 
         public void Dispose()
@@ -725,19 +686,24 @@ namespace ChocolArm64.Memory
 
         protected virtual void Dispose(bool disposing)
         {
-            if (Ram != IntPtr.Zero)
+            if (PageTable == null)
             {
-                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                return;
-                {
-                    AMemoryWin32.Free(Ram);
-                }
-                else
-                {
-                    Marshal.FreeHGlobal(Ram);
             }
 
-                Ram = IntPtr.Zero;
+            for (int L0 = 0; L0 < PTLvl0Size; L0++)
-            }
+            {
+                if (PageTable[L0] != null)
+                {
+                    Marshal.FreeHGlobal((IntPtr)PageTable[L0]);
+                }
+
+                PageTable[L0] = null;
+            }
+
+            Marshal.FreeHGlobal((IntPtr)PageTable);
+
+            PageTable = null;
         }
     }
 }

Memory/AMemoryHelper.cs

@@ -26,12 +26,9 @@ namespace ChocolArm64.Memory
         {
             long Size = Marshal.SizeOf<T>();
 
-            if ((ulong)(Position + Size) > AMemoryMgr.AddrSize)
+            Memory.EnsureRangeIsValid(Position, Size);
-            {
-                throw new ArgumentOutOfRangeException(nameof(Position));
-            }
 
-            IntPtr Ptr = new IntPtr((byte*)Memory.Ram + Position);
+            IntPtr Ptr = (IntPtr)Memory.Translate(Position);
 
             return Marshal.PtrToStructure<T>(Ptr);
         }
@@ -40,12 +37,9 @@ namespace ChocolArm64.Memory
         {
             long Size = Marshal.SizeOf<T>();
 
-            if ((ulong)(Position + Size) > AMemoryMgr.AddrSize)
+            Memory.EnsureRangeIsValid(Position, Size);
-            {
-                throw new ArgumentOutOfRangeException(nameof(Position));
-            }
 
-            IntPtr Ptr = new IntPtr((byte*)Memory.Ram + Position);
+            IntPtr Ptr = (IntPtr)Memory.TranslateWrite(Position);
 
             Marshal.StructureToPtr<T>(Value, Ptr, false);
         }
@@ -69,15 +63,5 @@ namespace ChocolArm64.Memory
                 return Encoding.ASCII.GetString(MS.ToArray());
             }
         }
-
-        public static long PageRoundUp(long Value)
-        {
-            return (Value + AMemoryMgr.PageMask) & ~AMemoryMgr.PageMask;
-        }
-
-        public static long PageRoundDown(long Value)
-        {
-            return Value & ~AMemoryMgr.PageMask;
-        }
     }
 }

Memory/AMemoryMapInfo.cs

@@ -1,21 +0,0 @@
-namespace ChocolArm64.Memory
-{
-    public class AMemoryMapInfo
-    {
-        public long Position { get; private set; }
-        public long Size     { get; private set; }
-        public int  Type     { get; private set; }
-        public int  Attr     { get; private set; }
-
-        public AMemoryPerm Perm { get; private set; }
-
-        public AMemoryMapInfo(long Position, long Size, int Type, int Attr, AMemoryPerm Perm)
-        {
-            this.Position = Position;
-            this.Size     = Size;
-            this.Type     = Type;
-            this.Attr     = Attr;
-            this.Perm     = Perm;
-        }
-    }
-}

Memory/AMemoryMgr.cs

@@ -1,258 +0,0 @@
-using System;
-
-namespace ChocolArm64.Memory
-{
-    public class AMemoryMgr
-    {
-        public const long RamSize  = 4L * 1024 * 1024 * 1024;
-        public const long AddrSize = RamSize;
-
-        private const int PTLvl0Bits = 10;
-        private const int PTLvl1Bits = 10;
-        private const int PTPageBits = 12;
-
-        private const int PTLvl0Size = 1 << PTLvl0Bits;
-        private const int PTLvl1Size = 1 << PTLvl1Bits;
-        public  const int PageSize   = 1 << PTPageBits;
-
-        private const int PTLvl0Mask = PTLvl0Size - 1;
-        private const int PTLvl1Mask = PTLvl1Size - 1;
-        public  const int PageMask   = PageSize   - 1;
-
-        private const int PTLvl0Bit = PTPageBits + PTLvl1Bits;
-        private const int PTLvl1Bit = PTPageBits;
-
-        private enum PTMap
-        {
-            Unmapped,
-            Mapped
-        }
-
-        private struct PTEntry
-        {
-            public PTMap       Map;
-            public AMemoryPerm Perm;
-
-            public int Type;
-            public int Attr;
-
-            public PTEntry(PTMap Map, AMemoryPerm Perm, int Type, int Attr)
-            {
-                this.Map  = Map;
-                this.Perm = Perm;
-                this.Type = Type;
-                this.Attr = Attr;
-            }
-        }
-
-        private PTEntry[][] PageTable;
-
-        public AMemoryMgr()
-        {
-            PageTable = new PTEntry[PTLvl0Size][];
-        }
-
-        public void Map(long Position, long Size, int Type, AMemoryPerm Perm)
-        {
-            SetPTEntry(Position, Size, new PTEntry(PTMap.Mapped, Perm, Type, 0));
-        }
-
-        public void Unmap(long Position, long Size)
-        {
-            SetPTEntry(Position, Size, new PTEntry(PTMap.Unmapped, 0, 0, 0));
-        }
-
-        public void Unmap(long Position, long Size, int Type)
-        {
-            SetPTEntry(Position, Size, Type, new PTEntry(PTMap.Unmapped, 0, 0, 0));
-        }
-
-        public void Reprotect(long Position, long Size, AMemoryPerm Perm)
-        {
-            Position = AMemoryHelper.PageRoundDown(Position);
-
-            Size = AMemoryHelper.PageRoundUp(Size);
-
-            long PagesCount = Size / PageSize;
-
-            while (PagesCount-- > 0)
-            {
-                PTEntry Entry = GetPTEntry(Position);
-
-                Entry.Perm = Perm;
-
-                SetPTEntry(Position, Entry);
-
-                Position += PageSize;
-            }
-        }
-
-        public AMemoryMapInfo GetMapInfo(long Position)
-        {
-            if (!IsValidPosition(Position))
-            {
-                return null;
-            }
-
-            Position = AMemoryHelper.PageRoundDown(Position);
-
-            PTEntry BaseEntry = GetPTEntry(Position);
-
-            bool IsSameSegment(long Pos)
-            {
-                if (!IsValidPosition(Pos))
-                {
-                    return false;
-                }
-
-                PTEntry Entry = GetPTEntry(Pos);
-
-                return Entry.Map  == BaseEntry.Map  &&
-                       Entry.Perm == BaseEntry.Perm &&
-                       Entry.Type == BaseEntry.Type &&
-                       Entry.Attr == BaseEntry.Attr;
-            }
-
-            long Start = Position;
-            long End   = Position + PageSize;
-
-            while (Start > 0 && IsSameSegment(Start - PageSize))
-            {
-                Start -= PageSize;
-            }
-
-            while (End < AddrSize && IsSameSegment(End))
-            {
-                End += PageSize;
-            }
-
-            long Size = End - Start;
-
-            return new AMemoryMapInfo(
-                Start,
-                Size,
-                BaseEntry.Type,
-                BaseEntry.Attr,
-                BaseEntry.Perm);
-        }
-
-        public void ClearAttrBit(long Position, long Size, int Bit)
-        {
-            while (Size > 0)
-            {
-                PTEntry Entry = GetPTEntry(Position);
-
-                Entry.Attr &= ~(1 << Bit);
-
-                SetPTEntry(Position, Entry);
-
-                Position += PageSize;
-                Size     -= PageSize;
-            }
-        }
-
-        public void SetAttrBit(long Position, long Size, int Bit)
-        {
-            while (Size > 0)
-            {
-                PTEntry Entry = GetPTEntry(Position);
-
-                Entry.Attr |= (1 << Bit);
-
-                SetPTEntry(Position, Entry);
-
-                Position += PageSize;
-                Size     -= PageSize;
-            }
-        }
-
-        public bool HasPermission(long Position, AMemoryPerm Perm)
-        {
-            return GetPTEntry(Position).Perm.HasFlag(Perm);
-        }
-
-        public bool IsValidPosition(long Position)
-        {
-            if (Position >> PTLvl0Bits + PTLvl1Bits + PTPageBits != 0)
-            {
-                return false;
-            }
-
-            return true;
-        }
-
-        public bool IsMapped(long Position)
-        {
-            if (Position >> PTLvl0Bits + PTLvl1Bits + PTPageBits != 0)
-            {
-                return false;
-            }
-
-            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
-            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
-
-            if (PageTable[L0] == null)
-            {
-                return false;
-            }
-
-            return PageTable[L0][L1].Map != PTMap.Unmapped;
-        }
-
-        private PTEntry GetPTEntry(long Position)
-        {
-            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
-            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
-
-            if (PageTable[L0] == null)
-            {
-                return default(PTEntry);
-            }
-
-            return PageTable[L0][L1];
-        }
-
-        private void SetPTEntry(long Position, long Size, PTEntry Entry)
-        {
-            while (Size > 0)
-            {
-                SetPTEntry(Position, Entry);
-
-                Position += PageSize;
-                Size     -= PageSize;
-            }
-        }
-
-        private void SetPTEntry(long Position, long Size, int Type, PTEntry Entry)
-        {
-            while (Size > 0)
-            {
-                if (GetPTEntry(Position).Type == Type)
-                {
-                    SetPTEntry(Position, Entry);
-                }
-
-                Position += PageSize;
-                Size     -= PageSize;
-            }
-        }
-
-        private void SetPTEntry(long Position, PTEntry Entry)
-        {
-            if (!IsValidPosition(Position))
-            {
-                throw new ArgumentOutOfRangeException(nameof(Position));
-            }
-
-            long L0 = (Position >> PTLvl0Bit) & PTLvl0Mask;
-            long L1 = (Position >> PTLvl1Bit) & PTLvl1Mask;
-
-            if (PageTable[L0] == null)
-            {
-                PageTable[L0] = new PTEntry[PTLvl1Size];
-            }
-
-            PageTable[L0][L1] = Entry;
-        }
-    }
-}

Memory/AMemoryPerm.cs

@@ -1,15 +0,0 @@
-using System;
-
-namespace ChocolArm64.Memory
-{
-    [Flags]
-    public enum AMemoryPerm
-    {
-        None    = 0,
-        Read    = 1 << 0,
-        Write   = 1 << 1,
-        Execute = 1 << 2,
-        RW      = Read | Write,
-        RX      = Read | Execute
-    }
-}

Memory/AMemoryWin32.cs

@@ -1,92 +0,0 @@
-using System;
-using System.Runtime.InteropServices;
-
-namespace ChocolArm64.Memory
-{
-    static class AMemoryWin32
-    {
-        private const int MEM_COMMIT      = 0x00001000;
-        private const int MEM_RESERVE     = 0x00002000;
-        private const int MEM_WRITE_WATCH = 0x00200000;
-
-        private const int PAGE_READWRITE = 0x04;
-
-        private const int MEM_RELEASE = 0x8000;
-
-        private const int WRITE_WATCH_FLAG_RESET = 1;
-
-        [DllImport("kernel32.dll")]
-        private static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, int flAllocationType, int flProtect);
-
-        [DllImport("kernel32.dll")]
-        private static extern bool VirtualFree(IntPtr lpAddress, IntPtr dwSize, int dwFreeType);
-
-        [DllImport("kernel32.dll")]
-        private unsafe static extern int GetWriteWatch(
-            int      dwFlags,
-            IntPtr   lpBaseAddress,
-            IntPtr   dwRegionSize,
-            IntPtr[] lpAddresses,
-            long*    lpdwCount,
-            long*    lpdwGranularity);
-
-        public static IntPtr Allocate(IntPtr Size)
-        {
-            const int Flags = MEM_COMMIT | MEM_RESERVE | MEM_WRITE_WATCH;
-
-            IntPtr Address = VirtualAlloc(IntPtr.Zero, Size, Flags, PAGE_READWRITE);
-
-            if (Address == IntPtr.Zero)
-            {
-                throw new InvalidOperationException();
-            }
-
-            return Address;
-        }
-
-        public static void Free(IntPtr Address)
-        {
-            VirtualFree(Address, IntPtr.Zero, MEM_RELEASE);
-        }
-
-        public unsafe static int GetPageSize(IntPtr Address, IntPtr Size)
-        {
-            IntPtr[] Addresses = new IntPtr[1];
-
-            long Count = Addresses.Length;
-
-            long Granularity;
-
-            GetWriteWatch(
-                0,
-                Address,
-                Size,
-                Addresses,
-                &Count,
-                &Granularity);
-
-            return (int)Granularity;
-        }
-
-        public unsafe static void IsRegionModified(
-            IntPtr   Address,
-            IntPtr   Size,
-            IntPtr[] Addresses,
-            out int  AddrCount)
-        {
-            long Count = Addresses.Length;
-
-            long Granularity;
-
-            GetWriteWatch(
-                WRITE_WATCH_FLAG_RESET,
-                Address,
-                Size,
-                Addresses,
-                &Count,
-                &Granularity);
-
-            AddrCount = (int)Count;
-        }
-    }
-}