GPU: Migrate buffers on GPU project, pre-emptively flush device local…

… mappings (#6794)

* GPU: Migrate buffers on GPU project, pre-emptively flush device local mappings

Essentially retreading #4540, but it's on the GPU project now instead of the backend. This allows us to have a lot more control + knowledge of where the buffer backing has been changed and allows us to pre-emptively flush pages to host memory for quicker readback. It will allow us to do other stuff in the future, but we'll get there when we get there.

Performance greatly improved in Hyrule Warriors: Age of Calamity. Performance notably improved in TOTK (average). Performance for BOTW restored to how it was before #4911, perhaps a bit better.

- Rewrites a bunch of buffer migration stuff. Might want to tighten up how dispose stuff works.
- Fixed an issue where the copy for texture pre-flush would happen _after_ the syncpoint.

TODO: remove a page from pre-flush if it isn't flushed after a certain number of copies.

* Add copy deactivation

* Fix dependent virtual buffers

* Remove logging

* Fix format issues (maybe)

* Vulkan: Remove backing swap

* Add explicit memory access types for most buffers

* Fix typo

* Add device local force expiry, change buffer inheritance behaviour

* General cleanup, OGL fix

* BufferPreFlush comments

* BufferBackingState comments

* Add an extra precaution to BufferMigration

This is very unlikely, but it's important to cover loose ends like this.

* Address some feedback

* Docs

src/Ryujinx.Graphics.GAL/BufferAccess.cs

@@ -6,8 +6,13 @@ namespace Ryujinx.Graphics.GAL
  public enum BufferAccess
  {
  Default = 0,
- FlushPersistent = 1 << 0,
- Stream = 1 << 1,
- SparseCompatible = 1 << 2,
+ HostMemory = 1,
+ DeviceMemory = 2,
+ DeviceMemoryMapped = 3,
+
+ MemoryTypeMask = 0xf,
+
+ Stream = 1 << 4,
+ SparseCompatible = 1 << 5,
  }
 }

src/Ryujinx.Graphics.GAL/Capabilities.cs

@@ -6,6 +6,7 @@ namespace Ryujinx.Graphics.GAL
  {
  public readonly TargetApi Api;
  public readonly string VendorName;
+ public readonly SystemMemoryType MemoryType;
 
  public readonly bool HasFrontFacingBug;
  public readonly bool HasVectorIndexingBug;
@@ -66,6 +67,7 @@ namespace Ryujinx.Graphics.GAL
  public Capabilities(
  TargetApi api,
  string vendorName,
+ SystemMemoryType memoryType,
  bool hasFrontFacingBug,
  bool hasVectorIndexingBug,
  bool needsFragmentOutputSpecialization,
@@ -120,6 +122,7 @@ namespace Ryujinx.Graphics.GAL
  {
  Api = api;
  VendorName = vendorName;
+ MemoryType = memoryType;
  HasFrontFacingBug = hasFrontFacingBug;
  HasVectorIndexingBug = hasVectorIndexingBug;
  NeedsFragmentOutputSpecialization = needsFragmentOutputSpecialization;

src/Ryujinx.Graphics.GAL/IRenderer.cs

@@ -17,7 +17,6 @@ public interface IRenderer : IDisposable
  void BackgroundContextAction(Action action, bool alwaysBackground = false);
 
  BufferHandle CreateBuffer(int size, BufferAccess access = BufferAccess.Default);
- BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint);
  BufferHandle CreateBuffer(nint pointer, int size);
  BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers);
 

src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs

@@ -44,7 +44,6 @@ private static int InitLookup()
  }
 
  Register<ActionCommand>(CommandType.Action);
- Register<CreateBufferCommand>(CommandType.CreateBuffer);
  Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
  Register<CreateBufferSparseCommand>(CommandType.CreateBufferSparse);
  Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);

src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs

@@ -3,7 +3,6 @@ namespace Ryujinx.Graphics.GAL.Multithreading
  enum CommandType : byte
  {
  Action,
- CreateBuffer,
  CreateBufferAccess,
  CreateBufferSparse,
  CreateHostBuffer,

src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs

@@ -272,15 +272,6 @@ public BufferHandle CreateBuffer(int size, BufferAccess access)
  return handle;
  }
 
- public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
- {
- BufferHandle handle = Buffers.CreateBufferHandle();
- New<CreateBufferCommand>().Set(handle, size, access, storageHint);
- QueueCommand();
-
- return handle;
- }
-
  public BufferHandle CreateBuffer(nint pointer, int size)
  {
  BufferHandle handle = Buffers.CreateBufferHandle();

src/Ryujinx.Graphics.GAL/SystemMemoryType.cs

@@ -0,0 +1,29 @@
+namespace Ryujinx.Graphics.GAL
+{
+ public enum SystemMemoryType
+ {
+ /// <summary>
+ /// The backend manages the ownership of memory. This mode never supports host imported memory.
+ /// </summary>
+ BackendManaged,
+
+ /// <summary>
+ /// Device memory has similar performance to host memory, usually because it's shared between CPU/GPU.
+ /// Use host memory whenever possible.
+ /// </summary>
+ UnifiedMemory,
+
+ /// <summary>
+ /// GPU storage to host memory goes though a slow interconnect, but it would still be preferable to use it if the data is flushed back often.
+ /// Assumes constant buffer access to host memory is rather fast.
+ /// </summary>
+ DedicatedMemory,
+
+ /// <summary>
+ /// GPU storage to host memory goes though a slow interconnect, that is very slow when doing access from storage.
+ /// When frequently accessed, copy buffers to host memory using DMA.
+ /// Assumes constant buffer access to host memory is rather fast.
+ /// </summary>
+ DedicatedMemorySlowStorage
+ }
+}

src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs

@@ -5,6 +5,7 @@
 using Ryujinx.Graphics.Gpu.Engine.GPFifo;
 using Ryujinx.Graphics.Gpu.Engine.Threed;
 using Ryujinx.Graphics.Gpu.Engine.Types;
+using Ryujinx.Graphics.Gpu.Memory;
 using Ryujinx.Memory.Range;
 using System;
 using System.Collections.Generic;
@@ -495,8 +496,8 @@ private void MultiDrawElementsIndirectCount(IDeviceState state, int arg0)
 
  ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride;
 
- MultiRange indirectBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
- MultiRange parameterBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, parameterBufferGpuVa, 4);
+ MultiRange indirectBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize, BufferStage.Indirect);
+ MultiRange parameterBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, parameterBufferGpuVa, 4, BufferStage.Indirect);
 
  _processor.ThreedClass.DrawIndirect(
  topology,

src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeContext.cs

@@ -438,7 +438,7 @@ private IndexBuffer CreateTopologyRemapBuffer(PrimitiveTopology topology, int co
 
  ReadOnlySpan<byte> dataBytes = MemoryMarshal.Cast<int, byte>(data);
 
- BufferHandle buffer = _context.Renderer.CreateBuffer(dataBytes.Length);
+ BufferHandle buffer = _context.Renderer.CreateBuffer(dataBytes.Length, BufferAccess.DeviceMemory);
  _context.Renderer.SetBufferData(buffer, 0, dataBytes);
 
  return new IndexBuffer(buffer, count, dataBytes.Length);
@@ -529,7 +529,7 @@ public BufferRange GetDummyBufferRange()
  {
  if (_dummyBuffer == BufferHandle.Null)
  {
- _dummyBuffer = _context.Renderer.CreateBuffer(DummyBufferSize);
+ _dummyBuffer = _context.Renderer.CreateBuffer(DummyBufferSize, BufferAccess.DeviceMemory);
  _context.Renderer.Pipeline.ClearBuffer(_dummyBuffer, 0, DummyBufferSize, 0);
  }
 
@@ -550,7 +550,7 @@ public BufferHandle GetSequentialIndexBuffer(int count)
  _context.Renderer.DeleteBuffer(_sequentialIndexBuffer);
  }
 
- _sequentialIndexBuffer = _context.Renderer.CreateBuffer(count * sizeof(uint));
+ _sequentialIndexBuffer = _context.Renderer.CreateBuffer(count * sizeof(uint), BufferAccess.DeviceMemory);
  _sequentialIndexBufferCount = count;
 
  Span<int> data = new int[count];
@@ -583,7 +583,7 @@ public BufferHandle GetSequentialIndexBuffer(int count)
  _context.Renderer.DeleteBuffer(buffer.Handle);
  }
 
- buffer.Handle = _context.Renderer.CreateBuffer(newSize);
+ buffer.Handle = _context.Renderer.CreateBuffer(newSize, BufferAccess.DeviceMemory);
  buffer.Size = newSize;
  }
 

src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeState.cs

@@ -3,6 +3,7 @@
 using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.Engine.Types;
 using Ryujinx.Graphics.Gpu.Image;
+using Ryujinx.Graphics.Gpu.Memory;
 using Ryujinx.Graphics.Gpu.Shader;
 using Ryujinx.Graphics.Shader;
 using Ryujinx.Graphics.Shader.Translation;
@@ -370,7 +371,7 @@ private static int GetMaxCompleteStrips(int verticesPerPrimitive, int maxOutputV
  {
  var memoryManager = _channel.MemoryManager;
 
- BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address, size));
+ BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address, size), BufferStage.VertexBuffer);
 
  ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format);
  bufferTexture.SetStorage(range);
@@ -412,7 +413,9 @@ private static int GetMaxCompleteStrips(int verticesPerPrimitive, int maxOutputV
  var memoryManager = _channel.MemoryManager;
 
  ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1);
- BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address + indexOffset - misalign, size + misalign));
+ BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(
+ memoryManager.GetPhysicalRegions(address + indexOffset - misalign, size + misalign),
+ BufferStage.IndexBuffer);
  misalignedOffset = (int)misalign >> shift;
 
  SetIndexBufferTexture(reservations, range, format);

src/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs

@@ -684,8 +684,8 @@ static float FixedToFloat(int fixedValue)
 
  if (hasCount)
  {
- var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
- var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferRange);
+ var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange, BufferStage.Indirect);
+ var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferRange, BufferStage.Indirect);
 
  if (indexed)
  {
@@ -698,7 +698,7 @@ static float FixedToFloat(int fixedValue)
  }
  else
  {
- var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
+ var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange, BufferStage.Indirect);
 
  if (indexed)
  {

src/Ryujinx.Graphics.Gpu/GpuContext.cs

@@ -393,17 +393,18 @@ internal void CreateHostSyncIfNeeded(HostSyncFlags flags)
 
  if (force || _pendingSync || (syncpoint && SyncpointActions.Count > 0))
  {
- Renderer.CreateSync(SyncNumber, strict);
-
  foreach (var action in SyncActions)
  {
  action.SyncPreAction(syncpoint);
  }
+
  foreach (var action in SyncpointActions)
  {
  action.SyncPreAction(syncpoint);
  }
 
+ Renderer.CreateSync(SyncNumber, strict);
+
  SyncNumber++;
 
  SyncActions.RemoveAll(action => action.SyncAction(syncpoint));

src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs

@@ -708,11 +708,11 @@ private void UpdateFromPool(TexturePool texturePool, SamplerPool samplerPool, Sh
  format = texture.Format;
  }
 
- _channel.BufferManager.SetBufferTextureStorage(entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format);
+ _channel.BufferManager.SetBufferTextureStorage(stage, entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format);
  }
  else
  {
- _channel.BufferManager.SetBufferTextureStorage(entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format);
+ _channel.BufferManager.SetBufferTextureStorage(stage, entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format);
  }
  }
  else if (isImage)
@@ -921,11 +921,11 @@ private void UpdateFromPool(TexturePool texturePool, SamplerPool samplerPool, Sh
  format = texture.Format;
  }
 
- _channel.BufferManager.SetBufferTextureStorage(entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format);
+ _channel.BufferManager.SetBufferTextureStorage(stage, entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format);
  }
  else
  {
- _channel.BufferManager.SetBufferTextureStorage(entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format);
+ _channel.BufferManager.SetBufferTextureStorage(stage, entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format);
  }
  }
  else if (isImage)

src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs

@@ -645,7 +645,7 @@ public void FlushIntoBuffer(TextureGroupHandle handle)
  }
  else
  {
- _flushBuffer = _context.Renderer.CreateBuffer((int)Storage.Size, BufferAccess.FlushPersistent);
+ _flushBuffer = _context.Renderer.CreateBuffer((int)Storage.Size, BufferAccess.HostMemory);
  _flushBufferImported = false;
  }