Merge pull request #23 from Nexus-Mods/rework-entity-loading

Cache entities as a block of data, and read values on-demand

benchmarks/NexusMods.MneumonicDB.Benchmarks/Benchmarks/ReadTests.cs

@@ -19,6 +19,7 @@ public class ReadTests : ABenchmark
     private IDb _db = null!;
     private EntityId[] _entityIds = null!;
     private EntityId _readId;
+    private File[] _preLoaded = null!;
 
     [Params(1, 1000, MaxCount)] public int Count { get; set; } = MaxCount;
 
@@ -29,7 +30,7 @@ public async Task Setup()
         var tx = Connection.BeginTransaction();
         var entityIds = new List<EntityId>();
 
-        var modId = new Mod(tx)
+        var tmpMod = new Mod(tx)
         {
             Name = "TestMod",
             Source = new Uri("https://www.nexusmods.com"),
@@ -43,9 +44,9 @@ public async Task Setup()
                 Hash = Hash.From((ulong)i),
                 Path = $"C:\\test_{i}.txt",
                 Size = Size.From((ulong)i),
-                ModId = modId.Id,
+                ModId = tmpMod.Header.Id,
             };
-            entityIds.Add(file.Id);
+            entityIds.Add(file.Header.Id);
         }
 
         var result = await tx.Commit();
@@ -55,6 +56,8 @@ public async Task Setup()
         _readId = _entityIds[_entityIds.Length / 2];
 
         _db = Connection.Db;
+
+        _preLoaded = _db.Get<File>(_entityIds).ToArray();
     }
 
     [Benchmark]
@@ -71,4 +74,11 @@ public long ReadAll()
         return _db.Get<File>(_entityIds)
             .Sum(e => (long)e.Size.Value);
     }
+
+    [Benchmark]
+    public long ReadAllPreloaded()
+    {
+        return _preLoaded
+            .Sum(e => (long)e.Size.Value);
+    }
 }

benchmarks/NexusMods.MneumonicDB.Benchmarks/NexusMods.MneumonicDB.Benchmarks.csproj

@@ -9,6 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="BenchmarkDotNet" Version="0.13.12"/>
+        <PackageReference Include="JetBrains.Profiler.Api" Version="1.4.2" />
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0"/>
         <PackageReference Update="JetBrains.Annotations" Version="2023.3.0"/>
     </ItemGroup>

benchmarks/NexusMods.MneumonicDB.Benchmarks/Program.cs

@@ -2,10 +2,13 @@
 
 using System;
 using System.Diagnostics;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Running;
+using JetBrains.Profiler.Api;
 using NexusMods.MneumonicDB.Benchmarks.Benchmarks;
 
 
-//#if DEBUG
+#if DEBUG
 
 var benchmark = new ReadTests
 {
@@ -14,15 +17,26 @@
 
 var sw = Stopwatch.StartNew();
 await benchmark.Setup();
+
 long result = 0;
-for (var i = 0; i < 10000; i++) result = benchmark.ReadAll();
+
+//MeasureProfiler.StartCollectingData();
+MemoryProfiler.CollectAllocations(true);
+for (var i = 0; i < 100000; i++)
+    result = benchmark.ReadAllPreloaded();
+MemoryProfiler.CollectAllocations(false);
+
+//MeasureProfiler.SaveData();
 Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result);
 
 
-/*
+
+
+
 #else
 
 BenchmarkRunner.Run<ReadTests>();
 
 #endif
-*/
+
+

src/NexusMods.MneumonicDB.Abstractions/IAttribute.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Runtime.CompilerServices;
+using NexusMods.MneumonicDB.Abstractions.Models;
 
 namespace NexusMods.MneumonicDB.Abstractions;
 
@@ -45,6 +47,7 @@ public interface IAttribute
     ///     Gets the type of the read datom for the given attribute.
     /// </summary>
     Type GetReadDatomType();
+
 }
 
 /// <summary>
@@ -65,4 +68,9 @@ public interface IAttribute<TVal> : IAttribute
     /// <param name="v"></param>
     /// <returns></returns>
     public static abstract IWriteDatom Assert(EntityId e, TVal v);
+
+    /// <summary>
+    /// Gets the serializer for the attribute
+    /// </summary>
+    public IValueSerializer<TVal> Serializer { get; }
 }

src/NexusMods.MneumonicDB.Abstractions/ICommitResult.cs

@@ -19,7 +19,7 @@ public interface ICommitResult
     /// <summary>
     ///   Remaps a ReadModel to a new instance with the new ids
     /// </summary>
-    public T Remap<T>(T model) where T : IReadModel;
+    public T Remap<T>(T model) where T : struct, IEntity;
 
     /// <summary>
     ///     Gets the new TxId after the commit

src/NexusMods.MneumonicDB.Abstractions/IDb.cs

@@ -24,7 +24,13 @@ public interface IDb : IDisposable
     ///     Returns a read model for each of the given entity ids.
     /// </summary>
     public IEnumerable<TModel> Get<TModel>(IEnumerable<EntityId> ids)
-        where TModel : IReadModel;
+        where TModel : struct, IEntity;
+
+    /// <summary>
+    /// Gets a single attribute value for the given entity id.
+    /// </summary>
+    public TValue Get<TAttribute, TValue>(ref ModelHeader header, EntityId id)
+        where TAttribute : IAttribute<TValue>;
 
 
     /// <summary>
@@ -34,14 +40,14 @@ public IEnumerable<TModel> Get<TModel>(IEnumerable<EntityId> ids)
     /// <typeparam name="TModel"></typeparam>
     /// <returns></returns>
     public TModel Get<TModel>(EntityId id)
-        where TModel : IReadModel;
+        where TModel : struct, IEntity;
 
     /// <summary>
     ///     Gets a read model for every enitity that references the given entity id
     ///     with the given attribute.
     /// </summary>
-    public IEnumerable<TModel> GetReverse<TAttribute, TModel>(EntityId id)
-        where TModel : IReadModel
+    public TModel[] GetReverse<TAttribute, TModel>(EntityId id)
+        where TModel : struct, IEntity
         where TAttribute : IAttribute<EntityId>;
 
     public IEnumerable<IReadDatom> Datoms(EntityId id);

src/NexusMods.MneumonicDB.Abstractions/ITransaction.cs

@@ -19,14 +19,6 @@ public interface ITransaction
     /// <returns></returns>
     EntityId TempId();
 
-    /// <summary>
-    ///     Adds a new read model to the transaction, the datoms are extracted from the read model
-    ///     as asserts for each property with the FromAttribute
-    /// </summary>
-    /// <param name="model"></param>
-    void Add<TReadModel>(TReadModel model)
-        where TReadModel : IReadModel;
-
     /// <summary>
     ///     Adds a new datom to the transaction
     /// </summary>
@@ -41,4 +33,6 @@ void Add<TAttribute, TVal>(EntityId entityId, TVal val)
     ///     Commits the transaction
     /// </summary>
     Task<ICommitResult> Commit();
+
+    ModelHeader New();
 }

src/NexusMods.MneumonicDB.Abstractions/Models/IEntity.cs

@@ -0,0 +1,8 @@
+using System.Collections.Generic;
+
+namespace NexusMods.MneumonicDB.Abstractions.Models;
+
+public interface IEntity
+{
+    public ModelHeader Header { get; }
+}

src/NexusMods.MneumonicDB.Abstractions/Models/ModelHeader.cs

@@ -0,0 +1,16 @@
+namespace NexusMods.MneumonicDB.Abstractions.Models;
+
+public struct ModelHeader
+{
+    public EntityId Id;
+    public IDb Db;
+    public ITransaction? Tx;
+    public object InlineCache;
+
+    public T[] GetReverse<TAttribute, T>()
+        where TAttribute : IAttribute<EntityId>,
+        new() where T : struct, IEntity
+    {
+        return Db.GetReverse<TAttribute, T>(Id);
+    }
+}

src/NexusMods.MneumonicDB.Abstractions/ScalarAttribute.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Buffers;
 using NexusMods.MneumonicDB.Abstractions.Internals;
+using NexusMods.MneumonicDB.Abstractions.Models;
 
 namespace NexusMods.MneumonicDB.Abstractions;
 
@@ -76,10 +77,11 @@ public void SetSerializer(IValueSerializer serializer)
     public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan<byte> value, TxId tx,
         bool isRetract)
     {
+        _serializer.Read(value, out var val);
         return new ReadDatom
         {
             E = entityId,
-            V = Read(value),
+            V = val,
             T = tx,
             IsRetract = isRetract
         };
@@ -107,6 +109,20 @@ public static IWriteDatom Assert(EntityId e, TValueType v)
         };
     }
 
+    /// <inheritdoc />
+    public IValueSerializer<TValueType> Serializer => _serializer;
+
+    public static TValueType Get(ref ModelHeader model)
+    {
+        return model.Db.Get<TAttribute, TValueType>(ref model, model.Id);
+    }
+
+    /// <inheritdoc />
+    public static void Add(ref ModelHeader model, TValueType value)
+    {
+        model.Tx!.Add<TAttribute, TValueType>(model.Id, value);
+    }
+
     /// <inheritdoc />
     public TValueType Read(ReadOnlySpan<byte> buffer)
     {

src/NexusMods.MneumonicDB/CommitResult.cs

@@ -11,9 +11,9 @@ public class CommitResult(IDb db, IDictionary<EntityId, EntityId> remaps) : ICom
     public EntityId this[EntityId id] =>
         remaps.TryGetValue(id, out var found) ? found : id;
 
-    public T Remap<T>(T model) where T : IReadModel
+    public T Remap<T>(T model) where T : struct, IEntity
     {
-        return db.Get<T>(remaps[model.Id]);
+        return db.Get<T>(remaps[model.Header.Id]);
     }
 
     /// <inheritdoc />

src/NexusMods.MneumonicDB/Connection.cs

@@ -16,16 +16,13 @@ public class Connection : IConnection
 {
     private readonly object _lock = new();
     private readonly IDatomStore _store;
-    internal readonly ModelReflector<Transaction> ModelReflector;
-    private ulong _nextEntityId = Ids.MinId(Ids.Partition.Entity);
 
     /// <summary>
     ///     Main connection class, co-ordinates writes and immutable reads
     /// </summary>
     private Connection(IDatomStore store)
     {
         _store = store;
-        ModelReflector = new ModelReflector<Transaction>(store);
     }