diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs deleted file mode 100644 index e8844885..00000000 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Storage; -using NexusMods.EventSourcing.TestModel; - -namespace NexusMods.EventSourcing.Benchmarks; - -public static class AppHost -{ - public static IServiceProvider Create() - { - var builder = Host.CreateDefaultBuilder() - .ConfigureServices(services => - { - services.AddEventSourcingStorage() - .AddEventSourcing() - .AddTestModel(); - }); - - return builder.Build().Services; - } - - public static async Task CreateConnection(IServiceProvider provider) - { - return await Connection.Start(provider); - } - -} diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ABenchmark.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ABenchmark.cs new file mode 100644 index 00000000..7b441734 --- /dev/null +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ABenchmark.cs @@ -0,0 +1,52 @@ +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage; +using NexusMods.EventSourcing.Storage.Abstractions; +using NexusMods.EventSourcing.Storage.InMemoryBackend; +using NexusMods.EventSourcing.TestModel; +using NexusMods.Paths; +using Xunit; + +namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; + +public class ABenchmark : IAsyncLifetime +{ + private IHost _host = null!; + protected IConnection Connection = null!; + + public IServiceProvider Services => _host.Services; + + public async Task InitializeAsync() + { + var builder = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddEventSourcingStorage() + .AddRocksDbBackend() + .AddEventSourcing() + .AddTestModel() + .AddDatomStoreSettings(new DatomStoreSettings + { + Path = FileSystem.Shared.FromUnsanitizedFullPath("benchmarks" + Guid.NewGuid()) + }); + }); + + _host = builder.Build(); + Connection = await NexusMods.EventSourcing.Connection.Start(Services); + } + + public async Task DisposeAsync() + { + var path = Services.GetRequiredService().Path; + await _host.StopAsync(); + _host.Dispose(); + + if (path.DirectoryExists()) + path.DeleteDirectory(); + if (path.FileExists) + path.Delete(); + } +} diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs index d93212bf..5f172c6e 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs @@ -13,20 +13,12 @@ namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; [MemoryDiagnoser] -public class ReadTests : IAsyncLifetime +public class ReadTests : ABenchmark { - private IConnection _connection = null!; - private List _entityIdsAscending = null!; - private List _entityIdsDescending = null!; - private List _entityIdsRandom = null!; - private readonly IServiceProvider _services; private EntityId _readId; private IDb _db = null!; + private EntityId[] _entityIds = null!; - public ReadTests() - { - _services = AppHost.Create(); - } private const int MaxCount = 10000; @@ -34,9 +26,9 @@ public ReadTests() public async Task Setup() { await InitializeAsync(); - var tx = _connection.BeginTransaction(); + var tx = Connection.BeginTransaction(); var entityIds = new List(); - for (var i = 0; i < MaxCount; i++) + for (var i = 0; i < Count; i++) { var file = new File(tx) { @@ -48,42 +40,16 @@ public async Task Setup() } var result = await tx.Commit(); - entityIds = entityIds.Select(e => result[e]).ToList(); - _entityIdsAscending = entityIds.OrderBy(id => id.Value).ToList(); - _entityIdsDescending = entityIds.OrderByDescending(id => id.Value).ToList(); + _entityIds = entityIds.Select(e => result[e]).ToArray(); - var idArray = entityIds.ToArray(); - Random.Shared.Shuffle(idArray); - _entityIdsRandom = idArray.ToList(); + _readId = _entityIds[_entityIds.Length / 2]; - _readId = Ids.Take(Count).Skip(Count / 2).First(); - - _db = _connection.Db; + _db = Connection.Db; } - [Params(1, 1000, MaxCount)] public int Count { get; set; } = MaxCount; - public enum SortOrder - { - Ascending, - Descending, - Random - } - - - //[Params(SortOrder.Ascending, SortOrder.Descending, SortOrder.Random)] - public SortOrder Order { get; set; } = SortOrder.Descending; - - public List Ids => Order switch - { - SortOrder.Ascending => _entityIdsAscending, - SortOrder.Descending => _entityIdsDescending, - SortOrder.Random => _entityIdsRandom, - _ => throw new ArgumentOutOfRangeException() - }; - [Benchmark] public ulong ReadFiles() { @@ -92,13 +58,10 @@ public ulong ReadFiles() return sum; } - public async Task InitializeAsync() - { - _connection = await AppHost.CreateConnection(_services); - } - - public Task DisposeAsync() + [Benchmark] + public long ReadAll() { - return Task.CompletedTask; + return _db.Get(_entityIds) + .Sum(e => (long)e.Index); } } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs index d093c8ca..b5b86b2f 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs @@ -11,6 +11,7 @@ namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; public class WriteTests { + /* private readonly IConnection _connection; public WriteTests() @@ -51,5 +52,6 @@ public async Task AddFiles() loaded.Should().NotBeNull("the entity should be in the database"); } } + */ } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj index b9cbe311..89c1d117 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj @@ -9,7 +9,8 @@ - + + diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs index d15d7c13..3658cae9 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs @@ -6,7 +6,7 @@ using NexusMods.EventSourcing.Benchmarks.Benchmarks; -#if DEBUG +//#if DEBUG var benchmark = new ReadTests { @@ -15,14 +15,20 @@ var sw = Stopwatch.StartNew(); await benchmark.Setup(); -ulong result = 0; -for (var i = 0; i < 1000000; i++) +long result = 0; +for (var i = 0; i < 10000; i++) { - result = benchmark.ReadFiles(); + result = benchmark.ReadAll(); } Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result); + + + +/* #else BenchmarkRunner.Run(); #endif +*/ + diff --git a/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj b/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj index d9d1b42f..a0f85d83 100644 --- a/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj +++ b/benchmarks/OneBillionDatomsTest/OneBillionDatomsTest.csproj @@ -8,7 +8,8 @@ - + + diff --git a/benchmarks/OneBillionDatomsTest/Program.cs b/benchmarks/OneBillionDatomsTest/Program.cs index cbd89993..82c3fdd7 100644 --- a/benchmarks/OneBillionDatomsTest/Program.cs +++ b/benchmarks/OneBillionDatomsTest/Program.cs @@ -16,6 +16,7 @@ { s.AddEventSourcingStorage() .AddEventSourcing() + .AddRocksDbBackend() .AddTestModel() .AddSingleton(_ => new DatomStoreSettings { diff --git a/docs/IndexFormat.md b/docs/IndexFormat.md index d3d07c7f..d339247b 100644 --- a/docs/IndexFormat.md +++ b/docs/IndexFormat.md @@ -5,9 +5,7 @@ hide: ## Index Format -The index format of the framework follows fairly closely to one found in Datomic, although differences likely exist due it having -a different set of constraints and requirements. The base format of the system is a sorted set of tuples. However each tuple -exists in multiple indexes with a different sorting and conflict resolution strategy for each. +The index format of the framework will look familiar to those who have used Datomic in the past, although there's some very critical In general data flows through 4 core indexes: diff --git a/src/NexusMods.EventSourcing.Abstractions/AttributeId.cs b/src/NexusMods.EventSourcing.Abstractions/AttributeId.cs index 536ebaf9..a2fd0d6b 100644 --- a/src/NexusMods.EventSourcing.Abstractions/AttributeId.cs +++ b/src/NexusMods.EventSourcing.Abstractions/AttributeId.cs @@ -13,4 +13,9 @@ public readonly partial struct AttributeId /// /// public EntityId ToEntityId() => EntityId.From(Value); + + /// + /// Minimum value for an AttributeId. + /// + public static AttributeId Min => new(ulong.MinValue); } diff --git a/src/NexusMods.EventSourcing.Abstractions/Datom.cs b/src/NexusMods.EventSourcing.Abstractions/Datom.cs deleted file mode 100644 index 91dbe9ac..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/Datom.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A untyped tuple of (E, A, T, F, V) values. -/// -public readonly struct Datom -{ - /// - /// Entity id. - /// - public EntityId E { get; init; } - - /// - /// Attribute id - /// - public AttributeId A { get; init; } - - /// - /// TX id - /// - public TxId T { get; init; } - - /// - /// Value Data - /// - public ReadOnlyMemory V { get; init; } - - /// - /// A datom with the maximum possible values for each field. - /// - public static Datom Max = new() - { - E = EntityId.From(ulong.MaxValue), - A = AttributeId.From(ulong.MaxValue), - T = TxId.MaxValue, - V = ReadOnlyMemory.Empty - }; - - /// - /// Assumes the value is a struct and unmarshals it. - /// - public T Unmarshal() where T : struct - { - return MemoryMarshal.Read(V.Span); - } - - /// - public override string ToString() - { - return $"({E}, {A}, {T}, {Convert.ToHexString(V.Span)}))"; - } -} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ExtensionMethods.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ExtensionMethods.cs new file mode 100644 index 00000000..656ef8f5 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ExtensionMethods.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; + +namespace NexusMods.EventSourcing.Abstractions.DatomIterators; + +/// +/// Extension methods for the IIterator interface +/// +public static class ExtensionMethods +{ + /// + /// Seeks to the given entity id in the iterator, assumes other values are 0 + /// + public static IIterator SeekTo(this TParent parent, EntityId eid) + where TParent : ISeekableIterator + { + var key = new KeyPrefix(); + key.Set(eid, AttributeId.Min, TxId.MinValue, false); + return parent.SeekTo(ref key); + } + + /// + /// Seeks to the given tx id in the iterator, assumes other values are 0 + /// + public static IIterator SeekTo(this TParent parent, TxId txId) + where TParent : ISeekableIterator + { + var key = new KeyPrefix(); + key.Set(EntityId.MinValue, AttributeId.Min, txId, false); + return parent.SeekTo(ref key); + } + + /// + /// Seeks to the given entity id in the iterator, assumes other values are 0 + /// + public static IIterator SeekTo(this TParent parent, AttributeId aid) + where TParent : ISeekableIterator + { + var key = new KeyPrefix(); + key.Set(EntityId.MinValue, aid, TxId.MinValue, false); + return parent.SeekTo(ref key); + } + + + /// + /// Seeks to the given attribute and value in the iterator, assumes that the other values are 0 + /// and that the value is unmanaged + /// + public static IIterator SeekTo(this TParent parent, AttributeId aid, TVal val) + where TParent : ISeekableIterator + where TVal : unmanaged + { + unsafe + { + Span span = stackalloc byte[sizeof(TVal) + sizeof(KeyPrefix)]; + var key = MemoryMarshal.Cast(span); + key[0].Set(EntityId.MinValue, aid, TxId.MinValue, false); + MemoryMarshal.Write(span.SliceFast(sizeof(KeyPrefix)), val); + return parent.Seek(span); + } + + } + + /// + /// Seeks to the given key prefix in the iterator, the value is null; + /// + public static IIterator SeekTo(this TParent parent, ref KeyPrefix prefix) + where TParent : ISeekableIterator + { + return parent.Seek(MemoryMarshal.CreateSpan(ref prefix, 1).CastFast()); + } + + /// + /// Reverses the order of the iterator so that a .Next will move backwards + /// + public static ReverseIterator Reverse(this TParent parent) + where TParent : IIterator + { + return new ReverseIterator(parent); + } + + /// + /// Converts the iterator to an IEnumerable of IReadDatom + /// + public static IEnumerable Resolve(this TParent parent) + where TParent : IIterator + { + var registry = parent.Registry; + while (parent.Valid) + { + yield return registry.Resolve(parent.Current); + parent.Next(); + } + } + + /// + /// Gets the current key prefix of the iterator + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static KeyPrefix CurrentKeyPrefix(this IIterator iterator) + { + return KeyPrefix.Read(iterator.Current); + } + + /// + /// Gets the current key prefix of the iterator as a value, assuming the value is unmanaged + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TValue CurrentValue(this TIterator iterator) + where TIterator : IIterator + where TValue : unmanaged + { + unsafe + { + return MemoryMarshal.Read(iterator.Current.SliceFast(sizeof(KeyPrefix))); + } + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// type + /// + public static WhileA While(this TParent parent, Type a) + where TParent : IIterator + { + var attrId = parent.Registry.GetAttributeId(a); + return new WhileA(attrId, parent); + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// type + /// + public static WhileUnmanagedV WhileUnmanagedV(this TParent parent, TVal tval) + where TParent : IIterator + where TVal : unmanaged, IEquatable + { + return new WhileUnmanagedV(tval, parent); + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// + public static WhileA While(this TParent parent, AttributeId a) + where TParent : IIterator + { + return new WhileA(a, parent); + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// + public static WhileTx While(this TParent parent, TxId txId) + where TParent : IIterator + { + return new WhileTx(txId, parent); + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// + public static IEnumerable Select(this TParent parent, Func f) + where TParent : IIterator + { + while (parent.Valid) + { + yield return f(parent); + parent.Next(); + } + } + + /// + /// Keeps the iterator valid while the attribute is equal to the given attribute + /// + public static WhileE While(this TParent parent, EntityId e) + where TParent : IIterator + { + return new WhileE(e, parent); + } + +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IDatomSource.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IDatomSource.cs new file mode 100644 index 00000000..4b50d54e --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IDatomSource.cs @@ -0,0 +1,12 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.DatomIterators; + +/// +/// The base interface for a datom iterator, also implements +/// IDisposable to allow for cleanup +/// +public interface IDatomSource : ISeekableIterator, IDisposable +{ + +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IIterator.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IIterator.cs new file mode 100644 index 00000000..c477c6b3 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/IIterator.cs @@ -0,0 +1,38 @@ +using System; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Abstractions.DatomIterators; + +/// +/// Base interface for an iterator, that allows for moving forward and backwards +/// over a sequence of spans +/// +public interface IIterator +{ + /// + /// True if the iterator is valid + /// + public bool Valid { get; } + + /// + /// Advance the iterator to the next element + /// + public void Next(); + + /// + /// Move to the previous element + /// + public void Prev(); + + /// + /// The current datom, this span is valid as until the next call to + /// .Next() or .Prev(); + /// + public ReadOnlySpan Current { get; } + + /// + /// Gets the registry for the attributes used to look up attribute types based on + /// the attribute ids + /// + public IAttributeRegistry Registry { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ISeekableIterator.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ISeekableIterator.cs new file mode 100644 index 00000000..48278ca7 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ISeekableIterator.cs @@ -0,0 +1,27 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.DatomIterators; + +/// +/// An iterator that can seek to the end, start or beginning. +/// +public interface ISeekableIterator +{ + /// + /// Move to the last element, returning this, casted to an + /// IIterator + /// + public IIterator SeekLast(); + + /// + /// Seek to the first datom before the given datom, returns this iterator + /// casted to an IIterator + /// + public IIterator Seek(ReadOnlySpan datom); + + /// + /// Set the iterator to the start of the datoms, returning this, casted + /// to an IIterator + /// + public IIterator SeekStart(); +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ReverseIterator.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ReverseIterator.cs new file mode 100644 index 00000000..a043bb7b --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/ReverseIterator.cs @@ -0,0 +1,38 @@ +using System; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Abstractions; + +/// +/// Reverses the order of the iterator so that a .Next will move backwards +/// +public class ReverseIterator : IIterator where TParent : IIterator +{ + private readonly TParent _parent; + internal ReverseIterator(TParent parent) + { + _parent = parent; + } + + /// + public bool Valid => _parent.Valid; + + /// + public void Next() + { + _parent.Prev(); + } + + /// + public void Prev() + { + _parent.Next(); + } + + /// + public ReadOnlySpan Current => _parent.Current; + + /// + public IAttributeRegistry Registry => _parent.Registry; +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomIterators/While.cs b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/While.cs new file mode 100644 index 00000000..79ff33dd --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/DatomIterators/While.cs @@ -0,0 +1,94 @@ +using System; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Abstractions.DatomIterators; + +/// +/// Iterates over the datoms while the attribute is equal to the given value +/// +public class WhileA(AttributeId a, TParent parent) : IIterator + where TParent : IIterator +{ + /// + public bool Valid => parent.Valid && this.CurrentKeyPrefix().A == a; + + /// + public void Next() => parent.Next(); + + /// + public void Prev() => parent.Prev(); + + /// + public ReadOnlySpan Current => parent.Current; + + /// + public IAttributeRegistry Registry => parent.Registry; +} + + +/// +/// Iterates over the datoms while the attribute is equal to the given value +/// +public class WhileE(EntityId e, TParent parent) : IIterator + where TParent : IIterator +{ + /// + public bool Valid => parent.Valid && this.CurrentKeyPrefix().E == e; + + /// + public void Next() => parent.Next(); + + /// + public void Prev() => parent.Prev(); + + /// + public ReadOnlySpan Current => parent.Current; + + /// + public IAttributeRegistry Registry => parent.Registry; +} + +/// +/// Iterates over the datoms while the attribute is equal to the given value +/// +public class WhileTx(TxId txId, TParent parent) : IIterator + where TParent : IIterator +{ + /// + public bool Valid => parent.Valid && this.CurrentKeyPrefix().T == txId; + + /// + public void Next() => parent.Next(); + + /// + public void Prev() => parent.Prev(); + + /// + public ReadOnlySpan Current => parent.Current; + + /// + public IAttributeRegistry Registry => parent.Registry; +} + +/// +/// Iterates over the datoms while the value (unmanaged) is equal to the given value +/// +public class WhileUnmanagedV(TValue v, TParent parent) : IIterator + where TParent : IIterator + where TValue : unmanaged, IEquatable +{ + /// + public bool Valid => parent.Valid && parent.CurrentValue().Equals(v); + + /// + public void Next() => parent.Next(); + + /// + public void Prev() => parent.Prev(); + + /// + public ReadOnlySpan Current => parent.Current; + + /// + public IAttributeRegistry Registry => parent.Registry; +} diff --git a/src/NexusMods.EventSourcing.Abstractions/DatomStoreTransactResult.cs b/src/NexusMods.EventSourcing.Abstractions/DatomStoreTransactResult.cs deleted file mode 100644 index 49fff4c2..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/DatomStoreTransactResult.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Collections.Generic; -using System.Collections.ObjectModel; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Result of a transact operation, contains the new transaction id and any entity remaps that were performed -/// during the transaction. -/// -public record DatomStoreTransactResult(TxId TxId, Dictionary Remaps); diff --git a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs index 938b6a22..07d3dca0 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs @@ -34,17 +34,12 @@ public interface IAttribute /// public Symbol Id { get; } - /// - /// Converts the datom to a typed datom - /// - /// - /// - IReadDatom Resolve(Datom datom); + bool IsIndexed { get; } /// /// Converts the given values into a typed datom /// - IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx); + IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx, bool isRetract); /// /// Gets the type of the read datom for the given attribute. diff --git a/src/NexusMods.EventSourcing.Abstractions/IAttributeRegistry.cs b/src/NexusMods.EventSourcing.Abstractions/IAttributeRegistry.cs deleted file mode 100644 index eb98282b..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IAttributeRegistry.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.Buffers; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A registry of attributes and serializers that supports operations that requires converting -/// between the database IDs, the code-level attributes and the native values -/// -public interface IAttributeRegistry -{ - /// - /// Compares the given values in the given spans assuming both are tagged with the given attribute - /// - public int CompareValues(AttributeId id, ReadOnlySpan a, ReadOnlySpan b); - - /// - /// Sets the attribute id and value in the given datom based on the given attribute and value - /// - void Explode(ref StackDatom datom, TValueType valueType, IBufferWriter writer) where TAttribute : IAttribute; -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IBlobColumn.cs b/src/NexusMods.EventSourcing.Abstractions/IBlobColumn.cs deleted file mode 100644 index 6802686b..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IBlobColumn.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A column that contains an array of blobs. -/// -public interface IBlobColumn : IEnumerable> -{ - /// - /// Get the blob at the given index. - /// - public ReadOnlyMemory this[int idx] { get; } - - /// - /// The number of blobs in the column. - /// - public int Length { get; } - - /// - /// Packs the column into a lightweight compressed form. - /// - /// - public IBlobColumn Pack(); - - /// - /// Writes the column to the given writer. - /// - void WriteTo(TWriter writer) where TWriter : IBufferWriter; -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IColumn.cs b/src/NexusMods.EventSourcing.Abstractions/IColumn.cs deleted file mode 100644 index 14c068e5..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IColumn.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Buffers; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Generic interface for a value column. -/// -/// -public interface IColumn -{ - /// - /// Gets the item at the specified index, this may be slow depending on the - /// encoding of the data. - /// - /// - public T this[int index] { get; } - - /// - /// Gets the length of the column in rows. - /// - public int Length { get; } - - /// - /// Packs the column into a more efficient representation. - /// - /// - public IColumn Pack(); - - /// - /// Writes the column to the specified writer. - /// - /// - /// - void WriteTo(TWriter writer) where TWriter : IBufferWriter; - - /// - /// Copies the column to the specified destination. - /// - /// - void CopyTo(Span destination); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs index 739482b7..c92dc242 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs @@ -27,14 +27,7 @@ public interface IConnection public ITransaction BeginTransaction(); /// - /// A sequential stream of commits to the database. + /// A sequential stream of database revisions. /// - public IObservable<(TxId TxId, IReadOnlyCollection Datoms)> Commits { get; } - - /// - /// Gets the active read model for the given entity id, this entity will - /// automatically update as new commits are made to the database that modify - /// its state. It will update via INotifyPropertyChanged. - /// - public T GetActive(EntityId id) where T : IActiveReadModel; + public IObservable Revisions { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomComparator.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomComparator.cs deleted file mode 100644 index c5962ad8..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomComparator.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.Collections.Generic; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Interface for a comparator. This is the backbone of the indexes construction -/// code in the storage layer. It is used to sort datoms in arrays, and to distribute -/// them between the B+ Tree nodes. Also, it is used for the sorted merge routines -/// used all throughout the code. -/// -public interface IDatomComparator -{ - /// - /// Get the enum value of the sort order. - /// - public SortOrders SortOrder { get; } - - /// - /// Compares two datoms and returns a value indicating whether one is less than, equal to, or greater than the other. - /// - public int Compare(in Datom x, in Datom y); - - /// - /// Make a comparer for the given datoms using indices as a key to the datoms. This is used - /// so that the .NET sorting algorithm can sort be used to sort datoms, but yet we can still - /// only retrieve the parts of of the datoms that we need. In other words if to datoms differ - /// only in the E part, we don't need to compare the A, T and V parts (V being a rather expensive - /// comparison to make). This comparer is unsafe as the MemoryDatom contains raw pointers to - /// arrays of integers being sorted in memory. - /// - public unsafe IComparer MakeComparer(MemoryDatom datoms) - where TBlob : IBlobColumn; -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomSink.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomSink.cs deleted file mode 100644 index a5fb798c..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomSink.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A sink is a typed interface that accepts the parts of a datom as values. It is generic and strongly typed -/// so that most of the data is transferred via the stack and does not require boxing. Think of all of this -/// something like IEnumerable, but as a pushed base interface instead of a pulled one. -/// -public interface IDatomSink -{ - /// - /// Inject a datom into the sink. - /// - public void Datom(ulong e, TVal v, bool isAssert) - where TAttr : IAttribute; -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs index ee0e15da..4a278045 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq.Expressions; using System.Threading.Tasks; +using NexusMods.EventSourcing.Abstractions.Internals; namespace NexusMods.EventSourcing.Abstractions; @@ -21,35 +22,19 @@ public interface IDatomStore : IDisposable /// /// Transacts (adds) the given datoms into the store. /// - public Task Transact(IEnumerable datoms); + public Task Transact(IEnumerable datoms); /// /// An observable of the transaction log, for getting the latest changes to the store. /// - public IObservable<(TxId TxId, IReadOnlyCollection Datoms)> TxLog { get; } + public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog { get; } /// /// Gets the latest transaction id found in the log. /// public TxId AsOfTxId { get; } - /// - /// Returns all the most recent datoms (less than or equal to txId) with the given attribute. - /// - IEnumerable Where(TxId txId) where TAttr : IAttribute; - - - - /// - /// Returns all the most recent datoms (less than or equal to txId) with the given attribute. - /// - IEnumerable Where(TxId txId, EntityId id); - - /// - /// Resolves the given datoms to typed datoms. - /// - /// - IEnumerable Resolved(IEnumerable datoms); + IAttributeRegistry Registry { get; } /// /// Registers new attributes with the store. These should already have been transacted into the store. @@ -57,12 +42,6 @@ public interface IDatomStore : IDisposable /// Task RegisterAttributes(IEnumerable newAttrs); - /// - /// Gets the attributeId for the given attribute. And returns an expression that reads the attribute - /// value from the expression valueSpan. - /// - Expression GetValueReadExpression(Type attribute, Expression valueSpan, out AttributeId attributeId); - /// /// Gets the entities that have the given attribute that reference the given entity id. /// @@ -95,12 +74,21 @@ IEnumerable GetReferencesToEntityThroughAttribute(EntityId EntityId GetMaxEntityId(); /// - /// Gets the most recent transaction id for the given entity id. + /// Gets the type of the read datom for the given attribute. /// - TxId GetMostRecentTxId(); + Type GetReadDatomType(Type attribute); + /// - /// Gets the type of the read datom for the given attribute. + /// Get all the datoms in a given index, not super useful as this may return a TOOON of datoms. /// - Type GetReadDatomType(Type attribute); + /// + /// + /// + public IEnumerable Datoms(ISnapshot snapshot, IndexType type); + + /// + /// Create a snapshot of the current state of the store. + /// + ISnapshot GetSnapshot(); } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDb.cs b/src/NexusMods.EventSourcing.Abstractions/IDb.cs index ee422fc5..00b75088 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDb.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDb.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -6,7 +7,7 @@ namespace NexusMods.EventSourcing.Abstractions; /// /// Represents an immutable database fixed to a specific TxId. /// -public interface IDb +public interface IDb : IDisposable { /// /// Gets the basis TxId of the database. @@ -42,8 +43,13 @@ public IEnumerable GetReverse(EntityId id) where TModel : IReadModel where TAttribute : IAttribute; + public IEnumerable Datoms(EntityId id); + /// - /// Reloads the active read model with the latest state from the database. + /// Gets the datoms for the given transaction id. /// - void Reload(TOuter aActiveReadModel) where TOuter : IActiveReadModel; + public IEnumerable Datoms(TxId txId); + + public IEnumerable Datoms(IndexType type) + where TAttribute : IAttribute; } diff --git a/src/NexusMods.EventSourcing.Abstractions/IEnumerableExtensions.cs b/src/NexusMods.EventSourcing.Abstractions/IEnumerableExtensions.cs deleted file mode 100644 index 5a036230..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IEnumerableExtensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Collections.Generic; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Extensions for . -/// -public static class IEnumerableExtensions -{ - - /// - /// Unpacks the datoms in the given sequence returning typed versions of them. - /// - public static IEnumerable Typed(this IEnumerable datoms, IDatomStore store) - { - return store.Resolved(datoms); - } - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IIndex.cs b/src/NexusMods.EventSourcing.Abstractions/IIndex.cs deleted file mode 100644 index b195834d..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IIndex.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions; - -public interface IIndex -{ - public void Add(IWriteBatch batch, IWriteDatom datom); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadDatom.cs b/src/NexusMods.EventSourcing.Abstractions/IReadDatom.cs index a9e69c0a..2d0b23b7 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IReadDatom.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IReadDatom.cs @@ -23,4 +23,14 @@ public interface IReadDatom /// The transaction id of the datom. /// public TxId T { get; } + + /// + /// Gets the value as a object (possibly boxed). + /// + object ObjectValue { get; } + + /// + /// True if this is a retraction of a previous datom. + /// + public bool IsRetract { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/ISnapshot.cs b/src/NexusMods.EventSourcing.Abstractions/ISnapshot.cs new file mode 100644 index 00000000..0bd7cf9f --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/ISnapshot.cs @@ -0,0 +1,20 @@ +using System; +using NexusMods.EventSourcing.Abstractions.DatomIterators; + +namespace NexusMods.EventSourcing.Abstractions; + +/// +/// Represents a snapshot of the database at a specific point of time. Snapshots are immutable +/// and do not live past the life of the application, or after the IDisposable.Dispose method is called. +/// Using snapshots to query the database is the most efficient way, and is leveraged by the IDb interface, +/// to provide a read-only view of the database. +/// +public interface ISnapshot +{ + /// + /// Gets an iterator for the given index type. + /// + /// + /// + IDatomSource GetIterator(IndexType type); +} diff --git a/src/NexusMods.EventSourcing.Abstractions/ITransactionResult.cs b/src/NexusMods.EventSourcing.Abstractions/ITransactionResult.cs new file mode 100644 index 00000000..2ef237f5 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/ITransactionResult.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions; + +/// +/// The result of a transaction commit, contains metadata useful for looking up the results of the transaction +/// +public interface ITransactionResult +{ + /// + /// The new transaction id after the commit + /// + public TxId NewTx { get; } + + /// + /// The datoms that were added to the store as a result of the transaction + /// + public IReadOnlyCollection Added { get; } + + /// + /// Gets a fresh reference to the database after the transaction, this should be disposed when done. + /// + /// + public IDb NewDb(); + + /// + /// The time it took to commit the transaction + /// + public TimeSpan Elapsed { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/IWriteBatch.cs b/src/NexusMods.EventSourcing.Abstractions/IWriteBatch.cs deleted file mode 100644 index b82f583a..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IWriteBatch.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions; - -public interface IWriteBatch -{ - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IWriteDatom.cs b/src/NexusMods.EventSourcing.Abstractions/IWriteDatom.cs index f0f1daee..54e9be16 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IWriteDatom.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IWriteDatom.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using NexusMods.EventSourcing.Abstractions.Internals; namespace NexusMods.EventSourcing.Abstractions; @@ -12,7 +13,14 @@ namespace NexusMods.EventSourcing.Abstractions; public interface IWriteDatom { /// - /// Extracts the entity and attribute from the datom, and writes the value to the buffer + /// Extracts the entity and attribute from the datom, and writes the value to the buffer. /// - public void Explode(IAttributeRegistry registry, Func remapFn, ref StackDatom datom, TWriter writer) where TWriter : IBufferWriter; + public void Explode(IAttributeRegistry registry, Func remapFn, + out EntityId e, out AttributeId a, TWriter vWriter, out bool isRetract) + where TWriter : IBufferWriter; + + /// + /// The entity id for this datom + /// + public EntityId E { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IndexType.cs b/src/NexusMods.EventSourcing.Abstractions/IndexType.cs new file mode 100644 index 00000000..b05f5ab5 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/IndexType.cs @@ -0,0 +1,20 @@ +namespace NexusMods.EventSourcing.Abstractions; + +public enum IndexType : int +{ + // Transaction log, the final source of truth, used + // for replaying the database + TxLog, + // Primary index for looking up values on an entity + EAVTHistory, + EAVTCurrent, + // Indexes for asking what entities have this attribute? + AEVTHistory, + AEVTCurrent, + // Backref index for asking "who references this entity?" + VAETCurrent, + VAETHistory, + // Secondary index for asking "who has this value on this attribute?" + AVETCurrent, + AVETHistory +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Internals/IAttributeRegistry.cs b/src/NexusMods.EventSourcing.Abstractions/Internals/IAttributeRegistry.cs new file mode 100644 index 00000000..f3b05eff --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Internals/IAttributeRegistry.cs @@ -0,0 +1,41 @@ +using System; +using System.Buffers; + +namespace NexusMods.EventSourcing.Abstractions.Internals; + +/// +/// A registry of attributes and serializers that supports operations that requires converting +/// between the database IDs, the code-level attributes and the native values +/// +public interface IAttributeRegistry +{ + /// + /// Compares the given values in the given spans assuming both are tagged with the given attribute + /// + public int CompareValues(AttributeId id, ReadOnlySpan a, ReadOnlySpan b); + + /// + /// Sets the attribute id and value in the given datom based on the given attribute and value + /// + void Explode(out AttributeId attrId, TValueType valueType, TBufferWriter writer) + where TBufferWriter : IBufferWriter + where TAttribute : IAttribute; + + /// + /// Gets the unique symbol for the given attribute + /// + Symbol GetSymbolForAttribute(Type attribute); + + /// + /// Gets the attribute id for the given attribute type + /// + public AttributeId GetAttributeId(Type datomAttributeType); + + /// + /// Resolve the given KeyPrefix + Value into a datom + /// + /// + /// + public IReadDatom Resolve(ReadOnlySpan datom); + +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Internals/KeyPrefix.cs b/src/NexusMods.EventSourcing.Abstractions/Internals/KeyPrefix.cs new file mode 100644 index 00000000..f4be81b2 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Internals/KeyPrefix.cs @@ -0,0 +1,75 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace NexusMods.EventSourcing.Abstractions.Internals; + +/// +/// The system encodes keys as a 16 byte prefix followed by the actual key data, the format +/// of the value is defined by the IValueSerializer and the length is maintained by the datastore. +/// This KeyPrefix then contains the other parts of the Datom: EntityId, AttributeId, TxId, and Flags. +/// +/// Encodes and decodes the prefix of a key, the format is: +/// [AttributeId: 2bytes] +/// [TxId: 8bytes - 1 bit for the assert/retract flag] +/// [EntityID + PartitionID: 7bytes] +/// [IsRetract: 1bit] +/// +/// The Entity Id is created by taking the last 6 bytes of the id and combining it with +/// the partition id. So the encoding logic looks like this: +/// +/// packed = (e & 0x00FFFFFFFFFFFFFF) >> 8 | (e & 0xFFFFFFFFFFFF) << 8 +/// +[StructLayout(LayoutKind.Explicit, Size = Size)] +public struct KeyPrefix +{ + /// + /// Fixed size of the KeyPrefix + /// + public const int Size = 16; + + [FieldOffset(0)] + private ulong _upper; + [FieldOffset(8)] + private ulong _lower; + + public void Set(EntityId id, AttributeId attributeId, TxId txId, bool isRetract) + { + _upper = (ulong)attributeId << 48 | (ulong)txId & 0x0000FFFFFFFFFFFF; + _lower = (ulong)id & 0xFF00000000000000 | ((ulong)id & 0x0000FFFFFFFFFFFF) << 8 | (isRetract ? 1UL : 0UL); + } + + /// + /// The EntityId + /// + public EntityId E => (EntityId)((_lower & 0xFF00000000000000) | (_lower >> 8) & 0x0000FFFFFFFFFFFF); + + /// + /// True if this is a retraction + /// + public bool IsRetract => (_lower & 1) == 1; + + /// + /// The attribute id, maximum of 2^16 attributes are supported in the system + /// + public AttributeId A => (AttributeId)(_upper >> 48); + + /// + /// The transaction id, maximum of 2^63 transactions are supported in the system, but really + /// it's 2^56 as the upper 8 bits are used for the partition id. + /// + public TxId T => (TxId)Ids.MakeId(Ids.Partition.Tx, _upper & 0x0000FFFFFFFFFFFF); + + /// + public override string ToString() => $"E: {E}, A: {A}, T: {T}, Retract: {IsRetract}"; + + + /// + /// Gets the KeyPrefix from the given bytes + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static KeyPrefix Read(ReadOnlySpan bytes) + { + return MemoryMarshal.Read(bytes); + } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/MemoryDatom.cs b/src/NexusMods.EventSourcing.Abstractions/MemoryDatom.cs deleted file mode 100644 index b671cfb1..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/MemoryDatom.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Untyped memory datom, this is used to get raw access to a linear memory representation of several -/// datoms. -/// -/// DEPRECATED: This is should probably be replaced by chunks and a more structured memory layout. -/// -/// -public unsafe struct MemoryDatom -where T : IBlobColumn -{ - public EntityId* EntityIds; - public AttributeId* AttributeIds; - public TxId* TransactionIds; - public T Values; -} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/AActiveReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/AActiveReadModel.cs deleted file mode 100644 index f969a142..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/Models/AActiveReadModel.cs +++ /dev/null @@ -1,92 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Runtime.CompilerServices; - -namespace NexusMods.EventSourcing.Abstractions.Models; - -/// -/// Abstract class for active read models. -/// -/// -public abstract class AActiveReadModel : IActiveReadModel -where TOuter : AActiveReadModel, IActiveReadModel -{ - private IDb _basisDb = null!; - - /// - /// Base constructor for an active read model. - /// - /// - /// - protected AActiveReadModel(IDb basisDb, EntityId id) - { - Id = id; - BasisDb = basisDb; - Attach((TOuter)this, basisDb.Connection); - } - - private class Box - { - public T? Value { get; set; } - } - - /// - /// Attaches the active read model to the given connection, once the last reference to the model is gone, - /// the subscription is disposed. - /// - /// - /// - public static void Attach(TOuter model, IConnection connection) - { - var weakRef = new WeakReference(model); - var box = new Box(); - - box.Value = connection.Commits.Subscribe(changes => - { - if (weakRef.TryGetTarget(out var target)) - { - target._basisDb = connection.Db; - if (changes.Datoms.Any(d => d.E == target.Id)) - connection.Db.Reload(target); - } - else - { - box.Value?.Dispose(); - } - }); - - } - - /// - /// The current database this entity is using for its state. - /// - public IDb BasisDb - { - get => _basisDb; - set - { - _basisDb = value; - OnBasisDbChanged(); - } - } - - private void OnBasisDbChanged() - { - _basisDb.Reload((TOuter)this); - } - - /// - /// The identifier for the entity. - /// - public EntityId Id { get; set; } - - /// - public event PropertyChangedEventHandler? PropertyChanged; - - protected virtual void OnPropertyChanged([CallerMemberName] string? propertyName = null) - { - PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); - } -} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/IActiveReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/IActiveReadModel.cs deleted file mode 100644 index 177ad3a3..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/Models/IActiveReadModel.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.ComponentModel; - -namespace NexusMods.EventSourcing.Abstractions.Models; - -/// -/// A read model that automatically updates as new commits are made to the database that modify its state. -/// -public interface IActiveReadModel : INotifyPropertyChanged -{ - /// - /// The unique identifier of the entity. - /// - public EntityId Id { get; } -} diff --git a/src/NexusMods.EventSourcing.Abstractions/NexusMods.EventSourcing.Abstractions.csproj b/src/NexusMods.EventSourcing.Abstractions/NexusMods.EventSourcing.Abstractions.csproj index 1b285df1..784b51f2 100644 --- a/src/NexusMods.EventSourcing.Abstractions/NexusMods.EventSourcing.Abstractions.csproj +++ b/src/NexusMods.EventSourcing.Abstractions/NexusMods.EventSourcing.Abstractions.csproj @@ -5,11 +5,16 @@ - - - - + + + + + + + + + diff --git a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs index 58dd5acc..8e3989c8 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using NexusMods.EventSourcing.Abstractions.Internals; namespace NexusMods.EventSourcing.Abstractions; @@ -16,13 +17,26 @@ public class ScalarAttribute : IAttribute /// /// Create a new attribute /// - protected ScalarAttribute(string uniqueName = "") + protected ScalarAttribute(string uniqueName = "", + bool isIndexed = false, + bool keepHistory = true, + bool multiArity = false) { + IsIndexed = isIndexed; + KeepHistory = keepHistory; + MultiArity = multiArity; Id = uniqueName == "" ? Symbol.Intern(typeof(TAttribute).FullName!) : Symbol.InternPreSanitized(uniqueName); } + public bool MultiArity { get; } + + public bool KeepHistory { get; } + + /// + public bool IsIndexed { get; } + /// /// Create a new attribute from an already parsed guid /// @@ -67,26 +81,14 @@ public void SetSerializer(IValueSerializer serializer) public Symbol Id { get; } /// - public IReadDatom Resolve(Datom datom) - { - _serializer.Read(datom.V.Span, out var read); - return new ReadDatom - { - E = datom.E, - V = read, - T = datom.T - }; - } - - - /// - public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx) + public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx, bool isRetract) { return new ReadDatom { E = entityId, V = Read(value), - T = tx + T = tx, + IsRetract = isRetract }; } @@ -135,22 +137,25 @@ public override string ToString() return $"({E.Value:x}, {typeof(TAttribute).Name}, {V})"; } - public void Explode(IAttributeRegistry registry, Func remapFn, ref StackDatom datom, TWriter writer) + public void Explode(IAttributeRegistry registry, Func remapFn, + out EntityId e, out AttributeId a, TWriter vWriter, out bool isRetract) where TWriter : IBufferWriter { - datom.E = Ids.IsPartition(E.Value, Ids.Partition.Tmp) ? remapFn(E).Value : E.Value; + isRetract = false; + e = EntityId.From(Ids.IsPartition(E.Value, Ids.Partition.Tmp) ? remapFn(E).Value : E.Value); if (V is EntityId id) { var newId = remapFn(id); if (newId is TValueType recasted) { - registry.Explode(ref datom, recasted, writer); + registry.Explode(out a, recasted, vWriter); return; } } - registry.Explode(ref datom, V, writer); + registry.Explode(out a, V, vWriter); } + } /// @@ -158,6 +163,8 @@ public void Explode(IAttributeRegistry registry, Func public readonly record struct ReadDatom : IReadDatom { + private readonly ulong _tx; + /// /// The entity id for this datom /// @@ -171,7 +178,20 @@ public void Explode(IAttributeRegistry registry, Func /// The transaction id for this datom /// - public required TxId T { get; init; } + public TxId T + { + get => TxId.From(_tx >> 1); + init => _tx = (_tx & 1) | (value.Value << 1); + } + + /// + public bool IsRetract + { + get => (_tx & 1) == 1; + init => _tx = (_tx & ~1UL) | (value ? 1UL : 0); + } + + public object ObjectValue => V!; /// public override string ToString() diff --git a/src/NexusMods.EventSourcing.Abstractions/SortOrders.cs b/src/NexusMods.EventSourcing.Abstractions/SortOrders.cs deleted file mode 100644 index d9ffc687..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/SortOrders.cs +++ /dev/null @@ -1,28 +0,0 @@ -// ReSharper disable InconsistentNaming -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// Common sort orders for datoms. -/// -public enum SortOrders : byte -{ - /// - /// TX log order - TEAV - /// - TxLog, - - /// - /// Common index for looking up all datoms for an entity - /// - EATV, - - /// - /// Index for looking up all entities that have a given attribute - /// - AETV, - - /// - /// Index for looking up all entities that have a given value for a given attribute - /// - AVTE, -} diff --git a/src/NexusMods.EventSourcing.Abstractions/StackDatom.cs b/src/NexusMods.EventSourcing.Abstractions/StackDatom.cs index 3dfda887..a5298f5e 100644 --- a/src/NexusMods.EventSourcing.Abstractions/StackDatom.cs +++ b/src/NexusMods.EventSourcing.Abstractions/StackDatom.cs @@ -10,9 +10,24 @@ namespace NexusMods.EventSourcing.Abstractions; /// public ref struct StackDatom { + /// + /// The entity id + /// public ulong E; + + /// + /// The Attribute id + /// public ushort A; + + /// + /// The Transaction id + /// public ulong T; + + /// + /// The value span + /// public ReadOnlySpan V; /// diff --git a/src/NexusMods.EventSourcing.Abstractions/StoreResult.cs b/src/NexusMods.EventSourcing.Abstractions/StoreResult.cs new file mode 100644 index 00000000..1a77323d --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/StoreResult.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions; + +public class StoreResult +{ + public required TxId AssignedTxId { get; init; } + public required Dictionary Remaps { get; init; } + public required ISnapshot Snapshot { get; init; } + public required IReadOnlyCollection Datoms { get; init; } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/AIndex.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/AIndex.cs new file mode 100644 index 00000000..6b558f24 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/AIndex.cs @@ -0,0 +1,46 @@ +using System; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public abstract class AIndex(AttributeRegistry registry, TIndexStore store) : IIndex +where TA : IElementComparer +where TB : IElementComparer +where TC : IElementComparer +where TD : IElementComparer +where TE : IElementComparer +where TIndexStore : class, IIndexStore +{ + public int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + var cmp = TA.Compare(registry, a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(registry, a, b); + if (cmp != 0) return cmp; + + cmp = TC.Compare(registry, a, b); + if (cmp != 0) return cmp; + + cmp = TD.Compare(registry, a, b); + if (cmp != 0) return cmp; + + return TE.Compare(registry, a, b); + } + + public void Put(IWriteBatch batch, ReadOnlySpan datom) + { + batch.Add(store, datom); + } + + public void Delete(IWriteBatch batch, ReadOnlySpan datom) + { + batch.Delete(store, datom); + } + + public IDatomSource GetIterator() + { + return store.GetIterator(); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AComparer.cs new file mode 100644 index 00000000..c42e4862 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AComparer.cs @@ -0,0 +1,14 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +public class AComparer : IElementComparer +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + return MemoryMarshal.Read(a).A.CompareTo(MemoryMarshal.Read(b).A); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AssertComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AssertComparer.cs new file mode 100644 index 00000000..48e8abe1 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/AssertComparer.cs @@ -0,0 +1,14 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +public class AssertComparer : IElementComparer +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + return MemoryMarshal.Read(a).IsRetract.CompareTo(MemoryMarshal.Read(b).IsRetract); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/EComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/EComparer.cs new file mode 100644 index 00000000..414d7061 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/EComparer.cs @@ -0,0 +1,14 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +public class EComparer : IElementComparer +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + return MemoryMarshal.Read(a).E.CompareTo(MemoryMarshal.Read(b).E); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/TxComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/TxComparer.cs new file mode 100644 index 00000000..6ff526a0 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/TxComparer.cs @@ -0,0 +1,13 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +public class TxComparer : IElementComparer +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + return MemoryMarshal.Read(a).T.CompareTo(MemoryMarshal.Read(b).T); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/UnmanagedValueComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/UnmanagedValueComparer.cs new file mode 100644 index 00000000..5cf01efc --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/UnmanagedValueComparer.cs @@ -0,0 +1,25 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +/// +/// Unmanaged value comparer, assumes that the values will be of the same attribute and of type T. +/// +/// +public class UnmanagedValueComparer : IElementComparer + where T : unmanaged, IComparable +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + unsafe + { + if (a.Length < sizeof(KeyPrefix) || b.Length < sizeof(KeyPrefix)) + return a.Length.CompareTo(b.Length); + return MemoryMarshal.Read(a.SliceFast(sizeof(KeyPrefix))) + .CompareTo(MemoryMarshal.Read(b.SliceFast(sizeof(KeyPrefix)))); + } + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/ValueComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/ValueComparer.cs new file mode 100644 index 00000000..4564619d --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/ElementComparers/ValueComparer.cs @@ -0,0 +1,22 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; + +namespace NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; + +/// +/// Compares values and assumes that some previous comparator will guarantee that the values are of the same attribute. +/// +public class ValueComparer : IElementComparer +{ + public static int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b) + { + var attrA = MemoryMarshal.Read(a).A; + + unsafe + { + return registry.CompareValues(attrA, a.SliceFast(sizeof(KeyPrefix)), b.SliceFast(sizeof(KeyPrefix))); + } + } +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/IElementComparer.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/IElementComparer.cs new file mode 100644 index 00000000..e9b413fa --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/IElementComparer.cs @@ -0,0 +1,8 @@ +using System; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public interface IElementComparer +{ + public static abstract int Compare(AttributeRegistry registry, ReadOnlySpan a, ReadOnlySpan b); +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/IIndex.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/IIndex.cs new file mode 100644 index 00000000..8aefee21 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/IIndex.cs @@ -0,0 +1,13 @@ +using System; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public interface IIndex { + int Compare(ReadOnlySpan readOnlySpan, ReadOnlySpan readOnlySpan1); + IDatomSource GetIterator(); + void Delete(IWriteBatch batch, ReadOnlySpan span); + void Put(IWriteBatch batch, ReadOnlySpan span); +} + diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/IIndexStore.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/IIndexStore.cs new file mode 100644 index 00000000..aed04245 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/IIndexStore.cs @@ -0,0 +1,11 @@ +using System.Collections.Immutable; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public interface IIndexStore +{ + IndexType Type { get; } + IDatomSource GetIterator(); +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/IStoreBackend.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/IStoreBackend.cs new file mode 100644 index 00000000..d0064349 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/IStoreBackend.cs @@ -0,0 +1,65 @@ + +using System; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions.ElementComparers; +using NexusMods.Paths; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public interface IStoreBackend : IDisposable +{ + public IWriteBatch CreateBatch(); + + public void Init(AbsolutePath location); + + public void DeclareIndex(IndexType name) + where TA : IElementComparer + where TB : IElementComparer + where TC : IElementComparer + where TD : IElementComparer + where TF : IElementComparer; + + + public IIndex GetIndex(IndexType name); + + /// + /// Gets a snapshot of the current state of the store that will not change + /// during calls to GetIterator + /// + public ISnapshot GetSnapshot(); + + /// + /// Create an EAVT index + /// + public void DeclareEAVT(IndexType name) => + DeclareIndex + (name); + + /// + /// Create an AEVT index + /// + public void DeclareAEVT(IndexType name) => + DeclareIndex + (name); + + /// + /// Create an AEVT index + /// + public void DeclareTxLog(IndexType name) => + DeclareIndex + (name); + + /// + /// Create a backref index + /// + /// + public void DeclareVAET(IndexType name) => + DeclareIndex, AComparer, EComparer, TxComparer, AssertComparer>(name); + + /// + /// Create a backref index + /// + /// + public void DeclareAVET(IndexType name) => + DeclareIndex(name); +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/IWriteBatch.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/IWriteBatch.cs new file mode 100644 index 00000000..5be2537c --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/IWriteBatch.cs @@ -0,0 +1,12 @@ +using System; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +public interface IWriteBatch : IDisposable +{ + public void Commit(); + + public void Add(IIndexStore store, ReadOnlySpan key); + public void Delete(IIndexStore store, ReadOnlySpan key); + +} diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/KeyPrefix.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/KeyPrefix.cs new file mode 100644 index 00000000..113c463d --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/KeyPrefix.cs @@ -0,0 +1,6 @@ +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + + diff --git a/src/NexusMods.EventSourcing.Storage/Abstractions/RefCountedDisposable.cs b/src/NexusMods.EventSourcing.Storage/Abstractions/RefCountedDisposable.cs new file mode 100644 index 00000000..7791f92e --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/Abstractions/RefCountedDisposable.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; + +namespace NexusMods.EventSourcing.Storage.Abstractions; + +/// +/// A ref counted disposable wrapper around a disposable object. Most often used to track the +/// lifetime of a snapshot +/// +public class RefCountedDisposable(T inner) : IDisposable where T : IDisposable +{ + private int _refCount; + + public T Inner => inner; + + public IDisposable AddRef() + { + Interlocked.Increment(ref _refCount); + return this; + } + + public void Dispose() + { + if (Interlocked.Decrement(ref _refCount) == 0) + { + inner.Dispose(); + } + } +} diff --git a/src/NexusMods.EventSourcing.Storage/AttributeRegistry.cs b/src/NexusMods.EventSourcing.Storage/AttributeRegistry.cs index 6fd359d3..e69f01c3 100644 --- a/src/NexusMods.EventSourcing.Storage/AttributeRegistry.cs +++ b/src/NexusMods.EventSourcing.Storage/AttributeRegistry.cs @@ -3,7 +3,10 @@ using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; +using System.Runtime.InteropServices; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Internals; +using Reloaded.Memory.Extensions; namespace NexusMods.EventSourcing.Storage; @@ -64,14 +67,6 @@ public void Populate(DbAttribute[] attributes) _attributesByAttributeId[id] = attr; } } - public void WriteValue(TVal val, in TWriter writer) - where TWriter : IBufferWriter - { - if (!_valueSerializersByNativeType.TryGetValue(typeof(TVal), out var serializer)) - throw new InvalidOperationException($"No serializer found for type {typeof(TVal)}"); - - ((IValueSerializer)serializer).Serialize(val, writer); - } public AttributeId GetAttributeId() where TAttr : IAttribute @@ -96,14 +91,17 @@ public AttributeId GetAttributeId(Type datomAttributeType) return dbAttribute.AttrEntityId; } - - public int CompareValues(in Datom a, in Datom b) + public IReadDatom Resolve(ReadOnlySpan datom) { - var attr = _dbAttributesByEntityId[a.A]; - var type = _valueSerializersByUniqueId[attr.ValueTypeId]; - return type.Compare(a.V.Span, b.V.Span); - } + var c = MemoryMarshal.Read(datom); + if (!_attributesByAttributeId.TryGetValue(c.A, out var attribute)) + throw new InvalidOperationException($"No attribute found for attribute ID {c.A}"); + unsafe + { + return attribute.Resolve(c.E, c.A, datom.SliceFast(sizeof(KeyPrefix)), c.T, c.IsRetract); + } + } public Expression GetReadExpression(Type attributeType, Expression valueSpan, out AttributeId attributeId) { @@ -123,17 +121,6 @@ private sealed class CompareCache } private CompareCache _compareCache = new(); - public int CompareValues(T datomsValues, AttributeId attributeId, int a, int b) where T : IBlobColumn - { - var cache = _compareCache; - if (cache.AttributeId == attributeId) - return cache.Serializer.Compare(datomsValues[a].Span, datomsValues[b].Span); - - var attr = _dbAttributesByEntityId[attributeId]; - var type = _valueSerializersByUniqueId[attr.ValueTypeId]; - _compareCache = new CompareCache {AttributeId = attributeId, Serializer = type}; - return type.Compare(datomsValues[a].Span, datomsValues[b].Span); - } public int CompareValues(AttributeId id, ReadOnlySpan a, ReadOnlySpan b) { @@ -152,57 +139,23 @@ public int CompareValues(AttributeId id, ReadOnlySpan a, ReadOnlySpan(ref StackDatom datom, TValueType value, IBufferWriter writer) + public void Explode(out AttributeId id, TValueType value, TBufferWriter writer) where TAttribute : IAttribute + where TBufferWriter : IBufferWriter { var attr = _attributesByType[typeof(TAttribute)]; var dbAttr = _dbAttributesByUniqueId[attr.Id]; var serializer = (IValueSerializer)_valueSerializersByUniqueId[dbAttr.ValueTypeId]; - datom.A = (ushort)dbAttr.AttrEntityId.Value; + id = dbAttr.AttrEntityId; serializer.Serialize(value, writer); } - public IReadDatom Resolve(Datom datom) + public Symbol GetSymbolForAttribute(Type attribute) { - if (!_dbAttributesByEntityId.TryGetValue(datom.A, out var dbAttr)) - throw new InvalidOperationException($"No attribute found for entity ID {datom.A}"); - - if (!_attributesById.TryGetValue(dbAttr.UniqueId, out var attr)) - throw new InvalidOperationException($"No attribute found for unique ID {dbAttr.UniqueId}"); + if (!_attributesByType.TryGetValue(attribute, out var attr)) + throw new InvalidOperationException($"No attribute found for type {attribute}"); - return attr.Resolve(datom); - } - - public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx) - { - if (!_attributesByAttributeId.TryGetValue(attributeId, out var attr)) - throw new InvalidOperationException($"No attribute found for AttributeId {attributeId}"); - - return attr.Resolve(entityId, attributeId, value, tx); - } - - public bool IsReference(AttributeId attributeId) - { - var dbAttr = _dbAttributesByEntityId[attributeId]; - var attrobj = _attributesById[dbAttr.UniqueId]; - return attrobj.IsReference; - } - - public IValueSerializer GetSerializer() - { - if (!_valueSerializersByNativeType.TryGetValue(typeof(TValueType), out var serializer)) - throw new InvalidOperationException($"No serializer found for type {typeof(TValueType)}"); - - return (IValueSerializer)serializer; - } - - public TValue Read(ReadOnlySpan tValueSpan) where TAttr : IAttribute - { - var attr = _attributesByType[typeof(TAttr)]; - var dbAttr = _dbAttributesByUniqueId[attr.Id]; - var serializer = (IValueSerializer)_valueSerializersByUniqueId[dbAttr.ValueTypeId]; - serializer.Read(tValueSpan, out var val); - return val; + return attr.Id; } public Type GetReadDatomType(Type attribute) @@ -211,4 +164,11 @@ public Type GetReadDatomType(Type attribute) return attr.GetReadDatomType(); } + public IAttribute GetAttribute(AttributeId attributeId) + { + if (!_attributesByAttributeId.TryGetValue(attributeId, out var attr)) + throw new InvalidOperationException($"No attribute found for AttributeId {attributeId}"); + + return attr; + } } diff --git a/src/NexusMods.EventSourcing.Storage/AttributeRegistryExtensions.cs b/src/NexusMods.EventSourcing.Storage/AttributeRegistryExtensions.cs deleted file mode 100644 index 76c01271..00000000 --- a/src/NexusMods.EventSourcing.Storage/AttributeRegistryExtensions.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Generic; -using NexusMods.EventSourcing.Abstractions; - -namespace NexusMods.EventSourcing.Storage; - -public static class AttributeRegistryExtensions -{ - /// - /// Converts the datoms to typed datoms using the given registry. - /// - /// - /// - /// - public static IEnumerable Typed(this IEnumerable datoms, AttributeRegistry registry) - { - foreach (var datom in datoms) - { - yield return registry.Resolve(datom); - } - } - -} diff --git a/src/NexusMods.EventSourcing.Storage/DatomStorageStructures/PendingTransaction.cs b/src/NexusMods.EventSourcing.Storage/DatomStorageStructures/PendingTransaction.cs index b1836bb3..b2d1f6ba 100644 --- a/src/NexusMods.EventSourcing.Storage/DatomStorageStructures/PendingTransaction.cs +++ b/src/NexusMods.EventSourcing.Storage/DatomStorageStructures/PendingTransaction.cs @@ -13,20 +13,10 @@ internal class PendingTransaction /// A completion source for the transaction, resolves when the transaction is commited to the /// transaction log and available to readers. /// - public TaskCompletionSource CompletionSource { get; } = new(); - - /// - /// Entity IDs that are remapped in the transaction - /// - public Dictionary Remaps { get; } = new(); + public TaskCompletionSource CompletionSource { get; } = new(); /// /// The data to be commited /// public required IWriteDatom[] Data { get; init; } - - /// - /// The transaction ID that was assigned to the transaction when it was commited - /// - public TxId? AssignedTxId { get; set; } } diff --git a/src/NexusMods.EventSourcing.Storage/DatomStore.cs b/src/NexusMods.EventSourcing.Storage/DatomStore.cs index 14e4612f..55652b90 100644 --- a/src/NexusMods.EventSourcing.Storage/DatomStore.cs +++ b/src/NexusMods.EventSourcing.Storage/DatomStore.cs @@ -1,17 +1,20 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Linq.Expressions; using System.Reactive.Subjects; +using System.Runtime.InteropServices; using System.Threading.Channels; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Abstractions.Internals; +using NexusMods.EventSourcing.Storage.Abstractions; using NexusMods.EventSourcing.Storage.DatomStorageStructures; -using NexusMods.EventSourcing.Storage.Indexes; using Reloaded.Memory.Extensions; -using RocksDbSharp; namespace NexusMods.EventSourcing.Storage; @@ -22,55 +25,31 @@ public class DatomStore : IDatomStore private readonly ILogger _logger; private readonly Channel _txChannel; private EntityId _nextEntityId; - private readonly Subject<(TxId TxId, IReadOnlyCollection Datoms)> _updatesSubject; + private readonly Subject<(TxId TxId, ISnapshot snapshot)> _updatesSubject; private readonly DatomStoreSettings _settings; - private readonly RocksDb _db; - - #region Indexes - private readonly TxLog _txLog; - - private readonly EATVCurrent _eatvCurrent; - private readonly EATVHistory _eatvHistory; - private readonly AETVCurrent _aetvCurrent; - private readonly BackrefHistory _backrefHistory; - - - - - - - - #endregion - private TxId _asOfTxId = TxId.MinValue; private readonly PooledMemoryBufferWriter _writer; - - - public DatomStore(ILogger logger, AttributeRegistry registry, DatomStoreSettings settings) + private readonly IStoreBackend _backend; + private readonly IIndex _eavtHistory; + private readonly IIndex _eavtCurrent; + private readonly IIndex _aevtCurrent; + private readonly IIndex _aevtHistory; + private readonly IIndex _txLog; + private readonly PooledMemoryBufferWriter _prevWriter; + private readonly IIndex _vaetCurrent; + private readonly IIndex _vaetHistory; + private readonly IIndex _avetCurrent; + private readonly IIndex _avetHistory; + + + public DatomStore(ILogger logger, AttributeRegistry registry, DatomStoreSettings settings, IStoreBackend backend) { - var options = new DbOptions() - .SetCreateIfMissing() - .SetCreateMissingColumnFamilies() - .SetCompression(Compression.Zstd); - - var columnFamilies = new ColumnFamilies(); - - _txLog = new TxLog(registry, columnFamilies); - _eatvCurrent = new EATVCurrent(registry, columnFamilies); - _eatvHistory = new EATVHistory(registry, columnFamilies); - _aetvCurrent = new AETVCurrent(registry, columnFamilies); - _backrefHistory = new BackrefHistory(registry, columnFamilies); + _backend = backend; - _db = RocksDb.Open(options, settings.Path.ToString(), columnFamilies); - - _txLog.Init(_db); - _eatvCurrent.Init(_db); - _eatvHistory.Init(_db); - _aetvCurrent.Init(_db); - _backrefHistory.Init(_db); _writer = new PooledMemoryBufferWriter(); + _prevWriter = new PooledMemoryBufferWriter(); _logger = logger; @@ -78,7 +57,30 @@ public DatomStore(ILogger logger, AttributeRegistry registry, DatomS _registry = registry; _nextEntityId = EntityId.From(Ids.MinId(Ids.Partition.Entity) + 1); - _updatesSubject = new Subject<(TxId TxId, IReadOnlyCollection Datoms)>(); + _backend.DeclareEAVT(IndexType.EAVTCurrent); + _backend.DeclareEAVT(IndexType.EAVTHistory); + _backend.DeclareAEVT(IndexType.AEVTCurrent); + _backend.DeclareAEVT(IndexType.AEVTHistory); + _backend.DeclareVAET(IndexType.VAETCurrent); + _backend.DeclareVAET(IndexType.VAETHistory); + _backend.DeclareAVET(IndexType.AVETCurrent); + _backend.DeclareAVET(IndexType.AVETHistory); + _backend.DeclareTxLog(IndexType.TxLog); + + _backend.Init(settings.Path); + + _txLog = _backend.GetIndex(IndexType.TxLog); + _eavtCurrent = _backend.GetIndex(IndexType.EAVTCurrent); + _eavtHistory = _backend.GetIndex(IndexType.EAVTHistory); + _aevtCurrent = _backend.GetIndex(IndexType.AEVTCurrent); + _aevtHistory = _backend.GetIndex(IndexType.AEVTHistory); + _vaetCurrent = _backend.GetIndex(IndexType.VAETCurrent); + _vaetHistory = _backend.GetIndex(IndexType.VAETHistory); + _avetCurrent = _backend.GetIndex(IndexType.AVETCurrent); + _avetHistory = _backend.GetIndex(IndexType.AVETHistory); + + + _updatesSubject = new Subject<(TxId TxId, ISnapshot Snapshot)>(); registry.Populate(BuiltInAttributes.Initial); @@ -98,17 +100,21 @@ private async Task ConsumeTransactions() // Sync transactions have no data, and are used to verify that the store is up to date. if (pendingTransaction.Data.Length == 0) { - pendingTransaction.AssignedTxId = _asOfTxId; - pendingTransaction.CompletionSource.SetResult(_asOfTxId); + var storeResult = new StoreResult() + { + Remaps = new Dictionary(), + AssignedTxId = _asOfTxId, + Snapshot = _backend.GetSnapshot(), + Datoms = Array.Empty() + }; + pendingTransaction.CompletionSource.SetResult(storeResult); continue; } - Log(pendingTransaction, out var readables); - - _updatesSubject.OnNext((_asOfTxId, readables)); - pendingTransaction.CompletionSource.SetResult(_asOfTxId); + Log(pendingTransaction, out var result); - //_logger.LogDebug("Transaction {TxId} processed in {Elapsed}ms, new in-memory size is {Count} datoms", pendingTransaction.AssignedTxId!.Value, sw.ElapsedMilliseconds, _indexes.InMemorySize); + _updatesSubject.OnNext((result.AssignedTxId, result.Snapshot)); + pendingTransaction.CompletionSource.SetResult(result); } catch (Exception ex) { @@ -124,7 +130,14 @@ private async Task Bootstrap() { try { - var lastTx = GetMostRecentTxId(); + var snapshot = _backend.GetSnapshot(); + using var txIterator = snapshot.GetIterator(IndexType.TxLog); + var lastTx = txIterator + .SeekLast() + .Reverse() + .Resolve() + .FirstOrDefault()?.T ?? TxId.MinValue; + if (lastTx == TxId.MinValue) { _logger.LogInformation("Bootstrapping the datom store no existing state found"); @@ -137,7 +150,14 @@ private async Task Bootstrap() _asOfTxId = lastTx; } - _nextEntityId = EntityId.From(GetMaxEntityId().Value + 1); + using var entIterator = snapshot.GetIterator(IndexType.EAVTCurrent); + var lastEnt = entIterator + .SeekLast() + .Reverse() + .Resolve() + .FirstOrDefault()?.E ?? EntityId.MinValue; + + _nextEntityId = EntityId.From(lastEnt.Value + 1); } catch (Exception ex) { @@ -145,74 +165,10 @@ private async Task Bootstrap() } } - /* - private EntityId GetLastEntityId(DatomStoreState indexes) - { - var toFind = new Datom() - { - E = EntityId.From(Ids.MakeId(Ids.Partition.Entity, ulong.MaxValue)), - A = AttributeId.From(ulong.MinValue), - T = TxId.MaxValue, - F = DatomFlags.Added, - V = Array.Empty() - }; - - ulong startValue = 0; - - var startInMemory = indexes.EAVT.InMemory.FindEATV(0, indexes.EAVT.InMemory.Length, toFind, _registry); - if (startInMemory == indexes.EAVT.InMemory.Length) - { - startValue = 0; - } - else - { - startValue = indexes.EAVT.InMemory[startInMemory].E.Value; - } - - var startHistory = indexes.EAVT.History.FindEATV(0, indexes.EAVT.History.Length, toFind, _registry); - if (startHistory == indexes.EAVT.History.Length) - { - startValue = 0; - } - else - { - var historyValue = indexes.EAVT.History[startHistory].E.Value; - if (historyValue > startValue) - { - startValue = historyValue; - } - } - - if (startValue == 0) - return EntityId.From(Ids.MakeId(Ids.Partition.Entity, 0)); - - - var entityInMemory = indexes.EAVT.InMemory[startInMemory].E; - var entityHistory = indexes.EAVT.History[startHistory].E; - - var max = EntityId.From(Math.Max(entityHistory.Value, entityInMemory.Value)); - - if (!Ids.IsPartition(max.Value, Ids.Partition.Entity)) - { - throw new InvalidOperationException("Invalid max id"); - } - - return max; - } - */ - public TxId AsOfTxId => _asOfTxId; + public IAttributeRegistry Registry => _registry; + - public void Dispose() - { - _txChannel.Writer.Complete(); - _db.Dispose(); - _txLog.Dispose(); - _eatvCurrent.Dispose(); - _eatvHistory.Dispose(); - _aetvCurrent.Dispose(); - _backrefHistory.Dispose(); - } public async Task Sync() { @@ -220,35 +176,16 @@ public async Task Sync() return _asOfTxId; } - public async Task Transact(IEnumerable datoms) + public async Task Transact(IEnumerable datoms) { var pending = new PendingTransaction { Data = datoms.ToArray() }; if (!_txChannel.Writer.TryWrite(pending)) throw new InvalidOperationException("Failed to write to the transaction channel"); - await pending.CompletionSource.Task; - - return new DatomStoreTransactResult(pending.AssignedTxId!.Value, pending.Remaps); + return await pending.CompletionSource.Task; } - public IObservable<(TxId TxId, IReadOnlyCollection Datoms)> TxLog => _updatesSubject; - - - public IEnumerable Where(TxId txId) where TAttr : IAttribute - { - throw new NotImplementedException(); - } - - - public IEnumerable Where(TxId txId, EntityId id) - { - throw new NotImplementedException(); - } - - public IEnumerable Resolved(IEnumerable datoms) - { - return datoms.Select(datom => _registry.Resolve(datom)); - } + public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog => _updatesSubject; public async Task RegisterAttributes(IEnumerable newAttrs) { @@ -274,25 +211,27 @@ public Expression GetValueReadExpression(Type attribute, Expression valueSpan, o public IEnumerable GetReferencesToEntityThroughAttribute(EntityId id, TxId txId) where TAttribute : IAttribute { - return _backrefHistory.GetReferencesToEntityThroughAttribute(id, txId); +// return _backrefHistory.GetReferencesToEntityThroughAttribute(id, txId); +throw new NotImplementedException(); } - public bool TryGetExact(EntityId e, TxId tx, out TValue val) where TAttr : IAttribute { - if (_eatvHistory.TryGetExact(e, tx, out var foundVal)) + /*if (_eatvHistory.TryGetExact(e, tx, out var foundVal)) { val = foundVal; return true; } val = default!; - return false; + return false;*/ + throw new NotImplementedException(); } public bool TryGetLatest(EntityId e, TxId tx, out TValue value) where TAttribute : IAttribute { + /* if (_eatvCurrent.TryGet(e, tx, out var foundVal) == LookupResult.Found) { value = foundVal; @@ -307,30 +246,20 @@ public bool TryGetLatest(EntityId e, TxId tx, out TValue val value = default!; return false; + */ + throw new NotImplementedException(); } public IEnumerable GetEntitiesWithAttribute(TxId txId) where TAttribute : IAttribute { - return _aetvCurrent.GetEntitiesWithAttribute(txId); + + throw new NotImplementedException(); } - public IEnumerable GetAttributesForEntity(EntityId realId, TxId txId) + public IEnumerable GetAttributesForEntity(EntityId entityId, TxId txId) { - foreach (var datom in _eatvCurrent.GetAttributesForEntity(realId)) - { - if (datom.T > txId) - { - if (_eatvHistory.TryGetLatest(datom.E, _registry.GetAttributeId(datom.AttributeType), txId, - out var val)) - { - yield return val; - } - } - else { - yield return datom; - } - } + throw new NotImplementedException(); } /// @@ -338,41 +267,49 @@ public IEnumerable GetAttributesForEntity(EntityId realId, TxId txId /// public EntityId GetMaxEntityId() { + /* return _eatvCurrent.GetMaxEntityId(); + */ + throw new NotImplementedException(); } - /// - /// Gets the most recent transaction id. - /// - public TxId GetMostRecentTxId() + public Type GetReadDatomType(Type attribute) { - return _txLog.GetMostRecentTxId(); + return _registry.GetReadDatomType(attribute); } - public Type GetReadDatomType(Type attribute) + public ISnapshot GetSnapshot() { - return _registry.GetReadDatomType(attribute); + return _backend.GetSnapshot(); } + public IEnumerable Datoms(ISnapshot snapshot, IndexType type) + { + using var source = snapshot.GetIterator(type); + var iter = source.SeekStart(); + foreach (var datom in iter.Resolve()) + yield return datom; + } + #region Internals - EntityId MaybeRemap(EntityId id, PendingTransaction pendingTransaction, TxId thisTx) + EntityId MaybeRemap(EntityId id, Dictionary remaps, TxId thisTx) { if (Ids.GetPartition(id) == Ids.Partition.Tmp) { - if (!pendingTransaction.Remaps.TryGetValue(id, out var newId)) + if (!remaps.TryGetValue(id, out var newId)) { if (id.Value == Ids.MinId(Ids.Partition.Tmp)) { var remapTo = EntityId.From(thisTx.Value); - pendingTransaction.Remaps.Add(id, remapTo); + remaps.Add(id, remapTo); return remapTo; } else { - pendingTransaction.Remaps.Add(id, _nextEntityId); + remaps.Add(id, _nextEntityId); var remapTo = _nextEntityId; _nextEntityId = EntityId.From(_nextEntityId.Value + 1); return remapTo; @@ -388,41 +325,77 @@ EntityId MaybeRemap(EntityId id, PendingTransaction pendingTransaction, TxId thi - private void Log(PendingTransaction pendingTransaction, out IReadOnlyCollection resultDatoms) + private void Log(PendingTransaction pendingTransaction, out StoreResult result) { + var output = new List(); var thisTx = TxId.From(_asOfTxId.Value + 1); - var stackDatom = new StackDatom(); - - var remapFn = (Func)(id => MaybeRemap(id, pendingTransaction, thisTx)); - using var batch = new WriteBatch(); + var remaps = new Dictionary(); + var remapFn = (Func)(id => MaybeRemap(id, remaps, thisTx)); + using var batch = _backend.CreateBatch(); var swPrepare = Stopwatch.StartNew(); foreach (var datom in pendingTransaction.Data) { _writer.Reset(); - _writer.Advance(StackDatom.PaddingSize); - datom.Explode(_registry, remapFn, ref stackDatom, _writer); - stackDatom.T = thisTx.Value; - stackDatom.PaddedSpan = _writer.GetWrittenSpanWritable(); - stackDatom.V = stackDatom.PaddedSpan.SliceFast(StackDatom.PaddingSize); + unsafe + { + _writer.Advance(sizeof(KeyPrefix)); + } + + var isRemapped = Ids.IsPartition(datom.E.Value, Ids.Partition.Tmp); + datom.Explode(_registry, remapFn, out var e, out var a, _writer, out var isAssert); + var keyPrefix = _writer.GetWrittenSpanWritable().CastFast(); + keyPrefix[0].Set(e, a, thisTx, isAssert); - _txLog.Add(batch, ref stackDatom); - _eatvHistory.Add(batch, ref stackDatom); - _eatvCurrent.Add(batch, ref stackDatom); - _aetvCurrent.Add(batch, ref stackDatom); + var attr = _registry.GetAttribute(a); + var isReference = attr.IsReference; + var isIndexed = attr.IsIndexed; - if (_registry.IsReference(AttributeId.From(stackDatom.A))) - _backrefHistory.Add(batch, ref stackDatom); + var havePrevious = false; + if (!isRemapped) + havePrevious = GetPrevious(keyPrefix[0]); - output.Add(_registry.Resolve(EntityId.From(stackDatom.E), AttributeId.From(stackDatom.A), stackDatom.V, TxId.From(stackDatom.T))); + if (havePrevious) + { + // Move the previous to the history index + var span = _prevWriter.GetWrittenSpan(); + _eavtCurrent.Delete(batch, span); + _eavtHistory.Put(batch, span); + + _aevtCurrent.Delete(batch, span); + _aevtHistory.Put(batch, span); + + if (isReference) + { + _vaetCurrent.Delete(batch, span); + _vaetHistory.Put(batch, span); + } + + if (isIndexed) + { + _avetCurrent.Delete(batch, span); + _avetHistory.Put(batch, span); + } + } + + var newSpan = _writer.GetWrittenSpan(); + _eavtCurrent.Put(batch, newSpan); + _aevtCurrent.Put(batch, newSpan); + _txLog.Put(batch, newSpan); + + if (isReference) + _vaetCurrent.Put(batch, newSpan); + + if (isIndexed) + _avetCurrent.Put(batch, newSpan); } var swWrite = Stopwatch.StartNew(); - _db.Write(batch); + batch.Commit(); if (_logger.IsEnabled(LogLevel.Debug)) { @@ -434,10 +407,38 @@ private void Log(PendingTransaction pendingTransaction, out IReadOnlyCollection< } + result = new StoreResult + { + AssignedTxId = thisTx, + Remaps = remaps, + Datoms = output, + Snapshot = _backend.GetSnapshot() + }; _asOfTxId = thisTx; - pendingTransaction.AssignedTxId = thisTx; - resultDatoms = output; + } + + private bool GetPrevious(KeyPrefix d) + { + var prefix = new KeyPrefix(); + prefix.Set(d.E, d.A, TxId.MinValue, false); + using var source = _eavtCurrent.GetIterator(); + var iter = source.Seek(MemoryMarshal.CreateSpan(ref prefix, 1).Cast()); + if (!iter.Valid) return false; + + var found = MemoryMarshal.Read(iter.Current); + if (found.E != d.E || found.A != d.A) return false; + + _prevWriter.Reset(); + _prevWriter.Write(iter.Current); + return true; } #endregion + + public void Dispose() + { + _updatesSubject.Dispose(); + _writer.Dispose(); + _prevWriter.Dispose(); + } } diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Backend.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Backend.cs new file mode 100644 index 00000000..531cc4f9 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Backend.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions; +using NexusMods.Paths; +using IWriteBatch = NexusMods.EventSourcing.Storage.Abstractions.IWriteBatch; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class Backend : IStoreBackend +{ + private readonly IIndex[] _indexes; + private IndexStore[] _stores; + + private readonly AttributeRegistry _registry; + + public Backend(AttributeRegistry registry) + { + _registry = registry; + _stores = new IndexStore[Enum.GetValues(typeof(IndexType)).Length]; + _indexes = new IIndex[Enum.GetValues(typeof(IndexType)).Length]; + } + + public IWriteBatch CreateBatch() + { + return new Batch(_stores); + } + + public void Init(AbsolutePath location) + { + } + + public void DeclareIndex(IndexType name) + where TA : IElementComparer + where TB : IElementComparer + where TC : IElementComparer + where TD : IElementComparer + where TF : IElementComparer + { + var store = new IndexStore(name, _registry); + _stores[(int)name] = store; + var index = new Index(_registry, store); + store.Init(index); + _indexes[(int)name] = index; + } + + public IIndex GetIndex(IndexType name) + { + return _indexes[(int)name]; + } + + public ISnapshot GetSnapshot() + { + return new Snapshot(_indexes + .Select(i => ((IInMemoryIndex)i).Set).ToArray(), + _registry); + } + + public void Dispose() + { + } +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Batch.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Batch.cs new file mode 100644 index 00000000..565e4379 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Batch.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions; +using IWriteBatch = NexusMods.EventSourcing.Storage.Abstractions.IWriteBatch; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class Batch(IndexStore[] stores) : IWriteBatch +{ + private Dictionary> _datoms = new(); + + public void Dispose() + { + + } + + public void Commit() + { + foreach (var (index, datoms) in _datoms) + { + var store = stores[(int)index]; + store.Commit(datoms); + } + } + + public void Add(IIndexStore store, ReadOnlySpan key) + { + if (store is not IndexStore indexStore) + throw new ArgumentException("Invalid store type", nameof(store)); + + if (!_datoms.TryGetValue(indexStore.Type, out var datoms)) + { + datoms = new List<(bool IsDelete, byte[] Data)>(); + _datoms.Add(indexStore.Type, datoms); + } + + datoms.Add((false, key.ToArray())); + } + + public void Delete(IIndexStore store, ReadOnlySpan key) + { + if (store is not IndexStore indexStore) + throw new ArgumentException("Invalid store type", nameof(store)); + + if (!_datoms.TryGetValue(indexStore.Type, out var datoms)) + { + datoms = new List<(bool IsDelete, byte[] Data)>(); + _datoms.Add(indexStore.Type, datoms); + } + + datoms.Add((true, key.ToArray())); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IInMemoryIndex.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IInMemoryIndex.cs new file mode 100644 index 00000000..241686fc --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IInMemoryIndex.cs @@ -0,0 +1,8 @@ +using System.Collections.Immutable; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public interface IInMemoryIndex +{ + public ImmutableSortedSet Set { get; } +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Index.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Index.cs new file mode 100644 index 00000000..94c014d3 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Index.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using NexusMods.EventSourcing.Storage.Abstractions; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class Index(AttributeRegistry registry, IndexStore store) : + AIndex(registry, store), IInMemoryIndex, IComparer + where TA : IElementComparer + where TB : IElementComparer + where TC : IElementComparer + where TD : IElementComparer + where TF : IElementComparer +{ + public int Compare(byte[]? x, byte[]? y) + { + return Compare(x.AsSpan(), y.AsSpan()); + } + + public ImmutableSortedSet Set => store.Set; +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IndexStore.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IndexStore.cs new file mode 100644 index 00000000..bd9aef8a --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/IndexStore.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Storage.Abstractions; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class IndexStore : IIndexStore +{ + private readonly AttributeRegistry _registry; + public IndexType Type { get; } + public ImmutableSortedSet Set { get; private set; } + + public IndexStore(IndexType type, AttributeRegistry registry) + { + _registry = registry; + Type = type; + Set = ImmutableSortedSet.Empty; + } + + public void Init(IComparer sorter) + { + Set = ImmutableSortedSet.Empty.WithComparer(sorter); + } + + public IDatomSource GetIterator() + { + return new SortedSetIterator(Set, _registry); + } + + + public void Commit(List<(bool IsDelete, byte[] Data)> datoms) + { + var builder = Set.ToBuilder(); + foreach (var (isRetract, datom) in datoms) + { + if (isRetract) + builder.Remove(datom); + else + builder.Add(datom); + } + Set = builder.ToImmutable(); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Snapshot.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Snapshot.cs new file mode 100644 index 00000000..b39352cf --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/Snapshot.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class Snapshot : ISnapshot +{ + private readonly ImmutableSortedSet[] _indexes; + private readonly AttributeRegistry _registry; + + public Snapshot(ImmutableSortedSet[] indexes, AttributeRegistry registry) + { + _registry = registry; + _indexes = indexes; + } + + public void Dispose() + { + + } + + public IDatomSource GetIterator(IndexType type) + { + return new SortedSetIterator(_indexes[(int)type], _registry); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/InMemoryBackend/SortedSetIterator.cs b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/SortedSetIterator.cs new file mode 100644 index 00000000..281fe7ec --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/InMemoryBackend/SortedSetIterator.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Immutable; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.Storage.InMemoryBackend; + +public class SortedSetIterator : IDatomSource, IIterator +{ + private readonly AttributeRegistry _registry; + private int _offset; + + public SortedSetIterator(ImmutableSortedSet set, AttributeRegistry registry) + { + _registry = registry; + Set = set; + _offset = -1; + } + + + public ImmutableSortedSet Set { get; set; } + + public void Dispose() + { + Set = ImmutableSortedSet.Empty; + _offset = -1; + } + + public bool Valid => _offset >= 0 && _offset < Set.Count; + public void Next() + { + _offset++; + } + + public void Prev() + { + _offset--; + } + + public IIterator SeekLast() + { + _offset = Set.Count - 1; + return this; + } + + IIterator ISeekableIterator.Seek(ReadOnlySpan datom) + { + var result = Set.IndexOf(datom.ToArray()); + if (result < 0) + _offset = ~result; + else + _offset = result; + return this; + } + + public IIterator SeekStart() + { + _offset = 0; + return this; + } + + public ReadOnlySpan Current => Set[_offset]; + public IAttributeRegistry Registry => _registry; +} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/AETVCurrent.cs b/src/NexusMods.EventSourcing.Storage/Indexes/AETVCurrent.cs deleted file mode 100644 index 71eb8386..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/AETVCurrent.cs +++ /dev/null @@ -1,78 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using NexusMods.EventSourcing.Abstractions; -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -internal class AETVCurrent(AttributeRegistry registry, ColumnFamilies columnFamilies) : AIndex(ColumnFamilyName, registry, columnFamilies) -{ - private static string ColumnFamilyName => "AETVCurrent"; - - [StructLayout(LayoutKind.Explicit, Size = sizeof(ushort) + sizeof(ulong))] - private struct Key - { - [FieldOffset(0)] - public ushort Attribute; - - [FieldOffset(sizeof(ushort))] - public ulong Entity; - } - - protected override int Compare(ReadOnlySpan a, ReadOnlySpan b) - { - var keyA = MemoryMarshal.Read(a); - var keyB = MemoryMarshal.Read(b); - - var cmp = keyA.Attribute.CompareTo(keyB.Attribute); - if (cmp != 0) - return cmp; - - return keyA.Entity.CompareTo(keyB.Entity); - } - - public void Add(WriteBatch batch, ref StackDatom stackDatom) - { - Key key = new() - { - Entity = stackDatom.E, - Attribute = stackDatom.A - }; - - var valueSpan = stackDatom.Padded(sizeof(ulong)); - MemoryMarshal.Write(valueSpan, stackDatom.T); - - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - batch.Put(keySpan, valueSpan, ColumnFamily); - } - - public IEnumerable GetEntitiesWithAttribute(TxId asOf) where TAttribute : IAttribute - { - Key key = new() - { - Attribute = (ushort)Registry.GetAttributeId(), - Entity = 0 - }; - - foreach (var value in Db.GetScopedIterator(key, ColumnFamily)) - { - var keyRead = MemoryMarshal.Read(value.KeySpan); - - if (keyRead.Attribute != key.Attribute) - break; - - var tx = MemoryMarshal.Read(value.ValueSpan); - if (tx <= asOf) - { - yield return EntityId.From(keyRead.Entity); - continue; - } - - throw new NotImplementedException(); - - - } - - } -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/AIndex.cs b/src/NexusMods.EventSourcing.Storage/Indexes/AIndex.cs deleted file mode 100644 index 8bbc2f31..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/AIndex.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -public abstract class AIndex -{ - protected RocksDb Db = null!; - protected ColumnFamilyHandle ColumnFamily = null!; - protected readonly AttributeRegistry Registry; - private ColumnFamilyOptions _options = null!; - private IntPtr _namePtr; - private NameDelegate _nameDelegate = null!; - private DestructorDelegate _destructorDelegate = null!; - private CompareDelegate _comparatorDelegate = null!; - private IntPtr _comparator; - private readonly string _columnFamilyName; - - protected AIndex(string columnFamilyName, AttributeRegistry registry, ColumnFamilies columnFamilies) - { - Registry = registry; - _columnFamilyName = columnFamilyName; - - _options = new ColumnFamilyOptions(); - _namePtr = Marshal.StringToHGlobalAnsi(_columnFamilyName); - - _nameDelegate = _ => _namePtr; - _destructorDelegate = _ => { }; - _comparatorDelegate = (_, a, alen, b, blen) => - { - unsafe - { - return Compare(new ReadOnlySpan((void*)a, (int)alen), new ReadOnlySpan((void*)b, (int)blen)); - } - }; - _comparator = Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate, _nameDelegate); - _options.SetComparator(_comparator); - - columnFamilies.Add(_columnFamilyName, _options); - } - - public void Init(RocksDb db) - { - Db = db; - ColumnFamily = Db.GetColumnFamily(_columnFamilyName); - } - - protected abstract int Compare(ReadOnlySpan a, ReadOnlySpan b); - - - public void Dispose() - { - if (_comparator != IntPtr.Zero) - { - Native.Instance.rocksdb_comparator_destroy(_comparator); - _comparator = IntPtr.Zero; - } - - if (_namePtr != IntPtr.Zero) - { - Marshal.FreeHGlobal(_namePtr); - _namePtr = IntPtr.Zero; - } - } - -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/BackrefHistory.cs b/src/NexusMods.EventSourcing.Storage/Indexes/BackrefHistory.cs deleted file mode 100644 index 41c5f7b0..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/BackrefHistory.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using NexusMods.EventSourcing.Abstractions; -using Reloaded.Memory.Extensions; -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -/// -/// A index of all datoms where the value is a reference to an entity. Indexed as [value, attribute, tx] -> entity -/// where the value is always an entity ID. -/// -/// -public class BackrefHistory(AttributeRegistry registry, ColumnFamilies columnFamilies) : AIndex(ColumnFamilyName, registry, columnFamilies) -{ - private static string ColumnFamilyName => "VATEHistory"; - - [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 10)] - private struct Key - { - public ulong ReferenceId; - public ushort AttributeId; - public ulong TxId; - public ulong EntityId; - } - - protected override int Compare(ReadOnlySpan a, ReadOnlySpan b) - { - var keyA = MemoryMarshal.Read(a); - var keyB = MemoryMarshal.Read(b); - - var cmp = keyA.ReferenceId.CompareTo(keyB.ReferenceId); - if (cmp != 0) - return cmp; - - cmp = keyA.AttributeId.CompareTo(keyB.AttributeId); - if (cmp != 0) - return cmp; - - cmp = -keyA.TxId.CompareTo(keyB.TxId); - if (cmp != 0) - return cmp; - - return keyA.EntityId.CompareTo(keyB.EntityId); - } - - - public void Add(WriteBatch batch, ref StackDatom stackDatom) - { - var key = new Key { - ReferenceId = MemoryMarshal.Read(stackDatom.V), - AttributeId = stackDatom.A, - TxId = stackDatom.T, - EntityId = stackDatom.E - }; - - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - - batch.Put(keySpan, ReadOnlySpan.Empty, ColumnFamily); - } - - public IEnumerable GetReferencesToEntityThroughAttribute(EntityId id, TxId tx) - where TAttribute : IAttribute - { - var attributeId = Registry.GetAttributeId(); - var key = new Key - { - ReferenceId = id.Value, - AttributeId = (ushort)attributeId.Value, - TxId = tx.Value, - EntityId = 0, - }; - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - - using var iterator = Db.NewIterator(ColumnFamily); - iterator.Seek(keySpan); - var lastReadId = ulong.MaxValue; - while (iterator.Valid()) - { - var currentKey = MemoryMarshal.Read(iterator.Key()); - if (currentKey.ReferenceId != id.Value) - yield break; - - if (currentKey.TxId > tx.Value) - yield break; - - if (currentKey.AttributeId != attributeId.Value) - continue; - - if (currentKey.EntityId != lastReadId) - yield return EntityId.From(currentKey.EntityId); - lastReadId = currentKey.EntityId; - iterator.Next(); - } - } -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/EATVCurrent.cs b/src/NexusMods.EventSourcing.Storage/Indexes/EATVCurrent.cs deleted file mode 100644 index 2f3d0a6d..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/EATVCurrent.cs +++ /dev/null @@ -1,120 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using NexusMods.EventSourcing.Abstractions; -using Reloaded.Memory.Extensions; -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -/// -/// A index that organizes data by (E, A, T, V) but only ever retains the current value and TX for each (E, A) pair, this -/// leverages the fact that users often only want the current value for a given attribute. Results can be filtered by -/// a given TX, and then the history index can be used to retrieve the full history of a given attribute to fill in the -/// filtered results. -/// -public class EATVCurrent(AttributeRegistry attributeRegistry, ColumnFamilies columnFamilies) : AIndex(ColumnFamilyName, attributeRegistry, columnFamilies) -{ - public static string ColumnFamilyName => "EATVCurrent"; - - [StructLayout(LayoutKind.Explicit, Size = sizeof(ulong) + sizeof(ushort))] - private unsafe struct Key - { - [FieldOffset(0)] public ulong Entity; - [FieldOffset(sizeof(ulong))] public ushort Attribute; - } - public void Add(WriteBatch batch, ref StackDatom stackDatom) - { - Key key = new() - { - Entity = stackDatom.E, - Attribute = stackDatom.A - }; - - var valueSpan = stackDatom.Padded(sizeof(ulong)); - MemoryMarshal.Write(valueSpan, stackDatom.T); - - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - batch.Put(keySpan, valueSpan, ColumnFamily); - } - - public LookupResult TryGet(EntityId e, TxId tx, out TValue val) where TAttr : IAttribute - { - Key key = new() - { - Entity = e.Value, - Attribute = (ushort)Registry.GetAttributeId() - }; - - using var tValue = Db.GetScoped(ref key, ColumnFamily); - if (!tValue.IsValid) - { - val = default!; - return LookupResult.NotFound; - } - - var datomTx = MemoryMarshal.Read(tValue.Span); - if (datomTx <= tx) - { - val = Registry.Read(tValue.Span.SliceFast(sizeof(ulong))); - return LookupResult.Found; - } - - val = default!; - return LookupResult.FoundNewer; - } - - protected override int Compare(ReadOnlySpan a, ReadOnlySpan b) - { - var keyA = MemoryMarshal.Read(a); - var keyB = MemoryMarshal.Read(b); - - var cmp = keyA.Entity.CompareTo(keyB.Entity); - if (cmp != 0) return cmp; - - return keyA.Attribute.CompareTo(keyB.Attribute); - } - - public IEnumerable GetAttributesForEntity(EntityId e) - { - var key = new Key - { - Entity = e.Value, - Attribute = 0 - }; - - foreach (var value in Db.GetScopedIterator(key, ColumnFamily)) - { - var thisKey = value.Key(); - if (thisKey.Entity != e.Value) - break; - - var thisTxId = MemoryMarshal.Read(value.ValueSpan); - - var valueSpan = value.ValueSpan.SliceFast(sizeof(ulong)); - - yield return Registry.Resolve(EntityId.From(thisKey.Entity), - AttributeId.From(thisKey.Attribute), valueSpan, thisTxId); - } - } - - public EntityId GetMaxEntityId() - { - var seekKey = new Key - { - Entity = Ids.MakeId(Ids.Partition.Entity, ulong.MaxValue), - Attribute = ushort.MaxValue - }; - - using var iterator = Db.NewIterator(ColumnFamily); - var span = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref seekKey, 1)); - iterator.SeekForPrev(span); - - if (!iterator.Valid()) - return EntityId.From(Ids.MakeId(Ids.Partition.Entity, 0)); - - var key = MemoryMarshal.Read(iterator.Key()); - - return EntityId.From(key.Entity); - } -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/EATVHistory.cs b/src/NexusMods.EventSourcing.Storage/Indexes/EATVHistory.cs deleted file mode 100644 index d06a2b7a..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/EATVHistory.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using NexusMods.EventSourcing.Abstractions; -using Reloaded.Memory.Extensions; -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -public class EATVHistory(AttributeRegistry registry, ColumnFamilies columnFamilies) : AIndex(ColumnFamilyName, registry, columnFamilies) -{ - public static string ColumnFamilyName => "EATVHistory"; - - [StructLayout(LayoutKind.Explicit, Size = sizeof(ulong) + sizeof(ulong) + sizeof(ushort))] - private unsafe struct Key - { - [FieldOffset(0)] - public ulong Entity; - [FieldOffset(sizeof(ulong))] - public ulong Tx; - [FieldOffset(sizeof(ulong) + sizeof(ulong))] - public ushort Attribute; - } - - - public void Add(WriteBatch batch, ref StackDatom datom) - { - Key key = new() - { - Entity = datom.E, - Tx = datom.T, - Attribute = datom.A - }; - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - batch.Put(keySpan, datom.V, ColumnFamily); - } - - - public bool TryGetExact(EntityId entityId, TxId tx, out TValue val) - where TAttr : IAttribute - { - Key key = new() - { - Entity = entityId.Value, - Tx = tx.Value, - Attribute = (ushort)Registry.GetAttributeId() - }; - using var tValue = Db.GetScoped(ref key, ColumnFamily); - if (!tValue.IsValid) - { - val = default!; - return false; - } - val = Registry.Read(tValue.Span); - return true; - } - - public bool TryGetLatest(EntityId entityId, TxId tx, out TValue foundVal) - where TAttribute : IAttribute - { - Key key = new() - { - Entity = entityId.Value, - Tx = tx.Value, - Attribute = (ushort)Registry.GetAttributeId() - }; - - foreach (var value in Db.GetScopedIterator(key, ColumnFamily)) - { - var currKey = value.Key(); - if (currKey.Tx > tx.Value || currKey.Entity != entityId.Value || currKey.Attribute != key.Attribute) - break; - foundVal = Registry.Read(value.ValueSpan); - return true; - } - - foundVal = default!; - return false; - } - - /// - /// Gets the latest value of the given attribute for the given entity id where the transaction id is less than or equal to the given txId. - /// - public bool TryGetLatest(EntityId entityId, AttributeId attributeId, TxId tx, out IReadDatom foundVal) - { - Key key = new() - { - Entity = entityId.Value, - Tx = tx.Value, - Attribute = (ushort)attributeId - }; - - foreach (var value in Db.GetScopedIterator(key, ColumnFamily)) - { - var currKey = value.Key(); - if (currKey.Tx > tx.Value || currKey.Entity != entityId.Value || currKey.Attribute != key.Attribute) - break; - foundVal = Registry.Resolve(EntityId.From(currKey.Entity), AttributeId.From(currKey.Attribute), value.ValueSpan, TxId.From(currKey.Tx)); - return true; - } - - foundVal = default!; - return false; - } - - protected override int Compare(ReadOnlySpan a, ReadOnlySpan b) - { - var casted = MemoryMarshal.Read(a); - var other = MemoryMarshal.Read(b); - - var cmp = casted.Entity.CompareTo(other.Entity); - if (cmp != 0) - return cmp; - - cmp = casted.Attribute.CompareTo(other.Attribute); - if (cmp != 0) - return cmp; - - // Reverse order so that we can get the latest value by iterating forward which may be slightly - // faster, but also simpler to implement. - return -casted.Tx.CompareTo(other.Tx); - } -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/LookupResult.cs b/src/NexusMods.EventSourcing.Storage/Indexes/LookupResult.cs deleted file mode 100644 index 72eb4aac..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/LookupResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace NexusMods.EventSourcing.Storage.Indexes; - -public enum LookupResult : int -{ - NotFound = 0, - Found = 1, - FoundNewer = 2 -} diff --git a/src/NexusMods.EventSourcing.Storage/Indexes/TxLog.cs b/src/NexusMods.EventSourcing.Storage/Indexes/TxLog.cs deleted file mode 100644 index fddd5767..00000000 --- a/src/NexusMods.EventSourcing.Storage/Indexes/TxLog.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; -using System.Runtime.InteropServices; -using NexusMods.EventSourcing.Abstractions; -using RocksDbSharp; - -namespace NexusMods.EventSourcing.Storage.Indexes; - -public class TxLog(AttributeRegistry registry, ColumnFamilies columnFamilies) : AIndex(ColumnFamilyName, registry, columnFamilies) -{ - private static string ColumnFamilyName => "TxLog"; - - [StructLayout(LayoutKind.Sequential, Pack = 1)] - private readonly struct Key - { - public readonly ulong Tx; - public readonly ulong Entity; - public readonly ushort Attribute; - - public Key(ulong tx, ulong entity, ushort attribute) - { - Tx = tx; - Entity = entity; - Attribute = attribute; - } - } - - protected override int Compare(ReadOnlySpan a, ReadOnlySpan b) - { - var keyA = MemoryMarshal.Read(a); - var keyB = MemoryMarshal.Read(b); - - var cmp = keyA.Tx.CompareTo(keyB.Tx); - if (cmp != 0) return cmp; - - cmp = keyA.Entity.CompareTo(keyB.Entity); - if (cmp != 0) return cmp; - - return keyA.Attribute.CompareTo(keyB.Attribute); - } - - public void Add(WriteBatch batch, ref StackDatom datom) - { - var key = new Key(datom.T, datom.E, datom.A); - - var keySpan = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref key, 1)); - batch.Put(keySpan, datom.V, ColumnFamily); - } - - public TxId GetMostRecentTxId() - { - using var iterator = Db.NewIterator(ColumnFamily); - iterator.SeekToLast(); - if (!iterator.Valid()) - return TxId.MinValue; - - var key = MemoryMarshal.Read(iterator.Key()); - return TxId.From(key.Tx); - - } -} diff --git a/src/NexusMods.EventSourcing.Storage/NexusMods.EventSourcing.Storage.csproj b/src/NexusMods.EventSourcing.Storage/NexusMods.EventSourcing.Storage.csproj index aed80d13..817594de 100644 --- a/src/NexusMods.EventSourcing.Storage/NexusMods.EventSourcing.Storage.csproj +++ b/src/NexusMods.EventSourcing.Storage/NexusMods.EventSourcing.Storage.csproj @@ -8,14 +8,15 @@ - + - - + + + diff --git a/src/NexusMods.EventSourcing.Storage/RocksDBExtensions.cs b/src/NexusMods.EventSourcing.Storage/RocksDBExtensions.cs index c66fbd7b..40dd670a 100644 --- a/src/NexusMods.EventSourcing.Storage/RocksDBExtensions.cs +++ b/src/NexusMods.EventSourcing.Storage/RocksDBExtensions.cs @@ -12,7 +12,7 @@ public static class RocksDBExtensions { private static readonly ReadOptions DefaultReadOptions = new(); - public ref struct ValueRef + public struct ValueRef : IDisposable { private IntPtr _ptr; private int _length; diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Backend.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Backend.cs new file mode 100644 index 00000000..6a5bd433 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Backend.cs @@ -0,0 +1,93 @@ +using System.Collections.Generic; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Storage.Abstractions; +using NexusMods.Paths; +using RocksDbSharp; +using IWriteBatch = NexusMods.EventSourcing.Storage.Abstractions.IWriteBatch; + +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +public class Backend(AttributeRegistry registry) : IStoreBackend +{ + private string _location = string.Empty; + private readonly Dictionary _indexes = new(); + private readonly Dictionary _stores = new(); + private RocksDb _db = null!; + private readonly ColumnFamilies _columnFamilies = new(); + + public IWriteBatch CreateBatch() + { + return new Batch(_db); + } + + public void DeclareIndex(IndexType name) + where TA : IElementComparer + where TB : IElementComparer + where TC : IElementComparer + where TD : IElementComparer + where TF : IElementComparer + { + var indexStore = new IndexStore(name.ToString(), name, registry); + _stores.Add(name, indexStore); + + var index = new Index(registry, indexStore); + _indexes.Add(name, index); + } + + public IIndex GetIndex(IndexType name) + { + return (IIndex)_indexes[name]; + } + + private class Snapshot(Backend backend, AttributeRegistry registry) : ISnapshot + { + private readonly RocksDbSharp.Snapshot _snapshot = backend._db.CreateSnapshot(); + + public void Dispose() + { + _snapshot.Dispose(); + } + + public IDatomSource GetIterator(IndexType type) + { + var options = new ReadOptions() + .SetSnapshot(_snapshot); + + var iterator = backend._db.NewIterator(backend._stores[type].Handle, options); + + return new IteratorWrapper(iterator, registry); + } + } + + public ISnapshot GetSnapshot() + { + return new Snapshot(this, registry); + } + + public void Init(AbsolutePath location) + { + var options = new DbOptions() + .SetCreateIfMissing() + .SetCreateMissingColumnFamilies() + .SetCompression(Compression.Zstd); + + foreach (var (name, store) in _stores) + { + var index = _indexes[name]; + store.SetupColumnFamily((IIndex)index, _columnFamilies); + } + + _db = RocksDb.Open(options, location.ToString(), _columnFamilies); + + foreach (var (name, store) in _stores) + { + store.PostOpenSetup(_db); + } + } + + public void Dispose() + { + _db.Dispose(); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Batch.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Batch.cs new file mode 100644 index 00000000..d84d58ca --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Batch.cs @@ -0,0 +1,30 @@ +using System; +using NexusMods.EventSourcing.Storage.Abstractions; +using RocksDbSharp; +using IWriteBatch = NexusMods.EventSourcing.Storage.Abstractions.IWriteBatch; + +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; +public class Batch(RocksDb db) : IWriteBatch +{ + private readonly WriteBatch _batch = new(); + + public void Dispose() + { + _batch.Dispose(); + } + + public void Add(IIndexStore store, ReadOnlySpan key) + { + _batch.Put(key, ReadOnlySpan.Empty, ((IndexStore)store).Handle); + } + + public void Delete(IIndexStore store, ReadOnlySpan key) + { + _batch.Delete(key, ((IndexStore)store).Handle); + } + + public void Commit() + { + db.Write(_batch); + } +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IRocksDbIndex.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IRocksDbIndex.cs new file mode 100644 index 00000000..d35d77b5 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IRocksDbIndex.cs @@ -0,0 +1,6 @@ +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +public interface IRocksDbIndex +{ + +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Index.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Index.cs new file mode 100644 index 00000000..74e0079d --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Index.cs @@ -0,0 +1,14 @@ +using NexusMods.EventSourcing.Storage.Abstractions; + +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +public class Index(AttributeRegistry registry, IndexStore store) : + AIndex(registry, store), IRocksDbIndex + where TA : IElementComparer + where TB : IElementComparer + where TC : IElementComparer + where TD : IElementComparer + where TF : IElementComparer +{ + +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IndexStore.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IndexStore.cs new file mode 100644 index 00000000..bc819438 --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IndexStore.cs @@ -0,0 +1,68 @@ +using System; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Storage.Abstractions; +using Reloaded.Memory.Extensions; +using RocksDbSharp; + +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +public class IndexStore : IIndexStore +{ + private readonly string _handleName; + private ColumnFamilyOptions _options = null!; + private IntPtr _namePtr; + private NameDelegate _nameDelegate = null!; + private DestructorDelegate _destructorDelegate = null!; + private CompareDelegate _comparatorDelegate = null!; + private IntPtr _comparator; + private ColumnFamilyHandle _columnHandle = null!; + private RocksDb _db = null!; + private readonly AttributeRegistry _registry; + + public IndexStore(string handleName, IndexType type, AttributeRegistry registry) + { + Type = type; + _registry = registry; + _handleName = handleName; + } + + public IndexType Type { get; } + + + public void SetupColumnFamily(IIndex index, ColumnFamilies columnFamilies) + { + _options = new ColumnFamilyOptions(); + _namePtr = Marshal.StringToHGlobalAnsi(_handleName); + + _nameDelegate = _ => _namePtr; + _destructorDelegate = _ => { }; + _comparatorDelegate = (_, a, alen, b, blen) => + { + unsafe + { + return index.Compare(new ReadOnlySpan((void*)a, (int)alen), new ReadOnlySpan((void*)b, (int)blen)); + } + }; + _comparator = Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate, _nameDelegate); + _options.SetComparator(_comparator); + + columnFamilies.Add(_handleName, _options); + } + + public ColumnFamilyHandle Handle => _columnHandle; + + public void PostOpenSetup(RocksDb db) + { + _db = db; + _columnHandle = db.GetColumnFamily(_handleName); + } + + + public IDatomSource GetIterator() + { + return new IteratorWrapper(_db.NewIterator(_columnHandle), _registry); + + } +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IteratorWrapper.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IteratorWrapper.cs new file mode 100644 index 00000000..c9f0d01c --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/IteratorWrapper.cs @@ -0,0 +1,57 @@ +using System; +using NexusMods.EventSourcing.Abstractions.DatomIterators; +using NexusMods.EventSourcing.Abstractions.Internals; +using RocksDbSharp; + +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +internal class IteratorWrapper : IDatomSource, IIterator +{ + private Iterator _iterator; + private readonly AttributeRegistry _registry; + + public IteratorWrapper(Iterator iterator, AttributeRegistry registry) + { + _registry = registry; + _iterator = iterator; + } + + public void Dispose() + { + _iterator.Dispose(); + _iterator = null!; + } + + public bool Valid => _iterator.Valid(); + public void Next() + { + _iterator.Next(); + } + + public void Prev() + { + _iterator.Prev(); + } + + + public ReadOnlySpan Current => _iterator.GetKeySpan(); + public IAttributeRegistry Registry => _registry; + + public IIterator SeekLast() + { + _iterator.SeekToLast(); + return this; + } + + public IIterator Seek(ReadOnlySpan datom) + { + _iterator.Seek(datom); + return this; + } + + public IIterator SeekStart() + { + _iterator.SeekToFirst(); + return this; + } +} diff --git a/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Snapshot.cs b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Snapshot.cs new file mode 100644 index 00000000..0ffd4d6f --- /dev/null +++ b/src/NexusMods.EventSourcing.Storage/RocksDbBackend/Snapshot.cs @@ -0,0 +1,6 @@ +namespace NexusMods.EventSourcing.Storage.RocksDbBackend; + +public class Snapshot(Backend backend) +{ + +} diff --git a/src/NexusMods.EventSourcing.Storage/Services.cs b/src/NexusMods.EventSourcing.Storage/Services.cs index eb441f59..6e465093 100644 --- a/src/NexusMods.EventSourcing.Storage/Services.cs +++ b/src/NexusMods.EventSourcing.Storage/Services.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions; using NexusMods.EventSourcing.Storage.Serializers; namespace NexusMods.EventSourcing.Storage; @@ -15,9 +16,20 @@ public static IServiceCollection AddEventSourcingStorage(this IServiceCollection services.AddValueSerializer(); services.AddAttribute(); services.AddAttribute(); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); return services; } + + public static IServiceCollection AddDatomStoreSettings(this IServiceCollection services, DatomStoreSettings settings) + { + services.AddSingleton(settings); + return services; + } + + public static IServiceCollection AddRocksDbBackend(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } } diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index dda1bb75..1c801401 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reactive.Linq; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; @@ -45,7 +46,9 @@ public static async Task Start(IServiceProvider provider) { var db = provider.GetRequiredService(); await db.Sync(); - return await Start(provider.GetRequiredService(), provider.GetRequiredService>(), provider.GetRequiredService>()); + return await Start(provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService>()); } @@ -76,14 +79,16 @@ private async Task AddMissingAttributes(IEnumerable valueSeria private IEnumerable ExistingAttributes() { - var attrIds = _store.GetEntitiesWithAttribute(TxId.MaxValue); + var db = Db; + var attrIds = db.Datoms(IndexType.AEVTCurrent) + .Select(d => d.E); - foreach (var attr in attrIds) + foreach (var attrId in attrIds) { var serializerId = Symbol.Unknown; var uniqueId = Symbol.Unknown; - foreach (var datom in _store.GetAttributesForEntity(attr, TxId.MaxValue)) + foreach (var datom in db.Datoms(attrId)) { switch (datom) { @@ -95,7 +100,7 @@ private IEnumerable ExistingAttributes() break; } } - yield return new DbAttribute(uniqueId, AttributeId.From(attr.Value), serializerId); + yield return new DbAttribute(uniqueId, AttributeId.From(attrId.Value), serializerId); } } @@ -103,7 +108,7 @@ private IEnumerable ExistingAttributes() /// - public IDb Db => new Db(_store, this, TxId); + public IDb Db => new Db(_store.GetSnapshot(), this, TxId, (AttributeRegistry)_store.Registry); /// @@ -114,7 +119,7 @@ private IEnumerable ExistingAttributes() public async Task Transact(IEnumerable datoms) { var newTx = await _store.Transact(datoms); - var result = new CommitResult(newTx.TxId, newTx.Remaps); + var result = new CommitResult(newTx.AssignedTxId, newTx.Remaps); return result; } @@ -125,14 +130,6 @@ public ITransaction BeginTransaction() } /// - public IObservable<(TxId TxId, IReadOnlyCollection Datoms)> Commits => _store.TxLog; - - /// - public T GetActive(EntityId id) where T : IActiveReadModel - { - var db = Db; - var ctor = ModelReflector.GetActiveModelConstructor(); - var activeModel = (T)ctor(db, id); - return activeModel; - } + public IObservable Revisions => _store.TxLog + .Select(log => new Db(log.Snapshot, this, log.TxId, (AttributeRegistry)_store.Registry)); } diff --git a/src/NexusMods.EventSourcing/Db.cs b/src/NexusMods.EventSourcing/Db.cs index 65458090..7419f281 100644 --- a/src/NexusMods.EventSourcing/Db.cs +++ b/src/NexusMods.EventSourcing/Db.cs @@ -1,47 +1,119 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.DatomIterators; using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.Storage; namespace NexusMods.EventSourcing; -internal class Db(IDatomStore store, Connection connection, TxId txId) : IDb +internal class Db : IDb { - public TxId BasisTxId => txId; - public IConnection Connection => connection; + private readonly Connection _connection; + private readonly TxId _txId; + private readonly ISnapshot _snapshot; + private readonly AttributeRegistry _registry; + + public Db(ISnapshot snapshot, Connection connection, TxId txId, AttributeRegistry registry) + { + _registry = registry; + _connection = connection; + _snapshot = snapshot; + _txId = txId; + } + + public TxId BasisTxId => _txId; + public IConnection Connection => _connection; public IEnumerable Get(IEnumerable ids) where TModel : IReadModel { - var reader = connection.ModelReflector.GetReader(); - foreach (var id in ids) + var reader = _connection.ModelReflector.GetReader(); + using var readerSource = _snapshot.GetIterator(IndexType.EAVTCurrent); + + foreach (var e in ids) { - var iterator = store.GetAttributesForEntity(id, txId).GetEnumerator(); - yield return reader(id, iterator, this); + // Inlining this code so that we can re-use the iterator means roughly a 25% speedup + // in loading a large number of entities. + using var enumerator = readerSource + .SeekTo(e) + .While(e) + .Resolve() + .GetEnumerator(); + yield return reader(e, enumerator, this); } } public TModel Get(EntityId id) where TModel : IReadModel { - var reader = connection.ModelReflector.GetReader(); - return reader(id, store.GetAttributesForEntity(id, txId).GetEnumerator(), this); + var reader = _connection.ModelReflector.GetReader(); + + using var source = _snapshot.GetIterator(IndexType.EAVTCurrent); + using var enumerator = source.SeekTo(id) + .While(id) + .Resolve() + .GetEnumerator(); + return reader(id, enumerator, this); } /// public IEnumerable GetReverse(EntityId id) where TAttribute : IAttribute where TModel : IReadModel { - var reader = connection.ModelReflector.GetReader(); - foreach (var entity in store.GetReferencesToEntityThroughAttribute(id, txId)) + using var attrSource = _snapshot.GetIterator(IndexType.VAETCurrent); + var attrId = _registry.GetAttributeId(); + var eIds = attrSource + .SeekTo(attrId, id) + .WhileUnmanagedV(id) + .While(attrId) + .Select(c => c.CurrentKeyPrefix().E); + + var reader = _connection.ModelReflector.GetReader(); + using var readerSource = _snapshot.GetIterator(IndexType.EAVTCurrent); + + foreach (var e in eIds) { - using var iterator = store.GetAttributesForEntity(entity, txId).GetEnumerator(); - yield return reader(entity, iterator, this); + // Inlining this code so that we can re-use the iterator means roughly a 25% speedup + // in loading a large number of entities. + using var enumerator = readerSource + .SeekTo(e) + .While(e) + .Resolve() + .GetEnumerator(); + yield return reader(e, enumerator, this); } + } + public IEnumerable Datoms(EntityId entityId) + { + using var iterator = _snapshot.GetIterator(IndexType.EAVTCurrent); + foreach (var datom in iterator.SeekTo(entityId) + .While(entityId) + .Resolve()) + yield return datom; + + } + + public IEnumerable Datoms(TxId txId) + { + using var iterator = _snapshot.GetIterator(IndexType.TxLog); + foreach (var datom in iterator + .SeekTo(txId) + .While(txId) + .Resolve()) + yield return datom; + } + + public IEnumerable Datoms(IndexType type) + where TAttribute : IAttribute + { + var a = _registry.GetAttributeId(); + using var iterator = _snapshot.GetIterator(type); + foreach (var datom in iterator + .SeekStart() + .While(a) + .Resolve()) + yield return datom; } - public void Reload(TOuter aActiveReadModel) where TOuter : IActiveReadModel + public void Dispose() { - var reader = connection.ModelReflector.GetActiveReader(); - var iterator = store.GetAttributesForEntity(aActiveReadModel.Id, txId).GetEnumerator(); - reader(aActiveReadModel, iterator); } } diff --git a/src/NexusMods.EventSourcing/ModelReflector.cs b/src/NexusMods.EventSourcing/ModelReflector.cs index 2bd2f4e1..83e39c5b 100644 --- a/src/NexusMods.EventSourcing/ModelReflector.cs +++ b/src/NexusMods.EventSourcing/ModelReflector.cs @@ -27,8 +27,6 @@ private delegate void EmitterFn(TTransaction tx, TReadModel model internal delegate TReadModel ReaderFn(EntityId id, IEnumerator iterator, IDb db) where TReadModel : IReadModel; - internal delegate void ActiveReaderFn(TModel model, IEnumerator datom); - public void Add(TTransaction tx, IReadModel model) { EmitterFn emitterFn; @@ -114,21 +112,6 @@ public ReaderFn GetReader() where TModel : IReadModel return readerFn; } - public ActiveReaderFn GetActiveReader() where TModel : IActiveReadModel - { - var modelType = typeof(TModel); - if (_activeReaders.TryGetValue(modelType, out var found)) - return (ActiveReaderFn)found; - - var readerFn = MakeActiveReader(); - _activeReaders.TryAdd(modelType, readerFn); - return readerFn; - } - - public Func GetActiveModelConstructor() where TModel : IActiveReadModel - { - return GetConstructor(typeof(TModel)); - } private ReaderFn MakeReader() where TModel : IReadModel { @@ -181,44 +164,4 @@ private ReaderFn MakeReader() where TModel : IReadModel var lambda = Expression.Lambda>(block, entityIdParameter, iteratorParameter, dbParameter); return lambda.Compile(); } - - private ActiveReaderFn MakeActiveReader() - { - var tmodel = typeof(TModel); - var properties = GetModelProperties(tmodel); - - var exprs = new List(); - - var whileTopLabel = Expression.Label("whileTop"); - var exitLabel = Expression.Label("exit"); - - var modelParameter = Expression.Parameter(tmodel, "model"); - var iteratorParameter = Expression.Parameter(typeof(IEnumerator), "iterator"); - - exprs.Add(Expression.Label(whileTopLabel)); - exprs.Add(Expression.IfThen( - Expression.Not(Expression.Call(iteratorParameter, typeof(IEnumerator).GetMethod("MoveNext")!)), - Expression.Break(exitLabel))); - - foreach (var (attribute, property) in properties) - { - var readDatomType = store.GetReadDatomType(attribute); - - var ifExpr = Expression.IfThen( - Expression.TypeIs(Expression.Property(iteratorParameter, "Current"), readDatomType), - Expression.Assign(Expression.Property(modelParameter, property), Expression.Property(Expression.Convert(Expression.Property(iteratorParameter, "Current"), readDatomType), "V"))); - - exprs.Add(ifExpr); - - } - - exprs.Add(Expression.Goto(whileTopLabel)); - exprs.Add(Expression.Label(exitLabel)); - exprs.Add(modelParameter); - - var block = Expression.Block(exprs); - - var lambda = Expression.Lambda>(block, modelParameter, iteratorParameter); - return lambda.Compile(); - } } diff --git a/src/NexusMods.EventSourcing/NexusMods.EventSourcing.csproj b/src/NexusMods.EventSourcing/NexusMods.EventSourcing.csproj index bee18345..8e7572ca 100644 --- a/src/NexusMods.EventSourcing/NexusMods.EventSourcing.csproj +++ b/src/NexusMods.EventSourcing/NexusMods.EventSourcing.csproj @@ -3,13 +3,14 @@ NexusMods.EventSourcing - - - - - + + + + + + diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/ABackendTest.cs b/tests/NexusMods.EventSourcing.Storage.Tests/ABackendTest.cs new file mode 100644 index 00000000..e6d926b1 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/ABackendTest.cs @@ -0,0 +1,63 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions; +using NexusMods.EventSourcing.TestModel.Helpers; +using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; +using NexusMods.Hashing.xxHash64; +using FileAttributes = NexusMods.EventSourcing.TestModel.ComplexModel.Attributes.FileAttributes; + + +namespace NexusMods.EventSourcing.Storage.Tests; + +public abstract class ABackendTest(IServiceProvider provider, Func backendFn) + : AStorageTest(provider, backendFn) + where TStoreType : IStoreBackend +{ + + [Theory] + [InlineData(IndexType.TxLog)] + [InlineData(IndexType.EAVTHistory)] + [InlineData(IndexType.EAVTCurrent)] + [InlineData(IndexType.AEVTCurrent)] + [InlineData(IndexType.AEVTHistory)] + [InlineData(IndexType.VAETCurrent)] + [InlineData(IndexType.VAETHistory)] + [InlineData(IndexType.AVETCurrent)] + [InlineData(IndexType.AVETHistory)] + public async Task InsertedDatomsShowUpInTheIndex(IndexType type) + { + var id1 = NextTempId(); + var id2 = NextTempId(); + + var modId1 = NextTempId(); + var modId2 = NextTempId(); + + var tx = await DatomStore.Transact([ + FileAttributes.Path.Assert(id1, "/foo/bar"), + FileAttributes.Hash.Assert(id1, Hash.From(0xDEADBEEF)), + FileAttributes.Size.Assert(id1, Paths.Size.From(42)), + FileAttributes.Path.Assert(id2, "/qix/bar"), + FileAttributes.Hash.Assert(id2, Hash.From(0xDEADBEAF)), + FileAttributes.Size.Assert(id2, Paths.Size.From(77)), + FileAttributes.ModId.Assert(id1, modId1), + FileAttributes.ModId.Assert(id2, modId1), + ModAttributes.Name.Assert(modId1, "Test Mod 1"), + ModAttributes.Name.Assert(modId2, "Test Mod 2") + ]); + + id1 = tx.Remaps[id1]; + id2 = tx.Remaps[id2]; + + tx = await DatomStore.Transact([ + // Rename file 1 and move file 1 to mod 2 + FileAttributes.Path.Assert(id2, "/foo/qux"), + FileAttributes.ModId.Assert(id1, modId2) + ]); + + var snapshot = DatomStore.GetSnapshot(); + var results = DatomStore.Datoms(snapshot, type).ToList(); + + await Verify(results.ToTable(Registry)) + .UseDirectory("BackendTestVerifyData") + .UseParameters(type); + } +} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs b/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs index 2111dc9d..9b1dbf9c 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs @@ -1,15 +1,19 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.Abstractions; +using NexusMods.EventSourcing.Storage.RocksDbBackend; using NexusMods.EventSourcing.Storage.Serializers; using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; +using NexusMods.EventSourcing.TestModel.ValueSerializers; using NexusMods.Paths; +using FileAttributes = NexusMods.EventSourcing.TestModel.ComplexModel.Attributes.FileAttributes; namespace NexusMods.EventSourcing.Storage.Tests; public abstract class AStorageTest : IAsyncLifetime { - protected readonly AttributeRegistry _registry; + protected readonly AttributeRegistry Registry; protected IDatomStore DatomStore; protected readonly DatomStoreSettings DatomStoreSettings; @@ -17,13 +21,21 @@ public abstract class AStorageTest : IAsyncLifetime private readonly IServiceProvider _provider; private readonly AbsolutePath _path; - protected AStorageTest(IServiceProvider provider) + private ulong _tempId = 1; + + protected AStorageTest(IServiceProvider provider, Func? backendFn = null) { _provider = provider; - _registry = new AttributeRegistry(provider.GetRequiredService>(), + Registry = new AttributeRegistry(provider.GetRequiredService>(), provider.GetRequiredService>()); - _registry.Populate([ - new DbAttribute(Symbol.Intern(), AttributeId.From(10), Symbol.Intern()) + Registry.Populate([ + new DbAttribute(Symbol.Intern(), AttributeId.From(10), Symbol.Intern()), + new DbAttribute(Symbol.Intern(), AttributeId.From(20), Symbol.Intern()), + new DbAttribute(Symbol.Intern(), AttributeId.From(21), Symbol.Intern()), + new DbAttribute(Symbol.Intern(), AttributeId.From(22), Symbol.Intern()), + new DbAttribute(Symbol.Intern(), AttributeId.From(23), Symbol.Intern()), + new DbAttribute(Symbol.Intern(), AttributeId.From(24), Symbol.Intern()), + ]); _path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Combine("tests_datomstore"+Guid.NewGuid()); @@ -32,11 +44,21 @@ protected AStorageTest(IServiceProvider provider) Path = _path, }; + backendFn ??= (registry) => new Backend(registry); + + + + DatomStore = new DatomStore(provider.GetRequiredService>(), Registry, DatomStoreSettings, + backendFn(Registry)); - DatomStore = new DatomStore(provider.GetRequiredService>(), _registry, DatomStoreSettings); Logger = provider.GetRequiredService>(); } + public EntityId NextTempId() + { + var id = Interlocked.Increment(ref _tempId); + return EntityId.From(Ids.MakeId(Ids.Partition.Tmp, id)); + } public async Task InitializeAsync() { diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt new file mode 100644 index 00000000..c7a07ac4 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt @@ -0,0 +1,14 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt new file mode 100644 index 00000000..b26c4860 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt new file mode 100644 index 00000000..4fbd53d8 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt @@ -0,0 +1,4 @@ ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt new file mode 100644 index 00000000..e65f7081 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt @@ -0,0 +1 @@ ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt new file mode 100644 index 00000000..7e6104f8 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt @@ -0,0 +1,14 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt new file mode 100644 index 00000000..1d8ab626 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt new file mode 100644 index 00000000..f698a279 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt @@ -0,0 +1,16 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt new file mode 100644 index 00000000..e4e846a6 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt new file mode 100644 index 00000000..251d8d2b --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/InMemoryTests.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt @@ -0,0 +1 @@ ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt new file mode 100644 index 00000000..c7a07ac4 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTCurrent.verified.txt @@ -0,0 +1,14 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt new file mode 100644 index 00000000..b26c4860 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AEVTHistory.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt new file mode 100644 index 00000000..4fbd53d8 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETCurrent.verified.txt @@ -0,0 +1,4 @@ ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt new file mode 100644 index 00000000..e65f7081 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=AVETHistory.verified.txt @@ -0,0 +1 @@ ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt new file mode 100644 index 00000000..7e6104f8 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTCurrent.verified.txt @@ -0,0 +1,14 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt new file mode 100644 index 00000000..1d8ab626 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=EAVTHistory.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt new file mode 100644 index 00000000..f698a279 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=TxLog.verified.txt @@ -0,0 +1,16 @@ ++ | 0000000000000001 | (0001) UniqueId | NexusMods.EventSourcing.DatomStore/UniqueId | 0100000000000001 ++ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0000000000000002 | (0001) UniqueId | NexusMods.EventSourcin...tore/ValueSerializerId | 0100000000000001 ++ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.EventSourcin...izers/SymbolSerializer | 0100000000000001 ++ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002 ++ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002 ++ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000002 | (0014) Path | /qix/bar | 0100000000000002 ++ | 0200000000000002 | (0015) Hash | 0x00000000DEADBEAF | 0100000000000002 ++ | 0200000000000002 | (0016) Size | 77 B | 0100000000000002 ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000003 | (0018) Name | Test Mod 1 | 0100000000000002 ++ | 0200000000000004 | (0018) Name | Test Mod 2 | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 ++ | 0200000000000002 | (0014) Path | /foo/qux | 0100000000000003 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt new file mode 100644 index 00000000..e4e846a6 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETCurrent.verified.txt @@ -0,0 +1,2 @@ ++ | 0200000000000002 | (0017) ModId | 0200000000000003 | 0100000000000002 ++ | 0200000000000001 | (0017) ModId | 0200000000000005 | 0100000000000003 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt new file mode 100644 index 00000000..251d8d2b --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTestVerifyData/RocksDB.InsertedDatomsShowUpInTheIndex_type=VAETHistory.verified.txt @@ -0,0 +1 @@ ++ | 0200000000000001 | (0017) ModId | 0200000000000003 | 0100000000000002 diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/InMemoryTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/InMemoryTests.cs new file mode 100644 index 00000000..e301ca63 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/InMemoryTests.cs @@ -0,0 +1,9 @@ +using NexusMods.EventSourcing.Storage.InMemoryBackend; + +namespace NexusMods.EventSourcing.Storage.Tests.BackendTests; + +public class InMemoryTests(IServiceProvider provider) : ABackendTest(provider, (registry) => new Backend(registry)) +{ + + +} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/RocksDB.cs b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/RocksDB.cs new file mode 100644 index 00000000..699cb2f5 --- /dev/null +++ b/tests/NexusMods.EventSourcing.Storage.Tests/BackendTests/RocksDB.cs @@ -0,0 +1,14 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.RocksDbBackend; +using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; +using NexusMods.EventSourcing.TestModel.Helpers; +using NexusMods.Hashing.xxHash64; +using FileAttributes = NexusMods.EventSourcing.TestModel.ComplexModel.Attributes.FileAttributes; + +namespace NexusMods.EventSourcing.Storage.Tests.BackendTests; + +public class RocksDB(IServiceProvider provider) : ABackendTest(provider, (registry) => new Backend(registry)) +{ + + +} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/AttributesAndValuesForEntity.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/AttributesAndValuesForEntity.cs deleted file mode 100644 index c997161a..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/AttributesAndValuesForEntity.cs +++ /dev/null @@ -1,24 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class AttributesAndValuesForEntity(IServiceProvider provider) : AStorageTest(provider) -{ - - [Fact] - public async Task CanGetAttributesAndValuesOnAnEntity() - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, "Mod 1")]); - - var realId = tx.Remaps[tmpId]; - - var attributes = DatomStore.GetAttributesForEntity(realId, TxId.MaxValue).ToList(); - - attributes.OfType() - .Where(d => d.E == realId) - .Should().ContainSingle().Which.V.Should().Be("Mod 1"); - } - -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/ByAttributeTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/ByAttributeTests.cs deleted file mode 100644 index 5e3ac623..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/ByAttributeTests.cs +++ /dev/null @@ -1,24 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class ByAttributeTests(IServiceProvider provider) : AStorageTest(provider) -{ - - [Fact] - public async Task GetEntitiesWithAttributes() - { - - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, "Mod 1")]); - - var realId = tx.Remaps[tmpId]; - - var entities = DatomStore.GetEntitiesWithAttribute(TxId.MaxValue).ToList(); - - entities.Should().Contain(realId); - - } - -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMaxEntityIdTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMaxEntityIdTests.cs deleted file mode 100644 index 9afc015e..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMaxEntityIdTests.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class GetMaxEntityIdTests(IServiceProvider provider) : AStorageTest(provider) -{ - [Fact] - public async Task CanGetMaxEntityId() - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, "Mod 1")]); - - var realId = tx.Remaps[tmpId]; - - var maxId = DatomStore.GetMaxEntityId(); - - maxId.Should().Be(realId); - } - -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMostRecentTxTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMostRecentTxTests.cs deleted file mode 100644 index e86824ef..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/GetMostRecentTxTests.cs +++ /dev/null @@ -1,18 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class GetMostRecentTxTests(IServiceProvider provider) : AStorageTest(provider) -{ - [Fact] - public async Task CanGetLatestTx() - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, "Mod 1")]); - - var latestTx = DatomStore.GetMostRecentTxId(); - - latestTx.Should().Be(tx.TxId); - } -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetExactTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetExactTests.cs deleted file mode 100644 index 073390d5..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetExactTests.cs +++ /dev/null @@ -1,87 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class TryGetExactTests(IServiceProvider provider) : AStorageTest(provider) -{ - - [Theory] - [InlineData("")] - [InlineData("Mod 1")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name")] - public async Task CanGetNewValue(string value) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, value)]); - - var realId = tx.Remaps[tmpId]; - - DatomStore.TryGetExact(realId, tx.TxId, out var modName).Should().BeTrue(); - - modName.Should().Be(value); - } - - [Theory] - [InlineData("", "Mod 1")] - [InlineData("Mod 1", "Some really long name that is definitely longer than most people would use for a mod name")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name", "")] - public async Task UpdatingValueReturnsNewValue(string first, string second) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, first)]); - var realId = tx.Remaps[tmpId]; - - DatomStore.TryGetExact(realId, tx.TxId, out var modName).Should().BeTrue(); - modName.Should().Be(first); - - var tx2 = await DatomStore.Transact([ModAttributes.Name.Assert(realId, second)]); - - DatomStore.TryGetExact(realId, tx2.TxId, out var modName2).Should().BeTrue(); - modName2.Should().Be(second); - } - - [Theory] - [InlineData("Mod 1", "Mod 2")] - [InlineData("Mod 2", "Mod 3")] - public async Task AssertsDoNotClobberOtherResults(string a, string b) - { - var tmpId1 = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tmpId2 = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 2)); - - var tx = await DatomStore.Transact([ - ModAttributes.Name.Assert(tmpId1, a), ModAttributes.Name.Assert(tmpId2, b) - ]); - var realId1 = tx.Remaps[tmpId1]; - var realId2 = tx.Remaps[tmpId2]; - - DatomStore.TryGetExact(realId1, tx.TxId, out var modName1).Should().BeTrue(); - modName1.Should().Be(a); - - DatomStore.TryGetExact(realId2, tx.TxId, out var modName2).Should().BeTrue(); - modName2.Should().Be(b); - } - - [Fact] - public void InvalidIdReturnsFalse() - { - DatomStore.TryGetExact(EntityId.From(Ids.MakeId(Ids.Partition.Entity, ulong.MaxValue)), - TxId.From(1), out var modName) - .Should().BeFalse(); - } - - [Theory] - [InlineData("")] - [InlineData("Mod 1")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name")] - public async Task InvalidTxReturnsFalse(string value) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, value)]); - var realId = tx.Remaps[tmpId]; - - DatomStore.TryGetExact(realId, TxId.From(1), out var modName) - .Should().BeFalse(); - } -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetLatestTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetLatestTests.cs deleted file mode 100644 index 00d1dd17..00000000 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStoreTests/TryGetLatestTests.cs +++ /dev/null @@ -1,121 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; - -namespace NexusMods.EventSourcing.Storage.Tests.DatomStoreTests; - -public class TryGetLatestTests(IServiceProvider provider) : AStorageTest(provider) -{ - - [Theory] - [InlineData("")] - [InlineData("Mod 1")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name")] - public async Task CanGetLatestValueAsOfTx(string value) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, value)]); - - var realId = tx.Remaps[tmpId]; - - DatomStore.TryGetLatest(realId, tx.TxId, out var modName).Should().BeTrue(); - - modName.Should().Be(value); - } - - [Theory] - [InlineData("")] - [InlineData("Mod 1")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name")] - public async Task CanGetLatestValueAsOfMaxTx(string value) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, value)]); - - var realId = tx.Remaps[tmpId]; - - DatomStore.TryGetLatest(realId, TxId.MaxValue, out var modName).Should().BeTrue(); - - modName.Should().Be(value); - } - - [Theory] - [InlineData("", "Mod 1")] - [InlineData("Mod 1", "Some really long name that is definitely longer than most people would use for a mod name")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name", "")] - public async Task CanGetOldValuesViaTx(string first, string second) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, first)]); - - var realId = tx.Remaps[tmpId]; - - var oldTx = tx.TxId; - - DatomStore.TryGetLatest(realId, TxId.MaxValue, out var modName).Should().BeTrue(); - modName.Should().Be(first); - - var tx2 = await DatomStore.Transact([ModAttributes.Name.Assert(realId, second)]); - - var newTx = tx2.TxId; - - - DatomStore.TryGetLatest(realId, newTx, out modName).Should().BeTrue(); - modName.Should().Be(second); - - DatomStore.TryGetLatest(realId, TxId.MaxValue, out modName).Should().BeTrue(); - modName.Should().Be(second); - - DatomStore.TryGetLatest(realId, oldTx, out modName).Should().BeTrue(); - modName.Should().Be(first); - } - - - [Theory] - [InlineData("", "Mod 1")] - [InlineData("Mod 1", "Some really long name that is definitely longer than most people would use for a mod name")] - [InlineData("Some really long name that is definitely longer than most people would use for a mod name", "")] - public async Task CanGetOldValuesViaFuzzyTx(string first, string second) - { - var tmpId = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 1)); - var tx = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId, first)]); - - var realId = tx.Remaps[tmpId]; - - var oldTx = tx.TxId; - - DatomStore.TryGetLatest(realId, TxId.MaxValue, out var modName).Should().BeTrue(); - modName.Should().Be(first); - - TxId midTx = default!; - { - var tmpId2 = EntityId.From(Ids.MakeId(Ids.Partition.Tmp, 2)); - for (var i = 0; i < 10; i += 1) - { - var midResult = await DatomStore.Transact([ModAttributes.Name.Assert(tmpId2, second)]); - if (i == 5) - midTx = midResult.TxId; - } - } - - var tx2 = await DatomStore.Transact([ModAttributes.Name.Assert(realId, second)]); - - var newTx = tx2.TxId; - - - // We can get newest value by it exact Tx - DatomStore.TryGetLatest(realId, newTx, out modName).Should().BeTrue(); - modName.Should().Be(second); - - // And by the max Tx - DatomStore.TryGetLatest(realId, TxId.MaxValue, out modName).Should().BeTrue(); - modName.Should().Be(second); - - // And the old value by the old Tx - DatomStore.TryGetLatest(realId, oldTx, out modName).Should().BeTrue(); - modName.Should().Be(first); - - // And the old value via an unrelated Tx (between the first two inserts to this entity) - DatomStore.TryGetLatest(realId, midTx, out modName).Should().BeTrue(); - modName.Should().Be(first); - } -} diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/NexusMods.EventSourcing.Storage.Tests.csproj b/tests/NexusMods.EventSourcing.Storage.Tests/NexusMods.EventSourcing.Storage.Tests.csproj index 9bcd571b..5485183c 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/NexusMods.EventSourcing.Storage.Tests.csproj +++ b/tests/NexusMods.EventSourcing.Storage.Tests/NexusMods.EventSourcing.Storage.Tests.csproj @@ -27,6 +27,34 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + + + + + + + + all + runtime; build; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/Startup.cs b/tests/NexusMods.EventSourcing.Storage.Tests/Startup.cs index 196b9b55..2f394f0f 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/Startup.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/Startup.cs @@ -1,8 +1,13 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Storage.RocksDbBackend; +using NexusMods.EventSourcing.Storage.Serializers; +using NexusMods.EventSourcing.TestModel; using NexusMods.EventSourcing.TestModel.ComplexModel.Attributes; +using NexusMods.EventSourcing.TestModel.ValueSerializers; using Xunit.DependencyInjection.Logging; +using FileAttributes = NexusMods.EventSourcing.TestModel.ComplexModel.Attributes.FileAttributes; namespace NexusMods.EventSourcing.Storage.Tests; @@ -11,7 +16,13 @@ public class Startup public void ConfigureServices(IServiceCollection services) { services.AddEventSourcingStorage() + .AddSingleton() .AddLogging(builder => builder.AddXunitOutput().SetMinimumLevel(LogLevel.Debug)) - .AddAttribute(); + .AddValueSerializer() + .AddValueSerializer() + .AddValueSerializer() + .AddValueSerializer() + .AddAttributeCollection() + .AddAttributeCollection(); } } diff --git a/tests/NexusMods.EventSourcing.TestModel/ComplexModel/Attributes/FileAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/ComplexModel/Attributes/FileAttributes.cs index 5895af65..9443d1b4 100644 --- a/tests/NexusMods.EventSourcing.TestModel/ComplexModel/Attributes/FileAttributes.cs +++ b/tests/NexusMods.EventSourcing.TestModel/ComplexModel/Attributes/FileAttributes.cs @@ -8,7 +8,7 @@ public class FileAttributes /// /// The path of the file /// - public class Path : ScalarAttribute; + public class Path() : ScalarAttribute(isIndexed: true); /// /// The size of the file @@ -18,7 +18,7 @@ public class Size : ScalarAttribute; /// /// The hashcode of the file /// - public class Hash : ScalarAttribute; + public class Hash() : ScalarAttribute(isIndexed: true); /// /// The mod this file belongs to diff --git a/tests/NexusMods.EventSourcing.TestModel/Helpers/ExtensionMethods.cs b/tests/NexusMods.EventSourcing.TestModel/Helpers/ExtensionMethods.cs new file mode 100644 index 00000000..c613088d --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Helpers/ExtensionMethods.cs @@ -0,0 +1,73 @@ +using System.Text; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Internals; + +namespace NexusMods.EventSourcing.TestModel.Helpers; + +public static class ExtensionMethods +{ + public static IEnumerable ToObjectDatoms(this IEnumerable datoms, IAttributeRegistry registry) + { + foreach (var datom in datoms) + { + var aSym = registry.GetSymbolForAttribute(datom.AttributeType); + yield return new ObjectTuple + { + E = datom.E, + A = aSym.Name, + V = datom.ObjectValue, + T = datom.T, + }; + } + } + + public static string ToTable(this IEnumerable datoms, IAttributeRegistry registry) + { + string TruncateOrPad(string val, int length) + { + if (val.Length > length) + { + var midPoint = length / 2; + return (val[..(midPoint - 2)] + "..." + val[^(midPoint - 2)..]).PadRight(length); + } + + return val.PadRight(length); + } + + var sb = new StringBuilder(); + foreach (var datom in datoms) + { + var isAssert = true; + + var symColumn = TruncateOrPad(registry.GetSymbolForAttribute(datom.AttributeType).Name, 24); + var attrId = registry.GetAttributeId(datom.AttributeType).Value.ToString("X4"); + + sb.Append(isAssert ? "+" : "-"); + sb.Append(" | "); + sb.Append(datom.E.Value.ToString("X16")); + sb.Append(" | "); + sb.Append($"({attrId}) {symColumn}"); + sb.Append(" | "); + + switch (datom.ObjectValue) + { + case EntityId eid: + sb.Append(eid.Value.ToString("X16").PadRight(48)); + break; + case ulong ul: + sb.Append(ul.ToString("X16").PadRight(48)); + break; + default: + sb.Append(TruncateOrPad(datom.ObjectValue.ToString()!, 48)); + break; + } + + sb.Append(" | "); + sb.Append(datom.T.Value.ToString("X16")); + + sb.AppendLine(); + + } + return sb.ToString(); + } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Helpers/ObjectTuple.cs b/tests/NexusMods.EventSourcing.TestModel/Helpers/ObjectTuple.cs new file mode 100644 index 00000000..3cdc2fe0 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Helpers/ObjectTuple.cs @@ -0,0 +1,55 @@ +using Argon; +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Helpers; + +public class ObjectTuple +{ + public EntityId E { get; init; } + public required string A { get; init; } + public required object V { get; init; } + public TxId T { get; init; } + + public bool IsAssert { get; init; } +} + + +public class ObjectTupleWriter : JsonConverter +{ + public override void WriteJson(JsonWriter writer, ObjectTuple value, JsonSerializer serializer) + { + var oldFormatting = writer.Formatting; + writer.WriteStartArray(); + writer.Formatting = Formatting.None; + + writer.WriteValue(value.E.Value.ToString("x")); + + writer.WriteValue(" "+value.A); + + switch (value.V) + { + case EntityId eid: + writer.WriteValue(eid.Value.ToString("x")); + break; + case ulong ul: + writer.WriteValue(ul.ToString("x")); + break; + default: + writer.WriteValue(value.V.ToString()); + break; + } + writer.WriteValue(value.T.Value.ToString("x")); + + writer.WriteValue(value.IsAssert ? "assert" : "retract"); + + writer.WriteEndArray(); + writer.Formatting = oldFormatting; + + } + + public override ObjectTuple ReadJson(JsonReader reader, Type type, ObjectTuple? existingValue, bool hasExisting, + JsonSerializer serializer) + { + throw new NotSupportedException(); + } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/LoadoutActiveReadModel.cs b/tests/NexusMods.EventSourcing.TestModel/Model/LoadoutActiveReadModel.cs deleted file mode 100644 index 18accf71..00000000 --- a/tests/NexusMods.EventSourcing.TestModel/Model/LoadoutActiveReadModel.cs +++ /dev/null @@ -1,15 +0,0 @@ -using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Abstractions.Models; -using NexusMods.EventSourcing.TestModel.Model.Attributes; - -namespace NexusMods.EventSourcing.TestModel.Model; - -public class LoadoutActiveReadModel : AActiveReadModel -{ - public LoadoutActiveReadModel(IDb basisDb, EntityId id) : base(basisDb, id) - { - } - - [From] - public string Name { get; set; } = string.Empty; -} diff --git a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj index 188094d1..91eee559 100644 --- a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj +++ b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj @@ -12,7 +12,36 @@ - + + + + + + + + + all + runtime; build; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs index 267eb1b5..f05765e7 100644 --- a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs +++ b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; using NexusMods.EventSourcing.Storage; +using NexusMods.EventSourcing.Storage.RocksDbBackend; using NexusMods.Paths; namespace NexusMods.EventSourcing.Tests; @@ -17,6 +18,7 @@ public class AEventSourcingTest : IAsyncLifetime private readonly IAttribute[] _attributes; private readonly IServiceProvider _provider; private readonly AttributeRegistry _registry; + private Backend _backend; protected AEventSourcingTest(IServiceProvider provider) @@ -32,8 +34,9 @@ protected AEventSourcingTest(IServiceProvider provider) Path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory) .Combine("tests_eventsourcing" + Guid.NewGuid()) }; + _backend = new Backend(_registry); - _store = new DatomStore(provider.GetRequiredService>(), _registry, Config); + _store = new DatomStore(provider.GetRequiredService>(), _registry, Config, _backend); Logger = provider.GetRequiredService>(); @@ -44,9 +47,11 @@ protected async Task RestartDatomStore() { _store.Dispose(); + _backend.Dispose(); - _store = new DatomStore(_provider.GetRequiredService>(), _registry, Config); + _backend = new Backend(_registry); + _store = new DatomStore(_provider.GetRequiredService>(), _registry, Config, _backend); await _store.Sync(); Connection = await Connection.Start(_store, _valueSerializers, _attributes); diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index 319ed61b..e8ee5124 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -148,11 +148,12 @@ public async Task CanGetCommitUpdates() var realId = result[file.Id]; - Connection.Commits.Subscribe(update => + Connection.Revisions.Subscribe(update => { + var datoms = update.Datoms(update.BasisTxId).ToArray(); // Only Txes we care about - if (update.Datoms.Any(d => d.E == realId)) - updates.Add(update.Datoms.ToArray()); + if (datoms.Any(d => d.E == realId)) + updates.Add(datoms); }); for (var idx = 0; idx < 10; idx++) @@ -161,7 +162,6 @@ public async Task CanGetCommitUpdates() ModFileAttributes.Index.Add(tx, realId, (ulong)idx); result = await tx.Commit(); - //result.Datoms.Should().BeEquivalentTo(updates[idx + 1]); await Task.Delay(100); updates.Should().HaveCount(idx + 1); @@ -198,43 +198,4 @@ public async Task CanGetChildEntities() loadout.Name.Should().Be("Test Loadout"); firstMod.Loadout.Name.Should().Be("Test Loadout"); } - - [Fact] - public async Task CanGetActiveReadModels() - { - - var tx = Connection.BeginTransaction(); - var staticLoadout1 = Loadout.Create(tx, "Test Loadout 1"); - var staticLoadout2 = Loadout.Create(tx, "Test Loadout 2"); - - var result = await tx.Commit(); - - var loadout1 = Connection.GetActive(result[staticLoadout1.Id]); - var loadout2 = Connection.GetActive(result[staticLoadout2.Id]); - - loadout1.Name.Should().Be("Test Loadout 1"); - loadout2.Name.Should().Be("Test Loadout 2"); - - var newTx = Connection.BeginTransaction(); - LoadoutAttributes.Name.Add(newTx, result[staticLoadout1.Id], "Test Loadout 1 Updated"); - await newTx.Commit(); - - var reloaded = Connection.Db.Get(result[staticLoadout1.Id]); - reloaded.Name.Should().Be("Test Loadout 1 Updated", "because the commit has been applied"); - - - var loadout1Reloaded = Connection.GetActive(result[staticLoadout1.Id]); - - await Task.Delay(100); - loadout1Reloaded.Name.Should().Be("Test Loadout 1 Updated", "because the model is reloaded from the db"); - - - loadout1.BasisDb.BasisTxId.Should().Be(loadout1Reloaded.BasisDb.BasisTxId, "the basis db should be updated"); - - loadout1.Name.Should().Be("Test Loadout 1 Updated", "because the model is active"); - - loadout2.Name.Should().Be("Test Loadout 2", "because the model is active, but not updated"); - } - - } diff --git a/tests/NexusMods.EventSourcing.Tests/NexusMods.EventSourcing.Tests.csproj b/tests/NexusMods.EventSourcing.Tests/NexusMods.EventSourcing.Tests.csproj index aa1b98bb..93408fca 100644 --- a/tests/NexusMods.EventSourcing.Tests/NexusMods.EventSourcing.Tests.csproj +++ b/tests/NexusMods.EventSourcing.Tests/NexusMods.EventSourcing.Tests.csproj @@ -9,15 +9,34 @@ - + all runtime; build; contentfiles; analyzers; buildtransitive - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/tests/SharedUsings.cs b/tests/SharedUsings.cs index def1ba23..48d1f957 100644 --- a/tests/SharedUsings.cs +++ b/tests/SharedUsings.cs @@ -3,3 +3,14 @@ global using FluentAssertions; global using AutoFixture; global using AutoFixture.Xunit2; +using System.Runtime.CompilerServices; +using NexusMods.EventSourcing.TestModel.Helpers; + +public static class Initializer +{ + [ModuleInitializer] + public static void Init() + => VerifierSettings + .AddExtraSettings(s => + s.Converters.Add(new ObjectTupleWriter())); +}