diff --git a/NexusMods.EventSourcing.sln b/NexusMods.EventSourcing.sln index add754d6..31b0183c 100644 --- a/NexusMods.EventSourcing.sln +++ b/NexusMods.EventSourcing.sln @@ -35,8 +35,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.DatomStore.Tests", "tests\NexusMods.EventSourcing.DatomStore.Tests\NexusMods.EventSourcing.DatomStore.Tests.csproj", "{81CCE07D-818D-4153-8486-5D2A860C4D9D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.SourceGenerator", "src\NexusMods.EventSourcing.SourceGenerator\NexusMods.EventSourcing.SourceGenerator.csproj", "{F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Tests", "tests\NexusMods.EventSourcing.Tests\NexusMods.EventSourcing.Tests.csproj", "{07E2C578-8644-474D-8F07-B25CFEB28408}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.EventSourcing.Benchmarks", "benchmarks\NexusMods.EventSourcing.Benchmarks\NexusMods.EventSourcing.Benchmarks.csproj", "{930B3AB7-56EA-48D6-B603-24D79C7DD00A}" @@ -52,7 +50,6 @@ Global {F2C1FB09-D01D-4E8B-B6BE-B548AB00187B} = {0377EBE6-F147-4233-86AD-32C821B9567E} {EC1570A4-18B9-4A76-84FF-275BAA76A357} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {81CCE07D-818D-4153-8486-5D2A860C4D9D} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A} = {0377EBE6-F147-4233-86AD-32C821B9567E} {07E2C578-8644-474D-8F07-B25CFEB28408} = {6ED01F9D-5E12-4EB2-9601-64A2ADC719DE} {930B3AB7-56EA-48D6-B603-24D79C7DD00A} = {72AFE85F-8C12-436A-894E-638ED2C92A76} EndGlobalSection @@ -77,10 +74,6 @@ Global {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Debug|Any CPU.Build.0 = Debug|Any CPU {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Release|Any CPU.ActiveCfg = Release|Any CPU {81CCE07D-818D-4153-8486-5D2A860C4D9D}.Release|Any CPU.Build.0 = Release|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F2A6F6B9-5D36-4416-BDD8-C7D30EE3ED4A}.Release|Any CPU.Build.0 = Release|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Debug|Any CPU.Build.0 = Debug|Any CPU {07E2C578-8644-474D-8F07-B25CFEB28408}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs index b558205b..5b16a922 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/AppHost.cs @@ -1,7 +1,6 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using NexusMods.EventSourcing.Benchmarks.Model; using NexusMods.EventSourcing.DatomStore; using NexusMods.Paths; @@ -16,7 +15,7 @@ public static IServiceProvider Create() { services.AddDatomStore() .AddEventSourcing() - .AddFileModel() + .AddTestModel() .AddSingleton(new DatomStoreSettings() { Path = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb") diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs index cde53e36..1933ad70 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/ReadTests.cs @@ -1,51 +1,89 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using BenchmarkDotNet.Attributes; using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Benchmarks.Model; +using NexusMods.EventSourcing.TestModel.Model; namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; +[MemoryDiagnoser] public class ReadTests { private readonly IConnection _connection; - private readonly List _entityIds; + private List _entityIdsAscending = null!; + private List _entityIdsDescending = null!; + private List _entityIdsRandom = null!; public ReadTests() { var services = AppHost.Create(); _connection = services.GetRequiredService(); - _entityIds = new List(); } + public const int MaxCount = 10000; + [GlobalSetup] public void Setup() { var tx = _connection.BeginTransaction(); - _entityIds.Clear(); - for (var i = 0; i < Count; i++) + var entityIds = new List(); + for (var i = 0; i < MaxCount; i++) { - var id = Ids.MakeId(Ids.Partition.Entity, (ulong)i); - File.Hash.Assert(tx.TempId(), (ulong)i, tx); - File.Path.Assert(tx.TempId(), $"C:\\test_{i}.txt", tx); - File.Index.Assert(tx.TempId(), (ulong)i, tx); - _entityIds.Add(EntityId.From(id)); + var file = new File(tx) + { + Hash = (ulong)i, + Path = $"C:\\test_{i}.txt", + Index = (ulong)i + }; + entityIds.Add(file.Id); } - tx.Commit(); + var result = tx.Commit(); + + entityIds = entityIds.Select(e => result[e]).ToList(); + _entityIdsAscending = entityIds.OrderBy(id => id.Value).ToList(); + _entityIdsDescending = entityIds.OrderByDescending(id => id.Value).ToList(); + + var idArray = entityIds.ToArray(); + Random.Shared.Shuffle(idArray); + _entityIdsRandom = idArray.ToList(); + } + + + [Params(1, 1000, MaxCount)] + public int Count { get; set; } = MaxCount; + + public enum SortOrder + { + Ascending, + Descending, + Random } - [Params(1, 10, 100, 1000)] - public int Count { get; set; } = 1000; + //[Params(SortOrder.Ascending, SortOrder.Descending, SortOrder.Random)] + public SortOrder Order { get; set; } = SortOrder.Descending; + + public List Ids => Order switch + { + SortOrder.Ascending => _entityIdsAscending, + SortOrder.Descending => _entityIdsDescending, + SortOrder.Random => _entityIdsRandom, + _ => throw new ArgumentOutOfRangeException() + }; [Benchmark] - public int ReadFiles() + public ulong ReadFiles() { var db = _connection.Db; - var read = db.Get(_entityIds).ToList(); - return read.Count; + ulong sum = 0; + foreach (var itm in db.Get(Ids.Take(Count))) + { + sum += itm.Index; + } + return (ulong)sum; } } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs index 1cafa79b..d523783c 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Benchmarks/WriteTests.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Attributes; using Microsoft.Extensions.DependencyInjection; using NexusMods.EventSourcing.Abstractions; -using NexusMods.EventSourcing.Benchmarks.Model; namespace NexusMods.EventSourcing.Benchmarks.Benchmarks; @@ -23,6 +22,7 @@ public WriteTests() [Benchmark] public void AddFiles() { + /* var tx = _connection.BeginTransaction(); for (var i = 0; i < Count; i++) { @@ -32,6 +32,7 @@ public void AddFiles() File.Index.Assert(tx.TempId(), (ulong)i, tx); } tx.Commit(); + */ } } diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs deleted file mode 100644 index ef4dadb9..00000000 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Model/File.cs +++ /dev/null @@ -1,14 +0,0 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; - -namespace NexusMods.EventSourcing.Benchmarks.Model; - -[ModelDefinition] -public static partial class File -{ - public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() - .Define("Path", "The path of the file") - .Define("Hash", "The hash of the file") - .Define("Index", "A index value for testing purposes") - .Build(); - -} diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj index 8652ecf5..b9cbe311 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/NexusMods.EventSourcing.Benchmarks.csproj @@ -14,8 +14,8 @@ - + diff --git a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs index 21dfbcdd..96992f4b 100644 --- a/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.EventSourcing.Benchmarks/Program.cs @@ -9,7 +9,7 @@ #if DEBUG var benchmark = new ReadTests(); -benchmark.Count = 1; +benchmark.Count = 1000; var sw = Stopwatch.StartNew(); benchmark.Setup(); diff --git a/docs/AttributeDefinitions.md b/docs/AttributeDefinitions.md new file mode 100644 index 00000000..8236bcf0 --- /dev/null +++ b/docs/AttributeDefinitions.md @@ -0,0 +1,159 @@ +--- +hide: + - toc +--- + + +## Attribute Definitions +The datoms stored in the store require typed attributes with unique name and serializer tagging. Storing this information in +a way that provides typed access to attribues is strangely difficult. There are several ways to tackle this issue, which we will talk about here. + +### Symbolic Names +The simplest approach is to do what Datomic does and use symbolic names for attributes. For example a `:loadout/name` attribute +would be registered as a string type: + +```csharp + + System.RegisterAttribute("loadout/name"); + var nameSymbol = Symbol.Intern("loadout/name"); + + tx.Add(eid, nameSymbol, "My Loadout"); +``` + +The problem with this approach is there is nothing stopping someone from using the wrong type for an attribute. The error happens +at runtime instead of at compile time. In addition loading values from the store may result in boxing unless care is taken. + +This approach does allow for the use of fairly dynamic queries however. + +```csharp +var query = from e in db.Entities + where e[Loadout_Name] == "My Loadout" + select e.Pull(Loadout_Name, Loadout_Version); + +// What is the type of query result? +var results = query.ToList(); +``` + +!!!info + Dapper uses this approach, as does Datomic, but it assumes a dynamic query result. This is not a bad thing, but it does +reduce the ability to use the type system to catch errors. + +If we want to predefine a query model, we end up with something even more complex + +```csharp + +interface QueryResult +{ + string Name { get; } + int Version { get; } +} + + +var results = from e in db.Entities() + where e.Name == "My Loadout" + select e; +```` + +The question here is how we map the symbolic names to the result type. We can use attributes, but attributes must have +constant values as arguments, so we can't symbolc names and must use strings or some sort of `nameof` expression. + +```csharp +interface QueryResult +{ + [From("loadout/name")] + string Name { get; } + [From(nameof(NexusMods.Model.Loadout_Version))] + int Version { get; } +} +``` + +Since `nameof` only names the specific type, we have to give it a fully qualified name. This is also suboptimal. + +### Attributes as Types + +Another approach would be to use the type system to define the attributes. This has the advantage of being able to use the types +to provide strict type checking, and we can use attribues to provide the symbolic names. + +```csharp +namespace Loadout { + public class Name : Attribute(); + public class Version : Attribute(); +} + +public class QueryResult { + [From] + public string Name { get; } + [From] + public int Version { get; } +} +``` + +Unfortunately in this approach we have to make sure to use the correct type for the getter in the read model. There's nothing +stopping us from accidentally defining `Name` as an `int` for example. This would not result in a compile time error. If we pre-register +all our read models (like `QueryResult`) we can use reflection to check the types and at least we get a startup time error. + +Another problem with this approach is that C#'s inference system is not good at resolving complex constraints, for example: + +```csharp + +// This requires us to know at usage time that Name is a string attribute, and to make sure that the type is correct. +tx.Add("foo"); + +// What we can do, is put the `.Bar` method in a static extension method + +tx.Add("foo"); + +public static class AttributeExtensions +{ + public static void Add(this Transaction tx, T attribute, string value) + where T : Attribute + { + tx.Add(attribute, value); + } + + public static void Add(this Transaction tx, T attribute, float value) + where T : Attribute + { + tx.Add(attribute, value); + } +} +``` + +This works quite well, and we only need to perform the operation once per attribute value type. After we do this we can easily define +models that use the attributes. + +```csharp + +record ReadModel +{ + [From] + public string Name { get; } + [From] + public int Version { get; } +} + +record WriteModel +{ + [From] + public required string Name { get; init;} + [From] + public required int Version { get; init;} +} + +/// Define a model that responds to new transactions and fires INotifyPropertyChanged events +public class ActiveModel : ActiveModel +{ + [From] + public string Name { get; set; } + [From] + public int Version { get; set; } +} + +/// Perform an ad-hoc query +/// Find files with the path of "c:\temp\foo.txt" and look up the name and version of the mods that contain them. +var results = from f in db.Where(@"c:\temp\foo.txt") + from m in db.Where(f.EntityId) + select db.Pull(m.EntityId); +``` + + diff --git a/src/NexusMods.EventSourcing.Abstractions/Datom.cs b/src/NexusMods.EventSourcing.Abstractions/Datom.cs index 4772508d..2fbc915a 100644 --- a/src/NexusMods.EventSourcing.Abstractions/Datom.cs +++ b/src/NexusMods.EventSourcing.Abstractions/Datom.cs @@ -10,11 +10,11 @@ public interface IDatom void Emit(ref TSink sink) where TSink : IDatomSink; /// - /// Duplicates the datom with a new entity id + /// The datom should call the remap function on each entity id it contains + /// to remap the entity ids to actual ids /// - /// - /// - IDatom RemapEntityId(ulong newId); + /// + void Remap(Func remapFn); } public interface IDatomWithTx : IDatom @@ -36,9 +36,13 @@ public void Emit(ref TSink sink) where TSink : IDatomSink } /// - public IDatom RemapEntityId(ulong newId) + public void Remap(Func remapFn) { - return new AssertDatom(newId, v); + e = remapFn(EntityId.From(e)).Value; + if (v is EntityId entityId) + { + v = (TVal) (object) EntityId.From(remapFn(entityId).Value); + } } } diff --git a/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs b/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs index aa3444f4..913bf18a 100644 --- a/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs +++ b/src/NexusMods.EventSourcing.Abstractions/DependencyInjectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -33,17 +34,4 @@ public static IServiceCollection AddValueSerializer(this IServ services.AddSingleton(); return services; } - - /// - /// Registers the specified read model factory type with the service collection. - /// - /// - /// - /// - public static IServiceCollection AddReadModelFactory(this IServiceCollection services) - where TReadModelFactory : class, IReadModelFactory - { - services.AddSingleton(); - return services; - } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs index 0dbc4db0..ee0be7a3 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IAttribute.cs @@ -50,7 +50,7 @@ public interface IAttribute /// Typed variant of IAttribute /// /// -public interface IAttribute : IAttribute +public interface IAttribute : IAttribute { /// @@ -60,4 +60,13 @@ public interface IAttribute : IAttribute /// public TVal Read(ReadOnlySpan buffer); + + /// + /// Creates a new assertion datom for the given entity and value + /// + /// + /// + /// + public static abstract void Add(ITransaction tx, EntityId entity, TVal value); + } diff --git a/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs b/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs index 5c0fe101..0e277483 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ICommitResult.cs @@ -1,4 +1,6 @@ -namespace NexusMods.EventSourcing.Abstractions; +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions; /// /// The result of a transaction commit, contains metadata useful for looking up the results of the transaction @@ -16,4 +18,9 @@ public interface ICommitResult /// Gets the new TxId after the commit /// public TxId NewTx { get; } + + /// + /// The datoms that were added to the store as a result of the transaction + /// + public IEnumerable Datoms { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs index 364e88f7..4964e142 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IConnection.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IConnection.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; namespace NexusMods.EventSourcing.Abstractions; @@ -29,4 +30,9 @@ public interface IConnection /// /// public ITransaction BeginTransaction(); + + /// + /// A sequential stream of commits to the database. + /// + public IObservable Commits { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs index 717fd8d3..cf489c33 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDatomStore.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq.Expressions; namespace NexusMods.EventSourcing.Abstractions; @@ -31,4 +32,19 @@ public interface IDatomStore : IDisposable /// /// void RegisterAttributes(IEnumerable newAttrs); + + /// + /// Gets the attributeId for the given attribute. And returns an expression that reads the attribute + /// value from the expression valueSpan. + /// + /// + /// + /// + /// + Expression GetValueReadExpression(Type attribute, Expression valueSpan, out ulong attributeId); + + /// + /// Gets all the entities that reference the given entity id with the given attribute. + /// + IEnumerable ReverseLookup(TxId txId) where TAttribute : IAttribute; } diff --git a/src/NexusMods.EventSourcing.Abstractions/IDb.cs b/src/NexusMods.EventSourcing.Abstractions/IDb.cs index 24a5df91..546d12c4 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IDb.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IDb.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -20,9 +21,24 @@ public IIterator Where() /// /// Returns a read model for each of the given entity ids. /// - /// + public IEnumerable Get(IEnumerable ids) + where TModel : IReadModel; + + + /// + /// Gets a read model for the given entity id. + /// + /// /// /// - public IEnumerable Get(IEnumerable ids) + public TModel Get(EntityId id) where TModel : IReadModel; + + /// + /// Gets a read model for every enitity that references the given entity id + /// with the given attribute. + /// + public IEnumerable GetReverse(EntityId id) + where TModel : IReadModel + where TAttribute : IAttribute; } diff --git a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs index 6acc1f1c..ffda6137 100644 --- a/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs +++ b/src/NexusMods.EventSourcing.Abstractions/IEntityIterator.cs @@ -1,9 +1,12 @@ -namespace NexusMods.EventSourcing.Abstractions; +using System; +using NexusMods.EventSourcing.Abstractions.Models; + +namespace NexusMods.EventSourcing.Abstractions; /// /// Represents an iterator over a set of datoms. /// -public interface IEntityIterator +public interface IEntityIterator : IDisposable { /// /// Move to the next datom for the current entity @@ -12,10 +15,11 @@ public interface IEntityIterator public bool Next(); /// - /// Sets the current entity id, this implicitly resets the iterator. + /// Sets the EntityId for the iterator, the next call to Next() will return the first datom for the given entity + /// that is less than or equal to the txId given to the iterator at creation. /// /// - public void SetEntityId(EntityId entityId); + public void Set(EntityId entityId); /// /// Gets the current datom as a distinct value. @@ -23,9 +27,22 @@ public interface IEntityIterator public IDatom Current { get; } /// - /// Sends the current datom to the read model. + /// Gets the current datom's value + /// + /// + /// + /// + public TValue GetValue() + where TAttribute : IAttribute; + + /// + /// Gets the current datom's attribute id + /// + public ulong AttributeId { get; } + + /// + /// Gets the current datom's value as a span, valid until the next call to Next() + /// or SetEntityId() /// - /// - /// - public void SetOn(TModel model) where TModel : IReadModel; + public ReadOnlySpan ValueSpan { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs deleted file mode 100644 index 45187d6f..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModel.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A read model is a set of attributes grouped together with a common entity id -/// -public interface IReadModel -{ - /// - /// The unique identifier of the entity in the read model - /// - public EntityId Id { get; } - - /// - /// Sets the value of an attribute in the model - /// - /// - /// - /// - public void Set(IAttribute attribute, ReadOnlySpan value); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs deleted file mode 100644 index 954ea499..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModelBuilder.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A push-based system for building up a IReadModel state -/// -public interface IReadModelBuilder -{ - /// - /// Sets the value of the given attribute to the given value - /// - /// - /// - /// - public void Set(IAttribute attr, ReadOnlySpan span); - - /// - /// Builds the collected data into a IReadModel - /// - /// - public IReadModel Build(EntityId id); -} - diff --git a/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs b/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs deleted file mode 100644 index 7e671d49..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/IReadModelFactory.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions; - -/// -/// A factory for creating read models. The attribute list is used to optimize -/// reading so that only the requested attributes have to be loaded from the store. -/// -public interface IReadModelFactory -{ - public Type ModelType { get; } - - /// - /// A collection of all the attributes in the model. - /// - public Type[] Attributes { get; } - - /// - /// Creates a new instance of the read model - /// - /// - public IReadModel Create(EntityId id); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs index d1971453..8906b293 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ITransaction.cs @@ -1,4 +1,5 @@ using System; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.Abstractions; @@ -15,8 +16,33 @@ public interface ITransaction /// void Add(IDatom datom); + + /// + /// Adds a new read model to the transaction, the datoms are extracted from the read model + /// as asserts for each property with the FromAttribute + /// + /// + TReadModel Add(TReadModel model) + where TReadModel : IReadModel; + + /// + /// Adds a new datom to the transaction + /// + /// + /// + /// + /// + /// + void Add(EntityId entityId, TVal val, bool isAssert = true) + where TAttribute : IAttribute; + /// /// Commits the transaction /// ICommitResult Commit(); + + /// + /// Gets the temporary id for the transaction + /// + public TxId ThisTxId { get; } } diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs deleted file mode 100644 index 8bce64f4..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitions.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -public class AttributeDefinitions -{ - public static AttributeDefinitions Instance = new(); -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs deleted file mode 100644 index a9cbab2e..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/AttributeDefinitionsBuilder.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder for a model definition for source generators -/// -public class AttributeDefinitionsBuilder -{ - /// - /// Placeholder for a model definition for source generators - /// - public AttributeDefinitionsBuilder() - { - - } - - /// - /// Defines a new attribute with the given name and description - /// - /// - /// - /// - /// - public AttributeDefinitionsBuilder Define(string name, string description) - { - return this; - } - - /// - /// Builds the final model definition - /// - /// - public AttributeDefinitions Build() - { - return AttributeDefinitions.Instance; - } - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs deleted file mode 100644 index 2d0135fd..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinition.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder class for the final result of the model generation -/// -public class ModelDefinition -{ - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs deleted file mode 100644 index a4798c91..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionAttribute.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System; - -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Marks a class as a model definition -/// -[AttributeUsage(AttributeTargets.Class)] -public class ModelDefinitionAttribute : Attribute; diff --git a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs b/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs deleted file mode 100644 index c42f8636..00000000 --- a/src/NexusMods.EventSourcing.Abstractions/ModelGeneration/ModelDefinitionBuilder.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace NexusMods.EventSourcing.Abstractions.ModelGeneration; - -/// -/// Placeholder class for generating model definitions for source generators -/// -public class ModelDefinitionBuilder -{ - /// - /// Placeholder class for generating model definitions for source generators - /// - public ModelDefinitionBuilder() - { - - } - - -} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs new file mode 100644 index 00000000..659adf54 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/AReadModel.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base class for all read models. +/// +/// +public abstract class AReadModel : IReadModel +where TOuter : AReadModel, IReadModel +{ + /// + /// Creates a new read model with a temporary id + /// + /// + protected AReadModel(ITransaction? tx) + { + if (tx is null) return; + Id = tx.TempId(); + tx.Add((TOuter)this); + } + + internal AReadModel(EntityId id) + { + Id = id; + } + + /// + /// Retrieves the read model from the database + /// + /// + /// + /// + /// + protected TReadModel Get(EntityId entityId) + where TReadModel : AReadModel, IReadModel + { + return Db.Get(entityId); + } + + /// + /// Retrieves the matching read models from the database via the specified reverse lookup attribute + /// + /// + /// + /// + /// + protected IEnumerable GetReverse() + where TReadModel : AReadModel, IReadModel + where TAttribute : ScalarAttribute + { + return Db.GetReverse(Id); + } + + /// + /// The base identifier for the entity. + /// + public EntityId Id { get; internal set; } + + /// + /// The database this read model is associated with. + /// + public IDb Db { get; internal set; } = null!; +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs new file mode 100644 index 00000000..b4eaadc2 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/FromAttribute.cs @@ -0,0 +1,17 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Marks a property as being derived from an attribute. +/// +/// +[AttributeUsage(AttributeTargets.Property)] +public class FromAttribute : Attribute, IFromAttribute + where TAttribute : IAttribute +{ + /// + /// Gets the type of the attribute. + /// + public Type AttributeType => typeof(TAttribute); +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs new file mode 100644 index 00000000..b907871e --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/IFromAttribute.cs @@ -0,0 +1,14 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base interface for all from attributes. +/// +public interface IFromAttribute +{ + /// + /// The attribute tag of this from attribute. + /// + public Type AttributeType { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs b/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs new file mode 100644 index 00000000..b6f3b1bc --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/IReadModel.cs @@ -0,0 +1,11 @@ +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Base interface for all read models. The AReadModel class takes a generic parameter +/// which makes it hard to use as a base class for all read models in cases where the user +/// doesn't care about the type of the read model. +/// +public interface IReadModel +{ + public EntityId Id { get; } +} diff --git a/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs new file mode 100644 index 00000000..6379306c --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/Models/ReverseLookupAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace NexusMods.EventSourcing.Abstractions.Models; + +/// +/// Defines a backwards lookup attribute +/// +public class ReverseLookupAttribute : Attribute +where TAttribute : ScalarAttribute +{ + +} diff --git a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs index 61f8906d..2c71b428 100644 --- a/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs +++ b/src/NexusMods.EventSourcing.Abstractions/ScalarAttribute.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.CompilerServices; -using System.Transactions; namespace NexusMods.EventSourcing.Abstractions; @@ -8,6 +7,7 @@ namespace NexusMods.EventSourcing.Abstractions; /// Interface for a specific attribute /// /// +/// public class ScalarAttribute : IAttribute where TAttribute : IAttribute { @@ -17,8 +17,10 @@ public class ScalarAttribute : IAttribute /// Create a new attribute /// /// - protected ScalarAttribute(string uniqueName) + protected ScalarAttribute(string uniqueName = "") { + if (uniqueName == "") + uniqueName = typeof(TAttribute).FullName!; Id = Symbol.Intern(uniqueName); } @@ -37,6 +39,13 @@ public TValueType Read(ReadOnlySpan buffer) return val; } + + /// + public static void Add(ITransaction tx, EntityId entity, TValueType value) + { + tx.Add(entity, value); + } + public void SetSerializer(IValueSerializer serializer) { if (serializer is not IValueSerializer valueSerializer) diff --git a/src/NexusMods.EventSourcing.Abstractions/Symbol.cs b/src/NexusMods.EventSourcing.Abstractions/Symbol.cs index b50e8781..daabea2b 100644 --- a/src/NexusMods.EventSourcing.Abstractions/Symbol.cs +++ b/src/NexusMods.EventSourcing.Abstractions/Symbol.cs @@ -14,8 +14,9 @@ public class Symbol /// private Symbol(string nsAndName) { + nsAndName = nsAndName.Replace("+", "."); Id = nsAndName; - var splitOn = nsAndName.LastIndexOf('/'); + var splitOn = nsAndName.LastIndexOf('.'); Name = nsAndName[(splitOn + 1)..]; Namespace = nsAndName[..splitOn]; } diff --git a/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs b/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs new file mode 100644 index 00000000..e4281922 --- /dev/null +++ b/src/NexusMods.EventSourcing.Abstractions/TransactionExtensionMethods.cs @@ -0,0 +1,6 @@ +namespace NexusMods.EventSourcing.Abstractions; + +public static class TransactionExtensionMethods +{ + +} diff --git a/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs b/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs index 6804158f..5325a7bf 100644 --- a/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs +++ b/src/NexusMods.EventSourcing.DatomStore/AttributeRegistry.cs @@ -2,7 +2,9 @@ using System.Buffers; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing.DatomStore; @@ -84,11 +86,23 @@ public IDatom ReadDatom(ref KeyHeader header, ReadOnlySpan valueSpan) return attribute.Read(header.Entity, header.Tx, header.IsAssert, valueSpan); } - public void SetOn(TModel model, ref KeyHeader key, ReadOnlySpan sliceFast) where TModel : IReadModel + public TValue ReadValue(ref KeyHeader currentHeader, ReadOnlySpan currentValue) + where TAttribute : IAttribute { - var attrId = key.AttributeId; + var attrId = currentHeader.AttributeId; var dbAttribute = _dbAttributesByEntityId[attrId]; - var attribute = _attributesById[dbAttribute.UniqueId]; - model.Set(attribute, sliceFast); + var attribute = (TAttribute)_attributesById[dbAttribute.UniqueId]; + return attribute.Read(currentValue); + } + + public Expression GetReadExpression(Type attributeType, Expression valueSpan, out ulong attributeId) + { + var attr = _attributesByType[attributeType]; + attributeId = _dbAttributesByUniqueId[attr.Id].AttrEntityId; + var serializer = _valueSerializersByNativeType[attr.ValueType]; + var readMethod = serializer.GetType().GetMethod("Read")!; + var valueExpr = Expression.Parameter(attr.ValueType, "retVal"); + var readExpression = Expression.Call(Expression.Constant(serializer), readMethod, valueSpan, valueExpr); + return Expression.Block([valueExpr], readExpression, valueExpr); } } diff --git a/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs new file mode 100644 index 00000000..34db9379 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/EntityIdSerializer.cs @@ -0,0 +1,31 @@ +using System; +using System.Buffers; +using System.Buffers.Binary; +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.DatomStore.BuiltInSerializers; + +public class EntityIdSerialzer : IValueSerializer +{ + public Type NativeType => typeof(EntityId); + + public static readonly UInt128 Id = "E2C3185E-C082-4641-B25E-7CEC803A2F48".ToUInt128Guid(); + public UInt128 UniqueId => Id; + public int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + return BinaryPrimitives.ReadUInt64LittleEndian(a).CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(b)); + } + + public void Write(EntityId value, TWriter buffer) where TWriter : IBufferWriter + { + var span = buffer.GetSpan(8); + BinaryPrimitives.WriteUInt64LittleEndian(span, value.Value); + buffer.Advance(8); + } + + public int Read(ReadOnlySpan buffer, out EntityId val) + { + val = EntityId.From(BinaryPrimitives.ReadUInt64LittleEndian(buffer)); + return 8; + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs new file mode 100644 index 00000000..77451c31 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/BuiltInSerializers/TxIdSerializer.cs @@ -0,0 +1,30 @@ +using System; +using System.Buffers; +using System.Buffers.Binary; +using NexusMods.EventSourcing.Abstractions; +namespace NexusMods.EventSourcing.DatomStore.BuiltInSerializers; + +public class TxIdSerializer : IValueSerializer +{ + public Type NativeType => typeof(TxId); + + public static readonly UInt128 Id = "BB2B2BAF-9AA8-4DB0-8BFC-A0A853ED9BA0".ToUInt128Guid(); + public UInt128 UniqueId => Id; + public int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + return BinaryPrimitives.ReadUInt64LittleEndian(a).CompareTo(BinaryPrimitives.ReadUInt64LittleEndian(b)); + } + + public void Write(TxId value, TWriter buffer) where TWriter : IBufferWriter + { + var span = buffer.GetSpan(8); + BinaryPrimitives.WriteUInt64LittleEndian(span, value.Value); + buffer.Advance(8); + } + + public int Read(ReadOnlySpan buffer, out TxId val) + { + val = TxId.From(BinaryPrimitives.ReadUInt64LittleEndian(buffer)); + return 8; + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs index c7d446bc..89f3c4a2 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AETVIndex.cs @@ -6,8 +6,9 @@ namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class AETVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "aetv") { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) +public class AETVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "aetv"), IComparatorIndex +{ + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareAttribute(a, b); @@ -18,7 +19,7 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs index 23c49dc3..a2293984 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AIndexDefinition.cs @@ -1,12 +1,22 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; using RocksDbSharp; namespace NexusMods.EventSourcing.DatomStore.Indexes; -public abstract class AIndexDefinition(AttributeRegistry registry, string columnFamilyName) : IDisposable +public interface IComparatorIndex where TOuter : IComparatorIndex { - protected readonly AttributeRegistry Registry = registry; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static abstract unsafe int Compare(AIndexDefinition instance, KeyHeader* a, uint aLength, + KeyHeader* b, uint bLength); +} + +public abstract class AIndexDefinition(AttributeRegistry registry, string columnFamilyName) +where TOuter : IComparatorIndex +{ + protected internal readonly AttributeRegistry Registry = registry; private ColumnFamilyOptions? _options; protected ColumnFamilyHandle? ColumnFamilyHandle; @@ -29,7 +39,7 @@ public void Init(RocksDb db) { unsafe { - return Compare((KeyHeader*)a, (uint)alen, (KeyHeader*)b, (uint)blen); + return TOuter.Compare(this, (KeyHeader*)a, (uint)alen, (KeyHeader*)b, (uint)blen); } }; _comparator = Native.Instance.rocksdb_comparator_create(IntPtr.Zero, _destructorDelegate, _comparatorDelegate, _nameDelegate); @@ -39,8 +49,6 @@ public void Init(RocksDb db) public string ColumnFamilyName { get; } = columnFamilyName; - public abstract unsafe int Compare(KeyHeader *a, uint aLength, KeyHeader *b, uint bLength); - public void Dispose() { if (_comparator != IntPtr.Zero) @@ -55,6 +63,12 @@ public void Dispose() _namePtr = IntPtr.Zero; } } + + public static unsafe int Compare(TOuter outer, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + { + throw new NotImplementedException(); + } + public void Put(WriteBatch batch, ReadOnlySpan span) { batch.Put(span, ReadOnlySpan.Empty, ColumnFamilyHandle); diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs new file mode 100644 index 00000000..1f7fc447 --- /dev/null +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/AVTEIndex.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using NexusMods.EventSourcing.Abstractions; +using Reloaded.Memory.Extensions; +using RocksDbSharp; + +namespace NexusMods.EventSourcing.DatomStore.Indexes; + +public class AVTEIndex(AttributeRegistry registry) : + AIndexDefinition(registry, "avte"), IComparatorIndex +{ + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + { + // Attribute, Value, TX, Entity + var cmp = KeyHeader.CompareAttribute(a, b); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareTx(a, b); + if (cmp != 0) return cmp; + cmp = KeyHeader.CompareEntity(a, b); + if (cmp != 0) return cmp; + return KeyHeader.CompareIsAssert(a, b); + } + + public unsafe struct AVTEIterator : IDisposable + { + private readonly KeyHeader* _key; + private KeyHeader* _current; + private UIntPtr _currentLength; + private readonly Iterator _iterator; + private readonly AttributeRegistry _registry; + private bool _needsSeek; + + public AVTEIterator(ulong txId, AttributeRegistry registry, AVTEIndex idx) + { + _registry = registry; + _iterator = idx.Db.NewIterator(idx.ColumnFamilyHandle); + _key = (KeyHeader*)Marshal.AllocHGlobal(KeyHeader.Size); + _key->Entity = ulong.MaxValue; + _key->AttributeId = ulong.MaxValue; + _key->Tx = txId; + _key->IsAssert = true; + _needsSeek = true; + } + + + public void Set() where TAttribute : IAttribute + { + _key->Entity = ulong.MaxValue; + _key->AttributeId = _registry.GetAttributeId(); + _needsSeek = true; + } + + public IDatom Current + { + get + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing Current"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadDatom(ref *_current, currentValue); + } + } + + public EntityId EntityId => EntityId.From(_current->Entity); + + public TValue GetValue() + where TAttribute : IAttribute + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing GetValue"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadValue(ref *_current, currentValue); + + } + + public ulong AttributeId + { + get + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing AttributeId"); + return _current->AttributeId; + } + } + + public ReadOnlySpan ValueSpan => _iterator.GetKeySpan().SliceFast(KeyHeader.Size); + + public bool Next() + { + if (_needsSeek) + { + _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); + _needsSeek = false; + } + else + { + _key->Entity = _current->Entity - 1; + _iterator.Prev(); + } + + while (true) + { + + if (!_iterator.Valid()) return false; + + _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); + + Debug.Assert(_currentLength < KeyHeader.Size, "Key length is less than KeyHeader.Size"); + + if (_current->AttributeId != _key->AttributeId) + return false; + + if (_current->Tx > _key->Tx) + { + _iterator.Prev(); + continue; + } + + if (_current->Entity > _key->Entity) + { + _iterator.Prev(); + continue; + } + + return true; + } + } + public void Dispose() + { + _iterator.Dispose(); + Marshal.FreeHGlobal((IntPtr)_key); + } + } +} diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs index dbd488db..e0f18969 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/EATVIndex.cs @@ -1,4 +1,6 @@ using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using NexusMods.EventSourcing.Abstractions; using Reloaded.Memory.Extensions; @@ -6,9 +8,10 @@ namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class EATVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "eatv") +public class EATVIndex(AttributeRegistry registry) : AIndexDefinition(registry, "eatv"), IComparatorIndex { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareEntity(a, b); @@ -19,7 +22,7 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } @@ -54,16 +57,15 @@ public bool MaxId(Ids.Partition partition, out ulong o) public unsafe struct EATVIterator : IEntityIterator, IDisposable { - private readonly EATVIndex _idx; - private KeyHeader* _key; + private readonly KeyHeader* _key; + private KeyHeader* _current; + private UIntPtr _currentLength; private readonly Iterator _iterator; - private readonly ulong _attrId; private readonly AttributeRegistry _registry; - private bool _justSet; + private bool _needsSeek; public EATVIterator(ulong txId, AttributeRegistry registry, EATVIndex idx) { - _idx = idx; _registry = registry; _iterator = idx.Db.NewIterator(idx.ColumnFamilyHandle); _key = (KeyHeader*)Marshal.AllocHGlobal(KeyHeader.Size); @@ -71,63 +73,86 @@ public EATVIterator(ulong txId, AttributeRegistry registry, EATVIndex idx) _key->AttributeId = ulong.MaxValue; _key->Tx = txId; _key->IsAssert = true; - _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); - _justSet = true; + _needsSeek = true; } - public void SetEntityId(EntityId entityId) + public void Set(EntityId entityId) { _key->Entity = entityId.Value; _key->AttributeId = ulong.MaxValue; - _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); - _justSet = true; + _needsSeek = true; } public IDatom Current { get { - var span = _iterator.GetKeySpan(); - var valueSpan = span.SliceFast(KeyHeader.Size); - var header = MemoryMarshal.AsRef(span); - return _registry.ReadDatom(ref header, valueSpan); + Debug.Assert(!_needsSeek, "Must call Next() before accessing Current"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadDatom(ref *_current, currentValue); } } - public void SetOn(TModel model) where TModel : IReadModel + public TValue GetValue() + where TAttribute : IAttribute { - var span = _iterator.GetKeySpan(); - var currentHeader = MemoryMarshal.AsRef(span); - var currentValue = span.SliceFast(KeyHeader.Size); - _registry.SetOn(model, ref currentHeader, currentValue); + Debug.Assert(!_needsSeek, "Must call Next() before accessing GetValue"); + var currentValue = new ReadOnlySpan((byte*)_current + KeyHeader.Size, (int)_currentLength - KeyHeader.Size); + return _registry.ReadValue(ref *_current, currentValue); + } + public ulong AttributeId + { + get + { + Debug.Assert(!_needsSeek, "Must call Next() before accessing AttributeId"); + return _current->AttributeId; + } + } + + public ReadOnlySpan ValueSpan => _iterator.GetKeySpan().SliceFast(KeyHeader.Size); + public bool Next() { - TOP: - if (!_justSet) + if (_needsSeek) { - _iterator.Prev(); + _iterator.SeekForPrev((byte*)_key, KeyHeader.Size); + _needsSeek = false; } else { - _justSet = false; + _key->AttributeId = _current->AttributeId - 1; + _iterator.Prev(); } - if (!_iterator.Valid()) return false; + while (true) + { + + if (!_iterator.Valid()) return false; - var current = _iterator.GetKeySpan(); - var currentHeader = MemoryMarshal.AsRef(current); + _current = (KeyHeader*)Native.Instance.rocksdb_iter_key(_iterator.Handle, out _currentLength); - if (currentHeader.Entity != _key->Entity) return false; + Debug.Assert(_currentLength < KeyHeader.Size, "Key length is less than KeyHeader.Size"); - if (currentHeader.IsRetraction) - { - _key->AttributeId = currentHeader.AttributeId - 1; - goto TOP; + if (_current->Entity != _key->Entity) + return false; + + if (_current->Tx > _key->Tx) + { + _iterator.Prev(); + continue; + } + + if (_current->AttributeId > _key->AttributeId) + { + _iterator.Prev(); + continue; + } + + return true; } - return true; } public void Dispose() { diff --git a/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs b/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs index da791237..8efe3a0f 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Indexes/TxIndex.cs @@ -1,8 +1,8 @@ namespace NexusMods.EventSourcing.DatomStore.Indexes; -public class TxIndex(AttributeRegistry registry) : AIndexDefinition(registry, "txLog") +public class TxIndex(AttributeRegistry registry) : AIndexDefinition(registry, "txLog"), IComparatorIndex { - public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) + public static unsafe int Compare(AIndexDefinition idx, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { // TX, Entity, Attribute, IsAssert, Value var cmp = KeyHeader.CompareTx(a, b); @@ -13,6 +13,6 @@ public override unsafe int Compare(KeyHeader* a, uint aLength, KeyHeader* b, uin if (cmp != 0) return cmp; cmp = KeyHeader.CompareIsAssert(a, b); if (cmp != 0) return cmp; - return KeyHeader.CompareValues(Registry, a, aLength, b, bLength); + return KeyHeader.CompareValues(idx.Registry, a, aLength, b, bLength); } } diff --git a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs index 887d73b4..db2256fa 100644 --- a/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs +++ b/src/NexusMods.EventSourcing.DatomStore/KeyHeader.cs @@ -49,22 +49,24 @@ public bool IsRetraction [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareEntity(KeyHeader* a, KeyHeader* b) { - if (a->Entity < b->Entity) return -1; - return a->Entity > b->Entity ? 1 : 0; + if (a->_entity < b->_entity) return -1; + return a->_entity > b->_entity ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareAttribute(KeyHeader* a, KeyHeader* b) { - if (a->AttributeId < b->AttributeId) return -1; - return a->AttributeId > b->AttributeId ? 1 : 0; + var aAttrId = a->_attrAndExtra & 0x7FFF; + var bAttrId = b->_attrAndExtra & 0x7FFF; + if (aAttrId < bAttrId) return -1; + return aAttrId > bAttrId ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareTx(KeyHeader* a, KeyHeader* b) { - if (a->Tx < b->Tx) return -1; - return a->Tx > b->Tx ? 1 : 0; + if (a->_tx < b->_tx) return -1; + return a->_tx > b->_tx ? 1 : 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -79,8 +81,16 @@ public static int CompareIsAssert(KeyHeader* a, KeyHeader* b) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareValues(AttributeRegistry registry, KeyHeader* a, uint aLength, KeyHeader* b, uint bLength) { - if (a->AttributeId < b->AttributeId) return 1; - if (a->AttributeId > b->AttributeId) return -1; + var aAttrId = a->_attrAndExtra & 0x7FFF; + var bAttrId = b->_attrAndExtra & 0x7FFF; + if (aAttrId < bAttrId) return -1; + if (aAttrId > bAttrId) + return 1; + + // Iterators will pass a value length of 0, and in that case we want to have the valueless key be the "largest" + // so the iterator will seek to the the entry with the value + if (aLength == Size || bLength == Size) + return aLength == bLength ? 0 : aLength == Size ? 1 : -1; var aVal = (byte*) a + Size; var bVal = (byte*) b + Size; diff --git a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs index bb6b40df..847dcbb0 100644 --- a/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs +++ b/src/NexusMods.EventSourcing.DatomStore/RocksDBDatomStore.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Runtime.InteropServices; using Microsoft.Extensions.Logging; using NexusMods.EventSourcing.Abstractions; @@ -19,10 +20,10 @@ public class RocksDBDatomStore : IDatomStore private readonly RocksDb _db; private readonly PooledMemoryBufferWriter _pooledWriter; private readonly AttributeRegistry _registry; - private readonly AIndexDefinition[] _indexes; private ulong _tx; private readonly AETVIndex _avetIndex; private readonly EATVIndex _eatvIndex; + private readonly AVTEIndex _avteIndex; public RocksDBDatomStore(ILogger logger, AttributeRegistry registry, DatomStoreSettings settings) { @@ -39,18 +40,11 @@ public RocksDBDatomStore(ILogger logger, AttributeRegistry re _db = RocksDb.Open(_options, _settings.Path.ToString(), new ColumnFamilies()); _eatvIndex = new EATVIndex(_registry); + _eatvIndex.Init(_db); _avetIndex = new AETVIndex(_registry); - _indexes = - [ - //new TxIndex(_registry), - _eatvIndex, - _avetIndex, - ]; - - foreach (var index in _indexes) - { - index.Init(_db); - } + _avetIndex.Init(_db); + _avteIndex = new AVTEIndex(_registry); + _avteIndex.Init(_db); _pooledWriter = new PooledMemoryBufferWriter(128); @@ -75,10 +69,9 @@ private void Serialize(WriteBatch batch, ulong e, TVal val, ulong t _registry.WriteValue(val, in _pooledWriter); var span = _pooledWriter.GetWrittenSpan(); - foreach (var index in _indexes) - { - index.Put(batch, span); - } + _eatvIndex.Put(batch, span); + _avetIndex.Put(batch, span); + _avteIndex.Put(batch, span); } private struct TransactSink(RocksDBDatomStore store, WriteBatch batch, ulong tx) : IDatomSink @@ -136,6 +129,21 @@ public void RegisterAttributes(IEnumerable newAttrs) _registry.Populate(newAttrs.ToArray()); } + public Expression GetValueReadExpression(Type attribute, Expression valueSpan, out ulong attributeId) + { + return _registry.GetReadExpression(attribute, valueSpan, out attributeId); + } + + public IEnumerable ReverseLookup(TxId txId) where TAttribute : IAttribute + { + using var iterator = new AVTEIndex.AVTEIterator(txId.Value, _registry, _avteIndex); + iterator.Set(); + while (iterator.Next()) + { + yield return iterator.EntityId; + } + } + public void Dispose() { _db.Dispose(); diff --git a/src/NexusMods.EventSourcing.DatomStore/Services.cs b/src/NexusMods.EventSourcing.DatomStore/Services.cs index 90f133cc..dab1e318 100644 --- a/src/NexusMods.EventSourcing.DatomStore/Services.cs +++ b/src/NexusMods.EventSourcing.DatomStore/Services.cs @@ -27,7 +27,9 @@ public static IServiceCollection AddDatomStore(this IServiceCollection services) .AddValueSerializer() .AddValueSerializer() .AddValueSerializer() - .AddValueSerializer(); + .AddValueSerializer() + .AddValueSerializer() + .AddValueSerializer(); return services; } diff --git a/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs b/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs index c37f5276..14927af1 100644 --- a/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs +++ b/src/NexusMods.EventSourcing.SourceGenerator/AttributeData.cs @@ -9,6 +9,8 @@ public class AttributeData public string Description { get; set; } = ""; public string Namespace { get; set; } = ""; public string Entity { get; set; } = ""; + + public bool IsInclude { get; set; } = false; } public class AttributeGroup diff --git a/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs b/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs index a8220d17..19f6ab2e 100644 --- a/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs +++ b/src/NexusMods.EventSourcing.SourceGenerator/ModelGenerator.cs @@ -83,6 +83,7 @@ private string GenerateSource(IEnumerable attributeData) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; sb.ClassComment(attribute.Description); var withoutQuotes = attribute.Name.Replace("\"", ""); var uniqueName = $"{attribute.Namespace}.{attribute.Entity}/{withoutQuotes}"; @@ -103,6 +104,7 @@ private string GenerateSource(IEnumerable attributeData) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"services.AddAttribute<{withoutQuotes}>();"); } @@ -137,6 +139,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.ClassComment(attribute.Description); sb.Line($"public {attribute.AttributeType} {withoutQuotes} {{get; private set; }} = default!;"); @@ -151,6 +154,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) sb.Line("{"); foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"case {group.Namespace}.{group.Entity}.{withoutQuotes} a:"); sb.Line("{"); @@ -183,6 +187,7 @@ private void EmitReadModel(AttributeGroup group, CodeWriter sb) sb.Line("public Type[] Attributes => new Type[] {"); foreach (var attribute in group.Attributes) { + if (attribute.IsInclude) continue; var withoutQuotes = attribute.Name.Replace("\"", ""); sb.Line($"typeof({group.Namespace}.{group.Entity}.{withoutQuotes}),"); } @@ -217,43 +222,8 @@ private static IEnumerable FindAttributes(Compilation compilation var builderExpression = initializer.Value as InvocationExpressionSyntax; if (builderExpression != null) { - var defineCalls = builderExpression.DescendantNodesAndSelf() - .OfType() - .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax - { - Name.Identifier.Text: "Define" - }); - - foreach (var defineCall in defineCalls) - { - var genericDefineCall = - ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; - if (genericDefineCall == null) - { - continue; - } - - var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); - var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; - Console.WriteLine($"Type: {typeSymbol}"); - - - var args = defineCall.ArgumentList.Arguments.Select(argument => - { - var name = argument.Expression as LiteralExpressionSyntax; - - return name; - }).ToArray(); - - foundAttributes.Add(new AttributeData - { - Name = args[0]!.Token.ValueText, - Entity = declaredSymbol!.Name, - AttributeType = typeSymbol?.ToString() ?? "", - Description = args[1]!.Token.ValueText, - Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "" - }); - } + ExtractDefines(builderExpression, semanticModel, foundAttributes, declaredSymbol); + ExtractIncludes(builderExpression, semanticModel, foundAttributes, declaredSymbol); } } } @@ -263,4 +233,80 @@ private static IEnumerable FindAttributes(Compilation compilation return foundAttributes; } + + private static void ExtractDefines(InvocationExpressionSyntax builderExpression, SemanticModel semanticModel, + List foundAttributes, ISymbol? declaredSymbol) + { + var defineCalls = builderExpression.DescendantNodesAndSelf() + .OfType() + .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax + { + Name.Identifier.Text: "Define" + }); + + foreach (var defineCall in defineCalls) + { + var genericDefineCall = + ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; + if (genericDefineCall == null) + { + continue; + } + + var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); + var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; + + var args = defineCall.ArgumentList.Arguments.Select(argument => + { + var name = argument.Expression as LiteralExpressionSyntax; + + return name; + }).ToArray(); + + foundAttributes.Add(new AttributeData + { + Name = args[0]!.Token.ValueText, + Entity = declaredSymbol!.Name, + AttributeType = typeSymbol?.ToString() ?? "", + Description = args[1]!.Token.ValueText, + Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "" + }); + } + } + + private static void ExtractIncludes(InvocationExpressionSyntax builderExpression, SemanticModel semanticModel, + List foundAttributes, ISymbol? declaredSymbol) + { + var defineCalls = builderExpression.DescendantNodesAndSelf() + .OfType() + .Where(invocation => invocation.Expression is MemberAccessExpressionSyntax + { + Name.Identifier.Text: "Include" + }); + + foreach (var defineCall in defineCalls) + { + var genericDefineCall = + ((MemberAccessExpressionSyntax)defineCall.Expression).Name as GenericNameSyntax; + if (genericDefineCall == null) + { + continue; + } + + var typeArgument = genericDefineCall.TypeArgumentList.Arguments.First(); + var typeSymbol = semanticModel.GetSymbolInfo(typeArgument).Symbol; + + + foundAttributes.Add(new AttributeData + { + Name = typeArgument.ToString(), + AttributeType = typeSymbol?.ToString() ?? "", + IsInclude = true, + Namespace = declaredSymbol?.ContainingNamespace.ToString() ?? "", + Entity = declaredSymbol!.Name, + Description = "" + }); + + } + } } diff --git a/src/NexusMods.EventSourcing/CommitResult.cs b/src/NexusMods.EventSourcing/CommitResult.cs index 48aa0c80..7adf0cbc 100644 --- a/src/NexusMods.EventSourcing/CommitResult.cs +++ b/src/NexusMods.EventSourcing/CommitResult.cs @@ -4,7 +4,7 @@ namespace NexusMods.EventSourcing; /// -public class CommitResult(TxId newTxId, IDictionary remaps) : ICommitResult +public class CommitResult(TxId newTxId, IDictionary remaps, Connection connection, IDatom[] datoms) : ICommitResult { /// public EntityId this[EntityId id] => @@ -12,4 +12,7 @@ public class CommitResult(TxId newTxId, IDictionary remaps) : ICom /// public TxId NewTx => newTxId; + + /// + public IEnumerable Datoms => datoms; } diff --git a/src/NexusMods.EventSourcing/Connection.cs b/src/NexusMods.EventSourcing/Connection.cs index 6a9065d5..0aea6152 100644 --- a/src/NexusMods.EventSourcing/Connection.cs +++ b/src/NexusMods.EventSourcing/Connection.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Reactive.Subjects; using NexusMods.EventSourcing.Abstractions; using NexusMods.EventSourcing.DatomStore; @@ -16,16 +17,19 @@ public class Connection : IConnection private ulong _nextEntityId = Ids.MinId(Ids.Partition.Entity); private readonly IDatomStore _store; private readonly IAttribute[] _declaredAttributes; - private readonly Dictionary _factories; + internal readonly ModelReflector ModelReflector; + private readonly Subject _updates; /// /// Main connection class, co-ordinates writes and immutable reads /// - public Connection(IDatomStore store, IEnumerable declaredAttributes, IEnumerable serializers, IEnumerable factories) + public Connection(IDatomStore store, IEnumerable declaredAttributes, IEnumerable serializers) { _store = store; _declaredAttributes = declaredAttributes.ToArray(); - _factories = factories.ToDictionary(f => f.ModelType); + ModelReflector = new ModelReflector(store); + + _updates = new Subject(); AddMissingAttributes(serializers); } @@ -68,7 +72,7 @@ private IEnumerable ExistingAttributes() var entIterator = _store.EntityIterator(tx); while (attrIterator.Next()) { - entIterator.SetEntityId(attrIterator.EntityId); + entIterator.Set(attrIterator.EntityId); var serializerId = UInt128.Zero; Symbol uniqueId = null!; @@ -92,7 +96,7 @@ private IEnumerable ExistingAttributes() /// - public IDb Db => new Db(_store, this, TxId, _factories); + public IDb Db => new Db(_store, this, TxId); /// @@ -103,33 +107,36 @@ private IEnumerable ExistingAttributes() public ICommitResult Transact(IEnumerable datoms) { var remaps = new Dictionary(); + var datomsArray = datoms.ToArray(); - lock (_lock) + EntityId RemapFn(EntityId input) { - var newDatoms = new List(); - foreach (var datom in datoms) + if (Ids.GetPartition(input) == Ids.Partition.Tmp) { - var eid = datom.E; - if (Ids.GetPartition(eid) == Ids.Partition.Tmp) + if (!remaps.TryGetValue(input.Value, out var id)) { - if (!remaps.TryGetValue(eid, out var id)) - { - var newId = _nextEntityId++; - remaps[eid] = newId; - newDatoms.Add(datom.RemapEntityId(newId)); - } - else - { - newDatoms.Add(datom.RemapEntityId(id)); - } - } - else - { - newDatoms.Add(datom); + var newId = _nextEntityId++; + remaps[input.Value] = newId; + return EntityId.From(newId); } + return EntityId.From(id); + } + return input; + } + + lock (_lock) + { + var newDatoms = new List(); + foreach (var datom in datomsArray) + { + datom.Remap(RemapFn); + newDatoms.Add(datom); } var newTx = _store.Transact(newDatoms); - return new CommitResult(newTx, remaps); + TxId = newTx; + var result = new CommitResult(newTx, remaps, this, datomsArray); + _updates.OnNext(result); + return result; } } @@ -138,4 +145,7 @@ public ITransaction BeginTransaction() { return new Transaction(this); } + + /// + public IObservable Commits => _updates; } diff --git a/src/NexusMods.EventSourcing/Db.cs b/src/NexusMods.EventSourcing/Db.cs index 4da096fb..a858431c 100644 --- a/src/NexusMods.EventSourcing/Db.cs +++ b/src/NexusMods.EventSourcing/Db.cs @@ -2,10 +2,11 @@ using System.Collections.Generic; using System.Linq; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing; -public class Db(IDatomStore store, IConnection connection, TxId txId, IDictionary factories) : IDb +public class Db(IDatomStore store, Connection connection, TxId txId) : IDb { public TxId BasisTxId => txId; @@ -22,20 +23,35 @@ public IIterator Where(EntityId id) public IEnumerable Get(IEnumerable ids) where TModel : IReadModel { - var factory = factories[typeof(TModel)]; + using var iterator = store.EntityIterator(txId); + var reader = connection.ModelReflector.GetReader(); + foreach (var id in ids) + { + iterator.Set(id); + var model = reader(id, iterator, this); + yield return model; + } + } - var iterator = store.EntityIterator(txId); + public TModel Get(EntityId id) where TModel : IReadModel + { + using var iterator = store.EntityIterator(txId); + iterator.Set(id); + var reader = connection.ModelReflector.GetReader(); + return reader(id, iterator, this); + } - foreach (var id in ids) + /// + public IEnumerable GetReverse(EntityId id) where TAttribute : IAttribute where TModel : IReadModel + { + var iterator = store.ReverseLookup(txId); + using var entityIterator = store.EntityIterator(txId); + var reader = connection.ModelReflector.GetReader(); + foreach (var entityId in iterator) { - var readModel = (TModel)factory.Create(id); - iterator.SetEntityId(id); - while (iterator.Next()) - { - iterator.SetOn(readModel); - } - - yield return readModel; + entityIterator.Set(entityId); + var model = reader(entityId, entityIterator, this); + yield return model; } } } diff --git a/src/NexusMods.EventSourcing/ModelReflector.cs b/src/NexusMods.EventSourcing/ModelReflector.cs new file mode 100644 index 00000000..34066aaf --- /dev/null +++ b/src/NexusMods.EventSourcing/ModelReflector.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; + +namespace NexusMods.EventSourcing; + +/// +/// Reflects over models and creates reader/writer functions for them. +/// +internal class ModelReflector(IDatomStore store) + where TTransaction : ITransaction +{ + private readonly ConcurrentDictionary _emitters = new(); + private readonly ConcurrentDictionary _readers = new(); + + private delegate void EmitterFn(TTransaction tx, TReadModel model) + where TReadModel : IReadModel; + + internal delegate TReadModel ReaderFn(EntityId id, IEntityIterator iterator, IDb db) + where TReadModel : IReadModel; + + public void Add(TTransaction tx, IReadModel model) + { + EmitterFn emitterFn; + var modelType = model.GetType(); + if (!_emitters.TryGetValue(model.GetType(), out var found)) + { + emitterFn = CreateEmitter(modelType); + _emitters.TryAdd(modelType, emitterFn); + } + else + { + emitterFn = (EmitterFn)found; + } + emitterFn(tx, model); + } + + /// + /// Reflects over + /// + /// + /// + private EmitterFn CreateEmitter(Type readModel) + { + var properties = GetModelProperties(readModel); + + var entityParameter = Expression.Parameter(typeof(IReadModel), "entity"); + var txParameter = Expression.Parameter(typeof(TTransaction), "tx"); + + var exprs = new List(); + var idVariable = Expression.Variable(typeof(EntityId), "entityId"); + var castedVariable = Expression.Variable(readModel, "castedEntity"); + + exprs.Add(Expression.Assign(castedVariable, Expression.Convert(entityParameter, readModel))); + exprs.Add(Expression.Assign(idVariable, Expression.Property(entityParameter, "Id"))); + + exprs.AddRange(from property in properties + let method = property.Attribute.GetMethod("Add", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)! + let value = Expression.Property(castedVariable, property.Property) + select Expression.Call(null, method, txParameter, idVariable, value)); + + var blockExpr = Expression.Block(new[] { idVariable, castedVariable}, exprs); + + var lambda = Expression.Lambda>(blockExpr, txParameter, entityParameter); + return lambda.Compile(); + } + + private static IEnumerable<(Type Attribute, PropertyInfo Property)> GetModelProperties(Type readModel) + { + var properties = readModel + .GetProperties(BindingFlags.Public | BindingFlags.Instance | BindingFlags.FlattenHierarchy) + .Where(p => p.GetCustomAttributes(typeof(FromAttribute<>), true).Any()) + .Select(p => ( + (p.GetCustomAttributes(typeof(FromAttribute<>), true).First() as IFromAttribute)!.AttributeType, + p)) + .ToList(); + return properties; + } + + public ReaderFn GetReader() where TModel : IReadModel + { + var modelType = typeof(TModel); + if (_readers.TryGetValue(modelType, out var found)) + return (ReaderFn)found; + + var readerFn = MakeReader(); + _readers.TryAdd(modelType, readerFn); + return readerFn; + } + + public ReaderFn MakeReader() where TModel : IReadModel + { + var properties = GetModelProperties(typeof(TModel)); + + var exprs = new List(); + + var whileTopLabel = Expression.Label("whileTop"); + var exitLabel = Expression.Label("exit"); + + + + + var entityIdParameter = Expression.Parameter(typeof(EntityId), "entityId"); + var iteratorParameter = Expression.Parameter(typeof(IEntityIterator), "iterator"); + var dbParameter = Expression.Parameter(typeof(IDb), "db"); + + var newModelExpr = Expression.Variable(typeof(TModel), "newModel"); + + var spanExpr = Expression.Property(iteratorParameter, "ValueSpan"); + var ctor = typeof(TModel).GetConstructor([typeof(ITransaction)])!; + + + exprs.Add(Expression.Assign(newModelExpr, Expression.New(ctor, Expression.Constant(null, typeof(ITransaction))))); + exprs.Add(Expression.Assign(Expression.Property(newModelExpr, "Id"), entityIdParameter)); + exprs.Add(Expression.Assign(Expression.Property(newModelExpr, "Db"), dbParameter)); + + exprs.Add(Expression.Label(whileTopLabel)); + exprs.Add(Expression.IfThen( + Expression.Not(Expression.Call(iteratorParameter, typeof(IEntityIterator).GetMethod("Next")!)), + Expression.Break(exitLabel))); + + var cases = new List(); + + foreach (var (attribute, property) in properties) + { + var readSpanExpr = store.GetValueReadExpression(attribute, spanExpr, out var attributeId); + + var assigned = Expression.Assign(Expression.Property(newModelExpr, property), readSpanExpr); + + cases.Add(Expression.SwitchCase(Expression.Block([assigned, Expression.Goto(whileTopLabel)]), + Expression.Constant(attributeId))); + } + + exprs.Add(Expression.Switch(Expression.Property(iteratorParameter, "AttributeId"), cases.ToArray())); + + exprs.Add(Expression.Goto(whileTopLabel)); + exprs.Add(Expression.Label(exitLabel)); + exprs.Add(newModelExpr); + + var block = Expression.Block(new[] {newModelExpr}, exprs); + + var lambda = Expression.Lambda>(block, entityIdParameter, iteratorParameter, dbParameter); + return lambda.Compile(); + } +} diff --git a/src/NexusMods.EventSourcing/Transaction.cs b/src/NexusMods.EventSourcing/Transaction.cs index c5c29eb2..b4a1214a 100644 --- a/src/NexusMods.EventSourcing/Transaction.cs +++ b/src/NexusMods.EventSourcing/Transaction.cs @@ -2,13 +2,15 @@ using System.Collections.Concurrent; using System.Threading; using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; namespace NexusMods.EventSourcing; public class Transaction(Connection connection) : ITransaction { - private ulong _tempId = Ids.MinId(Ids.Partition.Tmp); + private ulong _tempId = Ids.MinId(Ids.Partition.Tmp) + 1; private ConcurrentBag _datoms = new(); + private ConcurrentBag _models = new(); /// public EntityId TempId() @@ -21,8 +23,28 @@ public void Add(IDatom datom) _datoms.Add(datom); } + /// + public TReadModel Add(TReadModel model) + where TReadModel : IReadModel + { + _models.Add(model); + return model; + } + + public void Add(EntityId entityId, TVal val, bool isAssert = true) where TAttribute : IAttribute + { + _datoms.Add(new AssertDatom(entityId.Value, val)); + } + public ICommitResult Commit() { + foreach (var model in _models) + { + connection.ModelReflector.Add(this, model); + } return connection.Transact(_datoms); } + + /// + public TxId ThisTxId => TxId.From(Ids.MinId(Ids.Partition.Tmp)); } diff --git a/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs b/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs index bb86fb1f..5dbcdfb2 100644 --- a/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs +++ b/tests/NexusMods.EventSourcing.DatomStore.Tests/ADatomStoreTest.cs @@ -12,7 +12,7 @@ public abstract class ADatomStoreTest : IDisposable protected readonly RocksDBDatomStore Store; protected readonly Connection Connection; - protected ADatomStoreTest(IEnumerable valueSerializers, IEnumerable attributes, IEnumerable factories) + protected ADatomStoreTest(IEnumerable valueSerializers, IEnumerable attributes) { _tmpPath = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb"); var dbSettings = new DatomStoreSettings() @@ -21,7 +21,7 @@ protected ADatomStoreTest(IEnumerable valueSerializers, IEnume }; _registry = new AttributeRegistry(valueSerializers, attributes); Store = new RocksDBDatomStore(new NullLogger(), _registry, dbSettings); - Connection = new Connection(Store, attributes, valueSerializers, factories); + Connection = new Connection(Store, attributes, valueSerializers); } public void Dispose() diff --git a/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs b/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs index af3f34c6..0d6200c2 100644 --- a/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs +++ b/tests/NexusMods.EventSourcing.DatomStore.Tests/DatomStoreSetupTests.cs @@ -4,9 +4,8 @@ namespace NexusMods.EventSourcing.DatomStore.Tests; public class DatomStoreSetupTests : ADatomStoreTest { - public DatomStoreSetupTests(IEnumerable valueSerializers, IEnumerable attributes, - IEnumerable factories) - : base(valueSerializers, attributes, factories) + public DatomStoreSetupTests(IEnumerable valueSerializers, IEnumerable attributes) + : base(valueSerializers, attributes) { } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs new file mode 100644 index 00000000..82023fd5 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/ArchiveFile.cs @@ -0,0 +1,32 @@ +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; + +namespace NexusMods.EventSourcing.TestModel.Model; + +public class ArchiveFile(ITransaction? tx) : AReadModel(tx) +{ + /// + /// Base attribute + /// + [From] + public required string ModPath { get; init; } + + /// + /// Base attribute + /// + [From] + public required string Path { get; init; } + + /// + /// Example of "inheritance" of attributes from other namespaces + /// + [From] + public required ulong Hash { get; init; } + + /// + /// The index of the file in the archive used for debugging purposes + /// + [From] + public required ulong Index { get; init; } +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs new file mode 100644 index 00000000..42e5770d --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ArchiveFileAttributes.cs @@ -0,0 +1,17 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public static class ArchiveFileAttributes +{ + /// + /// Extra attribute with a different name + /// + public class ArchiveHash : ScalarAttribute; + + /// + /// Overlapping name with ModFileAttributes.Path + /// + public class Path : ScalarAttribute; + +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs new file mode 100644 index 00000000..084cdb4f --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/LoadoutAttributes.cs @@ -0,0 +1,16 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public class LoadoutAttributes +{ + /// + /// Name of the loadout + /// + public class Name : ScalarAttribute; + + /// + /// The last transaction that updated the loadout + /// + public class UpdatedTx : ScalarAttribute; +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs new file mode 100644 index 00000000..cd9ed563 --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModAttributes.cs @@ -0,0 +1,21 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public class ModAttributes +{ + /// + /// Name of the loadout + /// + public class Name : ScalarAttribute; + + /// + /// The last transaction that updated the loadout + /// + public class UpdatedTx : ScalarAttribute; + + /// + /// The id of the loadout this mod belongs to + /// + public class LoadoutId : ScalarAttribute; +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModFileAttributes.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModFileAttributes.cs new file mode 100644 index 00000000..c036f80d --- /dev/null +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Attributes/ModFileAttributes.cs @@ -0,0 +1,13 @@ +using NexusMods.EventSourcing.Abstractions; + +namespace NexusMods.EventSourcing.TestModel.Model.Attributes; + +public static class ModFileAttributes +{ + public class Hash : ScalarAttribute; + + public class Path : ScalarAttribute; + + public class Index : ScalarAttribute; + +} diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs index d57e6406..8dd28a42 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Model/File.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Model/File.cs @@ -1,14 +1,27 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; namespace NexusMods.EventSourcing.TestModel.Model; -[ModelDefinition] -public static partial class File + +public class File(ITransaction? tx) : AReadModel(tx) { - public static AttributeDefinitions Attributes = new AttributeDefinitionsBuilder() - .Define("Path", "The path of the file") - .Define("Hash", "The hash of the file") - .Define("Index", "A index value for testing purposes") - .Build(); + /// + /// Base attribute + /// + [From] + public required string Path { get; init; } + + /// + /// Example of "inheritance" of attributes from other namespaces + /// + [From] + public required ulong Hash { get; init; } + /// + /// The index of the file in the archive used for debugging purposes + /// + [From] + public required ulong Index { get; init; } } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs index 396b53c8..b86ce9ce 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Loadout.cs @@ -1,12 +1,50 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; namespace NexusMods.EventSourcing.TestModel.Model; -[ModelDefinition] -public static partial class Loadout +public class Loadout(ITransaction? tx) : AReadModel(tx) { - public static AttributeDefinitions AttributeDefinitions = new AttributeDefinitionsBuilder() - .Define("Name", "The name of the loadout") - .Build(); + /// + /// The name of the loadout. + /// + [From] + public required string Name { get; init; } + /// + /// The last tx that updated the loadout. + /// + [From] + public required TxId Invalidator { get; init; } + + /// + /// The mods in the loadout. + /// + public IEnumerable Mods => GetReverse(); + + + /// + /// Create a new loadout with the given name. + /// + /// + /// + /// + public static Loadout Create(ITransaction tx, string name) + { + return new Loadout(tx) + { + Name = name, + Invalidator = tx.ThisTxId + }; + } + + /// + /// Updates this loadout marking it as touched by the given transaction. + /// + /// + public void Touch(ITransaction tx) + { + LoadoutAttributes.UpdatedTx.Add(tx, Id, tx.ThisTxId); + } } diff --git a/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs b/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs index a4f19544..39304255 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Model/Mod.cs @@ -1,13 +1,37 @@ -using NexusMods.EventSourcing.Abstractions.ModelGeneration; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.Abstractions.Models; +using NexusMods.EventSourcing.TestModel.Model.Attributes; namespace NexusMods.EventSourcing.TestModel.Model; -[ModelDefinition] -public static partial class Mod +public class Mod(ITransaction? tx) : AReadModel(tx) { - public static AttributeDefinitions AttributeDefinitions = new AttributeDefinitionsBuilder() - .Define("Name", "The name of the mod") - .Define("Enabled", "Whether the mod is enabled") - .Build(); + [From] + public required string Name { get; init; } + + + [From] + public required EntityId LoadoutId { get; init; } + + /// + /// The loadout for this mod. + /// + public Loadout Loadout => Get(LoadoutId); + + + public static Mod Create(ITransaction tx, string name, EntityId loadoutId) + { + var mod = new Mod(tx) + { + Name = name, + LoadoutId = loadoutId + }; + return mod; + } + + public void Touch(ITransaction tx) + { + Loadout.Touch(tx); + } } diff --git a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj index fd36c77c..a4bb5538 100644 --- a/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj +++ b/tests/NexusMods.EventSourcing.TestModel/NexusMods.EventSourcing.TestModel.csproj @@ -8,7 +8,6 @@ - diff --git a/tests/NexusMods.EventSourcing.TestModel/Services.cs b/tests/NexusMods.EventSourcing.TestModel/Services.cs index 389d7529..72cf7667 100644 --- a/tests/NexusMods.EventSourcing.TestModel/Services.cs +++ b/tests/NexusMods.EventSourcing.TestModel/Services.cs @@ -1,12 +1,19 @@ using Microsoft.Extensions.DependencyInjection; -using NexusMods.EventSourcing.TestModel.Model; +using NexusMods.EventSourcing.Abstractions; +using NexusMods.EventSourcing.TestModel.Model.Attributes; -namespace NexusMods.EventSourcing.TestModel; public static class Services { public static IServiceCollection AddTestModel(this IServiceCollection services) => - services.AddModModel() - .AddFileModel() - .AddLoadoutModel(); + services.AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute() + .AddAttribute(); } diff --git a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs index 4605bc69..1b9c0696 100644 --- a/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs +++ b/tests/NexusMods.EventSourcing.Tests/AEventSourcingTest.cs @@ -13,7 +13,7 @@ public class AEventSourcingTest : IDisposable protected readonly Connection Connection; protected AEventSourcingTest(IEnumerable valueSerializers, - IEnumerable attributes, IEnumerable factories) + IEnumerable attributes) { _tmpPath = FileSystem.Shared.GetKnownPath(KnownPath.TempDirectory).Combine(Guid.NewGuid() + ".rocksdb"); var dbSettings = new DatomStoreSettings() @@ -22,7 +22,7 @@ protected AEventSourcingTest(IEnumerable valueSerializers, }; _registry = new AttributeRegistry(valueSerializers, attributes); Store = new RocksDBDatomStore(new NullLogger(), _registry, dbSettings); - Connection = new Connection(Store, attributes, valueSerializers, factories); + Connection = new Connection(Store, attributes, valueSerializers); } public void Dispose() diff --git a/tests/NexusMods.EventSourcing.Tests/DbTests.cs b/tests/NexusMods.EventSourcing.Tests/DbTests.cs index b8a6b530..fbc36c50 100644 --- a/tests/NexusMods.EventSourcing.Tests/DbTests.cs +++ b/tests/NexusMods.EventSourcing.Tests/DbTests.cs @@ -1,5 +1,6 @@ using NexusMods.EventSourcing.Abstractions; using NexusMods.EventSourcing.TestModel.Model; +using NexusMods.EventSourcing.TestModel.Model.Attributes; using File = NexusMods.EventSourcing.TestModel.Model.File; namespace NexusMods.EventSourcing.Tests; @@ -7,8 +8,8 @@ namespace NexusMods.EventSourcing.Tests; public class DbTests : AEventSourcingTest { - public DbTests(IEnumerable valueSerializers, IEnumerable attributes, IEnumerable factories) - : base(valueSerializers, attributes, factories) + public DbTests(IEnumerable valueSerializers, IEnumerable attributes) + : base(valueSerializers, attributes) { } @@ -21,13 +22,15 @@ public void ReadDatomsForEntity() var ids = new List(); - for (ulong i = 0; i < TOTAL_COUNT; i++) + for (ulong idx = 0; idx < TOTAL_COUNT; idx++) { - var fileId = tx.TempId(); - ids.Add(fileId); - File.Path.Assert(fileId, $"C:\\test_{i}.txt", tx); - File.Hash.Assert(fileId, i + 0xDEADBEEF, tx); - File.Index.Assert(fileId, i, tx); + var file = new File(tx) + { + Path = $"C:\\test_{idx}.txt", + Hash = idx + 0xDEADBEEF, + Index = idx + }; + ids.Add(file.Id); } var oldTx = Connection.TxId; @@ -36,7 +39,7 @@ public void ReadDatomsForEntity() result.NewTx.Value.Should().Be(oldTx.Value + 1, "transaction id should be incremented by 1"); var db = Connection.Db; - var resolved = db.Get(ids.Select(id => result[id])).ToArray(); + var resolved = db.Get(ids.Select(id => result[id])).ToArray(); resolved.Should().HaveCount(TOTAL_COUNT); foreach (var readModel in resolved) @@ -44,9 +47,11 @@ public void ReadDatomsForEntity() var idx = readModel.Index; readModel.Hash.Should().Be(idx + 0xDEADBEEF); readModel.Path.Should().Be($"C:\\test_{idx}.txt"); + readModel.Index.Should().Be(idx); } } + [Fact] public void DbIsImmutable() { @@ -54,18 +59,20 @@ public void DbIsImmutable() // Insert some data var tx = Connection.BeginTransaction(); - var fileId = tx.TempId(); - File.Path.Assert(fileId, "C:\\test.txt_mutate", tx); - File.Hash.Assert(fileId, 0xDEADBEEF, tx); - File.Index.Assert(fileId, 0, tx); + var file = new File(tx) + { + Path = "C:\\test.txt_mutate", + Hash = 0xDEADBEEF, + Index = 0 + }; var result = tx.Commit(); - var realId = result[fileId]; + var realId = result[file.Id]; var originalDb = Connection.Db; // Validate the data - var found = originalDb.Get([realId]).First(); + var found = originalDb.Get([realId]).First(); found.Path.Should().Be("C:\\test.txt_mutate"); found.Hash.Should().Be(0xDEADBEEF); found.Index.Should().Be(0); @@ -74,20 +81,121 @@ public void DbIsImmutable() for (var i = 0; i < TIMES; i++) { var newTx = Connection.BeginTransaction(); - File.Path.Assert(fileId, $"C:\\test_{i}.txt_mutate", newTx); + ModFileAttributes.Path.Add(newTx, realId, $"C:\\test_{i}.txt_mutate"); var newResult = newTx.Commit(); // Validate the data var newDb = Connection.Db; - var newId = newResult[fileId]; - var newFound = newDb.Get([newId]).First(); + newDb.BasisTxId.Value.Should().Be(originalDb.BasisTxId.Value + 1UL + (ulong)i, "transaction id should be incremented by 1 for each mutation"); + + var newFound = newDb.Get([realId]).First(); newFound.Path.Should().Be($"C:\\test_{i}.txt_mutate"); // Validate the original data - var orignalFound = originalDb.Get([realId]).First(); + var orignalFound = originalDb.Get([realId]).First(); orignalFound.Path.Should().Be("C:\\test.txt_mutate"); } } + + [Fact] + public void ReadModelsCanHaveExtraAttributes() + { + // Insert some data + var tx = Connection.BeginTransaction(); + var file = new File(tx) + { + Path = "C:\\test.txt", + Hash = 0xDEADBEEF, + Index = 77 + }; + // Attach extra attributes to the entity + ArchiveFileAttributes.Path.Add(tx, file.Id, "C:\\test.zip"); + ArchiveFileAttributes.ArchiveHash.Add(tx, file.Id, 0xFEEDBEEF); + var result = tx.Commit(); + + + var realId = result[file.Id]; + var db = Connection.Db; + // Original data exists + var readModel = db.Get([realId]).First(); + readModel.Path.Should().Be("C:\\test.txt"); + readModel.Hash.Should().Be(0xDEADBEEF); + readModel.Index.Should().Be(77); + + // Extra data exists and can be read with a different read model + var archiveReadModel = db.Get([realId]).First(); + archiveReadModel.ModPath.Should().Be("C:\\test.txt"); + archiveReadModel.Path.Should().Be("C:\\test.zip"); + archiveReadModel.Hash.Should().Be(0xFEEDBEEF); + archiveReadModel.Index.Should().Be(77); + } + + [Fact] + public void CanGetCommitUpdates() + { + List updates = new(); + + Connection.Commits.Subscribe(update => + { + updates.Add(update.Datoms.ToArray()); + }); + + var tx = Connection.BeginTransaction(); + var file = new File(tx) + { + Path = "C:\\test.txt", + Hash = 0xDEADBEEF, + Index = 77 + }; + var result = tx.Commit(); + + var realId = result[file.Id]; + + updates.Should().HaveCount(1); + + for (var idx = 0; idx < 10; idx++) + { + tx = Connection.BeginTransaction(); + ModFileAttributes.Index.Add(tx, realId, (ulong)idx); + result = tx.Commit(); + + result.Datoms.Should().BeEquivalentTo(updates[idx + 1]); + + updates.Should().HaveCount(idx + 2); + var updateDatom = updates[idx + 1].OfType>() + .First(); + updateDatom.V.Should().Be((ulong)idx); + } + + } + + [Fact] + public void CanGetChildEntities() + { + var tx = Connection.BeginTransaction(); + var loadout = Loadout.Create(tx, "Test Loadout"); + var mod1 = Mod.Create(tx, "Test Mod 1", loadout.Id); + var mod2 = Mod.Create(tx, "Test Mod 2", loadout.Id); + var result = tx.Commit(); + + var newDb = Connection.Db; + + loadout = newDb.Get([result[loadout.Id]]).First(); + + loadout.Mods.Count().Should().Be(2); + loadout.Mods.Select(m => m.Name).Should().BeEquivalentTo(["Test Mod 1", "Test Mod 2"]); + + var firstMod = loadout.Mods.First(); + Ids.IsPartition(firstMod.Loadout.Id.Value, Ids.Partition.Entity) + .Should() + .Be(true, "the temp id should be replaced with a real id"); + firstMod.Loadout.Id.Should().Be(loadout.Id); + firstMod.Db.Should().Be(newDb); + loadout.Name.Should().Be("Test Loadout"); + firstMod.Loadout.Name.Should().Be("Test Loadout"); + } + + }