Merge pull request #102 from Nexus-Mods/excision

Excision

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Changelog
 
+### 0.9.89 - 09/10/2024
+* Added support for historical databases. These are instances of `IDb` that contain all datoms, inserted, retracted, and historical.
+Can be useful for analytics or viewing the changes of an entity over time
+* Added excision support to the database that will allow for complete removal of datoms, including historical datoms.
+
 ### 0.9.87 - 08/10/2024
 * Added import/export functionality for the database.
 

src/NexusMods.MnemonicDB.Abstractions/Attributes/ULongAttribute.cs

@@ -0,0 +1,21 @@
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.Attributes;
+
+/// <summary>
+/// UInt64 attribute (ulong)
+/// </summary>
+public class ULongAttribute(string ns, string name) : ScalarAttribute<ulong, ulong>(ValueTags.UInt64, ns, name)
+{
+    /// <inheritdoc />
+    protected override ulong ToLowLevel(ulong value)
+    {
+        return value;
+    }
+
+    /// <inheritdoc />
+    protected override ulong FromLowLevel(ulong value, ValueTags tags, AttributeResolver resolver)
+    {
+        return value;
+    }
+}

src/NexusMods.MnemonicDB.Abstractions/BuiltInEntities/Transaction.cs

@@ -1,4 +1,5 @@
-using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
 using TimestampAttribute = NexusMods.MnemonicDB.Abstractions.Attributes.TimestampAttribute;
 
 namespace NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
@@ -15,4 +16,9 @@ public partial class Transaction : IModelDefinition
     /// The timestamp when the transaction was committed.
     /// </summary>
     public static readonly TimestampAttribute Timestamp = new(Namespace, "Timestamp");
+
+    /// <summary>
+    /// If this database is associated with a excision, this attribute will contain the number of datoms that were excised.
+    /// </summary>
+    public static readonly ULongAttribute ExcisedDatoms = new(Namespace, "ExcisedDatoms") { IsOptional = true };
 }

src/NexusMods.MnemonicDB.Abstractions/IConnection.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using NexusMods.MnemonicDB.Abstractions.IndexSegments;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 
@@ -66,6 +67,11 @@ public interface IConnection
     /// </summary>
     public IDb AsOf(TxId txId);
 
+    /// <summary>
+    /// Returns a snapshot of the database that contains all current and historical datoms.
+    /// </summary>
+    public IDb History();
+
     /// <summary>
     ///     Starts a new transaction.
     /// </summary>
@@ -76,4 +82,10 @@ public interface IConnection
     /// The analyzers that are available for this connection
     /// </summary>
     public IAnalyzer[] Analyzers { get; }
+
+    /// <summary>
+    /// Deletes the entities with the given ids, also deleting them from any historical indexes. Returns the total number
+    /// of datoms that were excised.
+    /// </summary>
+    public Task<ulong> Excise(EntityId[] entityIds);
 }

src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
+using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.IndexSegments;
 using NexusMods.MnemonicDB.Abstractions.TxFunctions;
 
@@ -60,4 +61,9 @@ public interface IDatomStore : IDisposable
     ///     Create a snapshot of the current state of the store.
     /// </summary>
     ISnapshot GetSnapshot();
+
+    /// <summary>
+    /// Deletes these datoms from any of the indexes they are in.
+    /// </summary>
+    ValueTask Excise(List<Datom> datomsToRemove);
 }

src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs

@@ -335,4 +335,19 @@ public static Datom Datom(EntityId e, AttributeId a, EntityId value, TxId id, bo
         return new Datom(data);
     }
 
+    /// <summary>
+    /// Creates a slice descriptor for the given entity range. IndexType must be either EAVTCurrent or EAVTHistory
+    /// </summary>
+    public static SliceDescriptor Create(IndexType indexType, EntityId eid)
+    {
+        if (indexType is not IndexType.EAVTCurrent and not IndexType.EAVTHistory)
+            throw new ArgumentException("IndexType must be EAVTCurrent or EAVTHistory", nameof(indexType));
+
+        return new SliceDescriptor
+        {
+            Index = indexType,
+            From = Datom(eid, AttributeId.Min, TxId.MinValue, false),
+            To = Datom(eid, AttributeId.Max, TxId.MaxValue, false)
+        };
+    }
 }

src/NexusMods.MnemonicDB/Connection.cs

@@ -7,8 +7,10 @@
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.IndexSegments;
+using NexusMods.MnemonicDB.Abstractions.Query;
 using NexusMods.MnemonicDB.Abstractions.TxFunctions;
 using ObservableExtensions = R3.ObservableExtensions;
 
@@ -115,6 +117,15 @@ public IDb AsOf(TxId txId)
         };
     }
 
+    /// <inheritdoc />
+    public IDb History()
+    {
+        return new Db(new HistorySnapshot(_store.GetSnapshot(), TxId, AttributeCache), TxId, AttributeCache)
+        {
+            Connection = this
+        };
+    }
+
     /// <inheritdoc />
     public ITransaction BeginTransaction()
     {
@@ -124,6 +135,51 @@ public ITransaction BeginTransaction()
     /// <inheritdoc />
     public IAnalyzer[] Analyzers => _analyzers;
 
+
+    /// <inheritdoc />
+    public async Task<ulong> Excise(EntityId[] entityIds)
+    {
+        // Retract all datoms for the given entity ids
+        var contextDb = Db;
+
+        List<Datom> datomsToRemove = new();
+        foreach (var entityId in entityIds)
+        {
+            var segment = contextDb.Datoms(entityId);
+            datomsToRemove.AddRange(segment);
+        }
+
+        {
+            using var tx = BeginTransaction();
+            foreach (var datom in datomsToRemove)
+            {
+                tx.Add(datom.Retract());
+            }
+            var results = await tx.Commit();
+            contextDb = results.Db;
+        }
+
+        // Now delete all the datoms from all indexes
+        datomsToRemove.Clear();
+
+        foreach (var entityId in entityIds)
+        {
+            var segment = contextDb.Datoms(SliceDescriptor.Create(IndexType.EAVTHistory, entityId));
+            datomsToRemove.AddRange(segment);
+        }
+
+        await _store.Excise(datomsToRemove);
+
+        {
+            using var tx = BeginTransaction();
+            tx.Add((EntityId)tx.ThisTxId.Value, Abstractions.BuiltInEntities.Transaction.ExcisedDatoms, (ulong)datomsToRemove.Count);
+            await tx.Commit();
+        }
+
+
+        return (ulong)datomsToRemove.Count;
+    }
+
     /// <inheritdoc />
     public IObservable<IDb> Revisions => _dbStream;
 

src/NexusMods.MnemonicDB/HistorySnapshot.cs

@@ -0,0 +1,60 @@
+using System.Collections.Generic;
+using System.Linq;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.IndexSegments;
+using NexusMods.MnemonicDB.Abstractions.Query;
+
+namespace NexusMods.MnemonicDB;
+
+/// <summary>
+/// This is a wrapper around snapshots that allows you to query the snapshot as of a specific transaction
+/// id, this requires merging two indexes together, and then the deduplication of the merged index (retractions
+/// removing assertions).
+/// </summary>
+internal class HistorySnapshot(ISnapshot inner, TxId asOfTxId, AttributeCache attributeCache) : ISnapshot
+{
+    public IndexSegment Datoms(SliceDescriptor descriptor)
+    {
+        // TODO: stop using IEnumerable and use IndexSegment directly
+        var current = inner.Datoms(descriptor with {Index = descriptor.Index.CurrentVariant()});
+        var history = inner.Datoms(descriptor with {Index = descriptor.Index.HistoryVariant()});
+        var comparatorFn = descriptor.Index.GetComparator();
+
+        using var builder = new IndexSegmentBuilder(attributeCache);
+
+        var merged = current.Merge(history,
+            (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent, dHistory));
+
+        foreach (var datom in merged)
+        {
+            builder.Add(datom);
+        }
+
+        return builder.Build();
+    }
+
+    public IEnumerable<IndexSegment> DatomsChunked(SliceDescriptor descriptor, int chunkSize)
+    {
+        // TODO: stop using IEnumerable and use IndexSegment directly
+        var current = inner.DatomsChunked(descriptor with {Index = descriptor.Index.CurrentVariant()}, chunkSize).SelectMany(c => c);
+        var history = inner.DatomsChunked(descriptor with {Index = descriptor.Index.HistoryVariant()}, chunkSize).SelectMany(c => c);
+        var comparatorFn = descriptor.Index.GetComparator();
+
+        using var builder = new IndexSegmentBuilder(attributeCache);
+
+        var merged = current.Merge(history,
+            (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent, dHistory));
+
+        foreach (var datom in merged)
+        {
+            builder.Add(datom);
+            if (builder.Count % chunkSize == 0)
+            {
+                yield return builder.Build();
+                builder.Reset();
+            }
+        }
+
+        yield return builder.Build();
+    }
+}

src/NexusMods.MnemonicDB/Storage/DatomStore.cs

@@ -200,6 +200,21 @@ public ISnapshot GetSnapshot()
         return _currentSnapshot!;
     }
 
+    public async ValueTask Excise(List<Datom> datomsToRemove)
+    {
+        var batch = _backend.CreateBatch();
+        foreach (var datom in datomsToRemove)
+        {
+            _eavtHistory.Delete(batch, datom);
+            _aevtHistory.Delete(batch, datom);
+            _vaetHistory.Delete(batch, datom);
+            _avetHistory.Delete(batch, datom);
+            _txLog.Delete(batch, datom);
+        }
+        batch.Commit();
+
+    }
+
     /// <inheritdoc />
     public void Dispose()
     {

tests/NexusMods.MnemonicDB.Tests/DbTests.cs

@@ -6,6 +6,7 @@
 using Microsoft.Extensions.Logging;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.Hashing.xxHash64;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
 using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
@@ -1137,4 +1138,61 @@ public async Task RefCountWorksWithObservables()
         // when it set the initial value
         await Verify(mods.Select(m => m.Name).Distinct());
     }
+
+    [Fact]
+    public async Task CanExciseEntities()
+    {
+        using var tx = Connection.BeginTransaction();
+        var l1 = new Loadout.New(tx)
+        {
+            Name = "Test Loadout 1"
+        };
+
+        var l2 = new Loadout.New(tx)
+        {
+            Name = "Test Loadout 2"
+        };
+
+        var results = await tx.Commit();
+
+        var l1RO = results.Remap(l1);
+        var l2RO = results.Remap(l2);
+
+        {
+            using var tx2 = Connection.BeginTransaction();
+            tx2.Add(l2RO, Loadout.Name, "Test Loadout 2 Updated");
+            await tx2.Commit();
+        }
+        l2RO = l2RO.Rebase();
+
+        l1RO.Name.Should().Be("Test Loadout 1");
+        l2RO.Name.Should().Be("Test Loadout 2 Updated");
+
+
+        var history = Connection.History();
+
+        history.Datoms(l2RO.Id)
+            .Resolved(Connection)
+            .OfType<StringAttribute.ReadDatom>()
+            .Select(d => (!d.IsRetract, d.V))
+            .Should()
+            .BeEquivalentTo([
+                (true, "Test Loadout 2"),
+                (false, "Test Loadout 2"),
+                (true, "Test Loadout 2 Updated")
+            ]);
+
+        await Connection.Excise([l2RO.Id]);
+
+        history = Connection.History();
+
+        history.Datoms(l2RO.Id)
+            .Should()
+            .BeEmpty();
+
+        history.Datoms(l1RO.Id)
+            .Should()
+            .NotBeEmpty();
+
+    }
 }