Skip to content

Commit

Permalink
Merge pull request #26 from Nexus-Mods/support-retractions
Browse files Browse the repository at this point in the history
Adds support for retractions
  • Loading branch information
halgari authored Apr 1, 2024
2 parents 3221ae4 + 2a8a6c1 commit 1b54eb4
Show file tree
Hide file tree
Showing 25 changed files with 190 additions and 75 deletions.
2 changes: 1 addition & 1 deletion benchmarks/OneBillionDatomsTest/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
var tasks = new List<Task<(TimeSpan Time, int loaded)>>();


for (int i = 0; i < 4; i++)
for (int i = 0; i < 1; i++)
{
var task = Task.Run(() =>
{
Expand Down
8 changes: 8 additions & 0 deletions src/NexusMods.MneumonicDB.Abstractions/IAttribute.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,14 @@ public interface IAttribute<TVal> : IAttribute
/// <returns></returns>
public static abstract IWriteDatom Assert(EntityId e, TVal v);

/// <summary>
/// Construct a new write Datom for the retraction of the given entity and value
/// </summary>
/// <param name="e"></param>
/// <param name="v"></param>
/// <returns></returns>
public static abstract IWriteDatom Retract(EntityId e, TVal v);

/// <summary>
/// Gets the serializer for the attribute
/// </summary>
Expand Down
47 changes: 0 additions & 47 deletions src/NexusMods.MneumonicDB.Abstractions/IDatomStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,53 +40,6 @@ public interface IDatomStore : IDisposable
/// <param name="newAttrs"></param>
Task RegisterAttributes(IEnumerable<DbAttribute> newAttrs);

/// <summary>
/// Gets the entities that have the given attribute that reference the given entity id.
/// </summary>
IEnumerable<EntityId> GetReferencesToEntityThroughAttribute<TAttribute>(EntityId id, TxId txId)
where TAttribute : IAttribute<EntityId>;

/// <summary>
/// Gets the value of the given attribute for the given entity id where the transaction id exactly matches the given
/// txId.
/// </summary>
bool TryGetExact<TAttr, TValue>(EntityId e, TxId tx, out TValue val) where TAttr : IAttribute<TValue>;

/// <summary>
/// Gets the latest value of the given attribute for the given entity id where the transaction id is less than or equal
/// to the given txId.
/// </summary>
bool TryGetLatest<TAttribute, TValue>(EntityId e, TxId tx, out TValue value) where TAttribute : IAttribute<TValue>;

/// <summary>
/// Gets all the entities that have the given attribute.
/// </summary>
IEnumerable<EntityId> GetEntitiesWithAttribute<TAttribute>(TxId tx) where TAttribute : IAttribute;

/// <summary>
/// Gets all the attributes for the given entity id where the transaction id is less than or equal to the given txId.
/// </summary>
IEnumerable<IReadDatom> GetAttributesForEntity(EntityId realId, TxId txId);

/// <summary>
/// Gets the maximum entity id in the store.
/// </summary>
EntityId GetMaxEntityId();

/// <summary>
/// Gets the type of the read datom for the given attribute.
/// </summary>
Type GetReadDatomType(Type attribute);


/// <summary>
/// Get all the datoms in a given index, not super useful as this may return a TOOON of datoms.
/// </summary>
/// <param name="snapshot"></param>
/// <param name="type"></param>
/// <returns></returns>
public IEnumerable<IReadDatom> Datoms(ISnapshot snapshot, IndexType type);

/// <summary>
/// Create a snapshot of the current state of the store.
/// </summary>
Expand Down
20 changes: 18 additions & 2 deletions src/NexusMods.MneumonicDB.Abstractions/ScalarAttribute.cs
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,18 @@ public static IWriteDatom Assert(EntityId e, TValueType v)
return new WriteDatom
{
E = e,
V = v
V = v,
IsRetract = false
};
}

public static IWriteDatom Retract(EntityId e, TValueType v)
{
return new WriteDatom
{
E = e,
V = v,
IsRetract = true
};
}

Expand Down Expand Up @@ -146,11 +157,16 @@ public TValueType Read(ReadOnlySpan<byte> buffer)
/// </summary>
public required EntityId E { get; init; }

/// <summary>
/// True if this is a retraction
/// </summary>
public required bool IsRetract { get; init; }

public void Explode<TWriter>(IAttributeRegistry registry, Func<EntityId, EntityId> remapFn,
out EntityId e, out AttributeId a, TWriter vWriter, out bool isRetract)
where TWriter : IBufferWriter<byte>
{
isRetract = false;
isRetract = IsRetract;
e = EntityId.From(Ids.IsPartition(E.Value, Ids.Partition.Tmp) ? remapFn(E).Value : E.Value);

if (V is EntityId id)
Expand Down
56 changes: 36 additions & 20 deletions src/NexusMods.MneumonicDB.Storage/DatomStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -201,15 +201,6 @@ public ISnapshot GetSnapshot()
return _backend.GetSnapshot();
}


public IEnumerable<IReadDatom> Datoms(ISnapshot snapshot, IndexType type)
{
using var source = snapshot.GetIterator(type);
var iter = source.SeekStart();
foreach (var datom in iter.Resolve())
yield return datom;
}

public void Dispose()
{
_updatesSubject.Dispose();
Expand Down Expand Up @@ -326,7 +317,7 @@ private EntityId MaybeRemap(EntityId id, Dictionary<EntityId, EntityId> remaps,
}


private void Log(PendingTransaction pendingTransaction, out StoreResult result)
private unsafe void Log(PendingTransaction pendingTransaction, out StoreResult result)
{
var thisTx = TxId.From(_asOfTxId.Value + 1);

Expand All @@ -345,9 +336,9 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
}

var isRemapped = Ids.IsPartition(datom.E.Value, Ids.Partition.Tmp);
datom.Explode(_registry, remapFn, out var e, out var a, _writer, out var isAssert);
datom.Explode(_registry, remapFn, out var e, out var a, _writer, out var isRetract);
var keyPrefix = _writer.GetWrittenSpanWritable().CastFast<byte, KeyPrefix>();
keyPrefix[0].Set(e, a, thisTx, isAssert);
keyPrefix[0].Set(e, a, thisTx, isRetract);

var attr = _registry.GetAttribute(a);
var isReference = attr.IsReference;
Expand All @@ -357,8 +348,19 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
if (!isRemapped)
havePrevious = GetPrevious(keyPrefix[0]);

// Put it in the tx log first
var newSpan = _writer.GetWrittenSpan();
_txLog.Put(batch, newSpan);

// Remove the previous if it exists
if (havePrevious)
{
var prevValSpan = _prevWriter.GetWrittenSpan().SliceFast(sizeof(KeyPrefix));
var currValSpan = _writer.GetWrittenSpan().SliceFast(sizeof(KeyPrefix));

if (!isRetract && prevValSpan.SequenceEqual(currValSpan))
continue;

// Move the previous to the history index
var span = _prevWriter.GetWrittenSpan();
_eavtCurrent.Delete(batch, span);
Expand All @@ -380,16 +382,30 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
}
}

var newSpan = _writer.GetWrittenSpan();
_eavtCurrent.Put(batch, newSpan);
_aevtCurrent.Put(batch, newSpan);
_txLog.Put(batch, newSpan);

if (isReference)
_vaetCurrent.Put(batch, newSpan);
// Add new state
if (!isRetract)
{
_eavtCurrent.Put(batch, newSpan);
_aevtCurrent.Put(batch, newSpan);

if (isReference)
_vaetCurrent.Put(batch, newSpan);

if (isIndexed)
_avetCurrent.Put(batch, newSpan);
if (isIndexed)
_avetCurrent.Put(batch, newSpan);
}
else
{
_eavtHistory.Put(batch, newSpan);
_aevtHistory.Put(batch, newSpan);

if (isReference)
_vaetHistory.Put(batch, newSpan);

if (isIndexed)
_aevtHistory.Put(batch, newSpan);
}
}

var swWrite = Stopwatch.StartNew();
Expand Down
44 changes: 41 additions & 3 deletions tests/NexusMods.MneumonicDB.Storage.Tests/ABackendTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using NexusMods.MneumonicDB.TestModel.ComplexModel.Attributes;
using NexusMods.MneumonicDB.TestModel.Helpers;
using NexusMods.Hashing.xxHash64;
using NexusMods.MneumonicDB.Abstractions.DatomIterators;
using NexusMods.Paths;
using FileAttributes = NexusMods.MneumonicDB.TestModel.ComplexModel.Attributes.FileAttributes;

Expand Down Expand Up @@ -55,10 +56,47 @@ public async Task InsertedDatomsShowUpInTheIndex(IndexType type)
FileAttributes.ModId.Assert(id1, modId2)
]);

var snapshot = DatomStore.GetSnapshot();
var results = DatomStore.Datoms(snapshot, type).ToList();
using var iterator = tx.Snapshot.GetIterator(type);
await Verify(iterator.SeekStart().Resolve().ToTable(Registry))
.UseDirectory("BackendTestVerifyData")
.UseParameters(type);
}

[Theory]
[InlineData(IndexType.TxLog)]
[InlineData(IndexType.EAVTHistory)]
[InlineData(IndexType.EAVTCurrent)]
[InlineData(IndexType.AEVTCurrent)]
[InlineData(IndexType.AEVTHistory)]
[InlineData(IndexType.VAETCurrent)]
[InlineData(IndexType.VAETHistory)]
[InlineData(IndexType.AVETCurrent)]
[InlineData(IndexType.AVETHistory)]
public async Task RetractedValuesAreSupported(IndexType type)
{
var id = NextTempId();
var modId = NextTempId();

var tx1 = await DatomStore.Transact([
FileAttributes.Path.Assert(id, "/foo/bar"),
FileAttributes.Hash.Assert(id, Hash.From(0xDEADBEEF)),
FileAttributes.Size.Assert(id, Size.From(42)),
FileAttributes.ModId.Assert(id, modId),
]);

id = tx1.Remaps[id];
modId = tx1.Remaps[modId];

var tx2 = await DatomStore.Transact([
FileAttributes.Path.Retract(id, "/foo/bar"),
FileAttributes.Hash.Retract(id, Hash.From(0xDEADBEEF)),
FileAttributes.Size.Retract(id, Size.From(42)),
FileAttributes.ModId.Retract(id, modId)
]);


await Verify(results.ToTable(Registry))
using var iterator = tx2.Snapshot.GetIterator(type);
await Verify(iterator.SeekStart().Resolve().ToTable(Registry))
.UseDirectory("BackendTestVerifyData")
.UseParameters(type);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
emptyString
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
emptyString
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
emptyString
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
+ | 0000000000000001 | (0001) UniqueId | NexusMods.MneumonicDB.DatomStore/UniqueId | 0100000000000001
+ | 0000000000000001 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0000000000000002 | (0001) UniqueId | NexusMods.MneumonicDB....tore/ValueSerializerId | 0100000000000001
+ | 0000000000000002 | (0002) ValueSerializerId | NexusMods.MneumonicDB....izers/SymbolSerializer | 0100000000000001
+ | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000002
+ | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000002
+ | 0200000000000001 | (0016) Size | 42 B | 0100000000000002
+ | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000002
- | 0200000000000001 | (0014) Path | /foo/bar | 0100000000000003
- | 0200000000000001 | (0015) Hash | 0x00000000DEADBEEF | 0100000000000003
- | 0200000000000001 | (0016) Size | 42 B | 0100000000000003
- | 0200000000000001 | (0017) ModId | 0200000000000002 | 0100000000000003
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
emptyString
Loading

0 comments on commit 1b54eb4

Please sign in to comment.