From bc1b4b5a6cbaefeb5e654b4a32c1b29d5b953e9f Mon Sep 17 00:00:00 2001 From: halgari Date: Thu, 7 Mar 2024 11:52:19 -0700 Subject: [PATCH] Finish renaming chunk->node --- ...chmarks.cs => AppendableNodeBenchmarks.cs} | 6 ++--- .../IndexBenchmarks.cs | 20 ++++++++--------- .../IteratorBenchmarks.cs | 12 +++++----- .../SortBenchmarks.cs | 8 +++---- .../Algorithms/BinarySearch.cs | 4 ++-- .../{ChunkReader.cs => NodeReader.cs} | 4 ++-- .../Algorithms/SortedMerge.cs | 16 +++++++------- .../Columns/AppendableBlobColumn.cs | 4 ++-- .../Columns/UnsignedIntegerColumn.cs | 2 +- .../DatomStore.cs | 20 ++++++++--------- src/NexusMods.EventSourcing.Storage/FourCC.cs | 2 +- .../NodeStore.cs | 6 ++--- .../Nodes/AppendableIndexNode.cs | 16 +++++++------- .../Nodes/AppendableNode.cs | 6 ++--- .../Nodes/DataNodeExtensions.cs | 6 ++--- .../Nodes/PackedIndexNode.cs | 8 +++---- .../Nodes/ReferenceIndexNode.cs | 16 +++++++------- .../Nodes/ReferenceNode.cs | 14 ++++++------ .../{RawDataChunk.cs => RawDataNode.cs} | 22 +++++++++---------- .../Sorters/TxLog.cs | 5 ----- .../AStorageTest.cs | 10 ++++----- .../AppendableNodeTests.cs | 2 +- .../ComparatorTests.cs | 8 +++---- .../DatomStorageTests.cs | 6 ++--- .../IndexTests.cs | 8 +++---- 25 files changed, 113 insertions(+), 118 deletions(-) rename benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/{AppendableChunkBenchmarks.cs => AppendableNodeBenchmarks.cs} (95%) rename src/NexusMods.EventSourcing.Storage/Algorithms/{ChunkReader.cs => NodeReader.cs} (85%) rename src/NexusMods.EventSourcing.Storage/{RawDataChunk.cs => RawDataNode.cs} (65%) diff --git a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableChunkBenchmarks.cs b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableNodeBenchmarks.cs similarity index 95% rename from benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableChunkBenchmarks.cs rename to benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableNodeBenchmarks.cs index 33ffa0ee..51ab5d85 100644 --- a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableChunkBenchmarks.cs +++ b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/AppendableNodeBenchmarks.cs @@ -13,13 +13,13 @@ namespace NexusMods.EventSourcing.Storage.Benchmarks; [MemoryDiagnoser] -public class AppendableChunkBenchmarks +public class AppendableNodeBenchmarks { private readonly IServiceProvider _services; private readonly AttributeRegistry _registry; private AppendableNode _node = null!; - public AppendableChunkBenchmarks() + public AppendableNodeBenchmarks() { var host = Host.CreateDefaultBuilder() .ConfigureServices(s => @@ -73,7 +73,7 @@ public void IterationSetup() public ulong EntityCount { get; set; } [Benchmark] - public void SortChunk() + public void SortNode() { var comparator = new EATV(_registry); _node.Sort(comparator); diff --git a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IndexBenchmarks.cs b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IndexBenchmarks.cs index b46b457d..b725d6ec 100644 --- a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IndexBenchmarks.cs +++ b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IndexBenchmarks.cs @@ -9,7 +9,7 @@ namespace NexusMods.EventSourcing.Storage.Benchmarks; public class IndexBenchmarks : AStorageBenchmark { - private List _chunks = null!; + private List _nodes = null!; private IDatomComparator _sorter = null!; private IDataNode _preBuilt = null!; private Datom _midPoint; @@ -28,17 +28,17 @@ public class IndexBenchmarks : AStorageBenchmark [GlobalSetup] public void GlobalSetup() { - _chunks = new List(); + _nodes = new List(); - for (ulong chunk = 0; chunk < TxCount; chunk++) + for (ulong node = 0; node < TxCount; node++) { - _chunks.Add(new AppendableNode()); + _nodes.Add(new AppendableNode()); } var emitters = new Action[] { - (e, tx, v) => _registry.Append(_chunks[(int)tx.Value], e, tx, DatomFlags.Added, v), - (e, tx, v) => _registry.Append(_chunks[(int)tx.Value], e, tx, DatomFlags.Added, "file " + v), + (e, tx, v) => _registry.Append(_nodes[(int)tx.Value], e, tx, DatomFlags.Added, v), + (e, tx, v) => _registry.Append(_nodes[(int)tx.Value], e, tx, DatomFlags.Added, "file " + v), }; for (ulong e = 0; e < Count; e++) @@ -54,9 +54,9 @@ public void GlobalSetup() _sorter = _registry.CreateComparator(SortOrder); - foreach (var chunk in _chunks) + foreach (var node in _nodes) { - chunk.Sort(_sorter); + node.Sort(_sorter); } _preBuilt = IndexAll().Flush(NodeStore); @@ -67,9 +67,9 @@ public void GlobalSetup() public AppendableIndexNode IndexAll() { var index = new AppendableIndexNode(_sorter); - foreach (var chunk in _chunks) + foreach (var node in _nodes) { - index = index.Ingest(chunk); + index = index.Ingest(node); } return index; diff --git a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IteratorBenchmarks.cs b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IteratorBenchmarks.cs index b94d2b84..b37eba3c 100644 --- a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IteratorBenchmarks.cs +++ b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/IteratorBenchmarks.cs @@ -29,22 +29,22 @@ public void GlobalSetup() var emitters = new Action[] { - (chunk, e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, v), - (chunk, e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, "file " + v), + (node, e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, v), + (node, e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, "file " + v), }; for (ulong tx = 0; tx < TxCount; tx++) { - var chunk = new AppendableNode(); + var node = new AppendableNode(); for (ulong e = 0; e < Count; e++) { for (var a = 0; a < 2; a++) { - emitters[a](chunk, EntityId.From(e), TxId.From(tx), tx); + emitters[a](node, EntityId.From(e), TxId.From(tx), tx); } } - chunk.Sort(_sorter); - _index = _index.Ingest(chunk); + node.Sort(_sorter); + _index = _index.Ingest(node); } } diff --git a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/SortBenchmarks.cs b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/SortBenchmarks.cs index 9cce103c..6df9db22 100644 --- a/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/SortBenchmarks.cs +++ b/benchmarks/NexusMods.EventSourcing.Storage.Benchmarks/SortBenchmarks.cs @@ -21,12 +21,12 @@ public class SortBenchmarks : AStorageBenchmark [GlobalSetup] public void GlobalSetup() { - var chunk = new AppendableNode(); + var node = new AppendableNode(); var emitters = new Action[] { - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, v), - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, "file " + v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, "file " + v), }; for (ulong e = 0; e < Count; e++) @@ -43,7 +43,7 @@ public void GlobalSetup() } } - _node = chunk; + _node = node; } diff --git a/src/NexusMods.EventSourcing.Storage/Algorithms/BinarySearch.cs b/src/NexusMods.EventSourcing.Storage/Algorithms/BinarySearch.cs index be0f66c4..bd8ac026 100644 --- a/src/NexusMods.EventSourcing.Storage/Algorithms/BinarySearch.cs +++ b/src/NexusMods.EventSourcing.Storage/Algorithms/BinarySearch.cs @@ -6,8 +6,8 @@ namespace NexusMods.EventSourcing.Storage.Algorithms; public static class BinarySearch { - public static int SeekEqualOrLess(TChunk node, TComparator comparator, int start, int end, in Datom target) - where TChunk : IDataNode + public static int SeekEqualOrLess(TNode node, TComparator comparator, int start, int end, in Datom target) + where TNode : IDataNode where TComparator : IDatomComparator { while (start < end) diff --git a/src/NexusMods.EventSourcing.Storage/Algorithms/ChunkReader.cs b/src/NexusMods.EventSourcing.Storage/Algorithms/NodeReader.cs similarity index 85% rename from src/NexusMods.EventSourcing.Storage/Algorithms/ChunkReader.cs rename to src/NexusMods.EventSourcing.Storage/Algorithms/NodeReader.cs index 4e93508d..1fdb0712 100644 --- a/src/NexusMods.EventSourcing.Storage/Algorithms/ChunkReader.cs +++ b/src/NexusMods.EventSourcing.Storage/Algorithms/NodeReader.cs @@ -6,9 +6,9 @@ namespace NexusMods.EventSourcing.Storage.Algorithms; -public static class ChunkReader +public static class NodeReader { - public static IDataNode ReadDataChunk(ReadOnlyMemory data) + public static IDataNode ReadDataNode(ReadOnlyMemory data) { var reader = new BufferReader(data); var header = reader.ReadFourCC(); diff --git a/src/NexusMods.EventSourcing.Storage/Algorithms/SortedMerge.cs b/src/NexusMods.EventSourcing.Storage/Algorithms/SortedMerge.cs index 197f27f4..2bfb3574 100644 --- a/src/NexusMods.EventSourcing.Storage/Algorithms/SortedMerge.cs +++ b/src/NexusMods.EventSourcing.Storage/Algorithms/SortedMerge.cs @@ -11,7 +11,7 @@ public static AppendableNode Merge(TNodeA a, TNodeB where TNodeB : IDataNode where TComparator : IDatomComparator { - var newChunk = new AppendableNode(); + var newNode = new AppendableNode(); int i = 0, j = 0; @@ -23,18 +23,18 @@ public static AppendableNode Merge(TNodeA a, TNodeB var cmp = comparator.Compare(aDatom, bDatom); if (cmp < 0) { - newChunk.Append(aDatom); + newNode.Append(aDatom); i++; } else if (cmp > 0) { - newChunk.Append(bDatom); + newNode.Append(bDatom); j++; } else { - newChunk.Append(aDatom); - newChunk.Append(bDatom); + newNode.Append(aDatom); + newNode.Append(bDatom); i++; j++; } @@ -42,18 +42,18 @@ public static AppendableNode Merge(TNodeA a, TNodeB while (i < a.Length) { - newChunk.Append(a[i]); + newNode.Append(a[i]); i++; } while (j < b.Length) { - newChunk.Append(b[j]); + newNode.Append(b[j]); j++; } - return newChunk; + return newNode; } } diff --git a/src/NexusMods.EventSourcing.Storage/Columns/AppendableBlobColumn.cs b/src/NexusMods.EventSourcing.Storage/Columns/AppendableBlobColumn.cs index 33ac112d..3f1e51a3 100644 --- a/src/NexusMods.EventSourcing.Storage/Columns/AppendableBlobColumn.cs +++ b/src/NexusMods.EventSourcing.Storage/Columns/AppendableBlobColumn.cs @@ -78,10 +78,10 @@ public void Initialize(IEnumerable> select) } } - public static AppendableBlobColumn UnpackFrom(IBlobColumn indexChunkValues) + public static AppendableBlobColumn UnpackFrom(IBlobColumn indexNodeValues) { var newColumn = new AppendableBlobColumn(); - foreach (var value in indexChunkValues) + foreach (var value in indexNodeValues) { newColumn.Append(value.Span); } diff --git a/src/NexusMods.EventSourcing.Storage/Columns/UnsignedIntegerColumn.cs b/src/NexusMods.EventSourcing.Storage/Columns/UnsignedIntegerColumn.cs index a41470b7..594ef3c2 100644 --- a/src/NexusMods.EventSourcing.Storage/Columns/UnsignedIntegerColumn.cs +++ b/src/NexusMods.EventSourcing.Storage/Columns/UnsignedIntegerColumn.cs @@ -16,7 +16,7 @@ public class UnsignedIntegerColumn : IAppendableColumn, IUnpackedColumn private uint _length; private T[] _data; - public UnsignedIntegerColumn(uint initialLength = RawDataChunk.DefaultChunkSize) + public UnsignedIntegerColumn(uint initialLength = RawDataNode.DefaultNodeSize) { _length = 0; _data = GC.AllocateUninitializedArray((int)initialLength); diff --git a/src/NexusMods.EventSourcing.Storage/DatomStore.cs b/src/NexusMods.EventSourcing.Storage/DatomStore.cs index 93b57ea4..f38a35b4 100644 --- a/src/NexusMods.EventSourcing.Storage/DatomStore.cs +++ b/src/NexusMods.EventSourcing.Storage/DatomStore.cs @@ -71,11 +71,11 @@ private async Task ConsumeTransactions() _logger.LogDebug("Processing transaction with {DatomCount} datoms", pendingTransaction.Data.Length); sw.Restart(); - Log(pendingTransaction, out var chunk); + Log(pendingTransaction, out var node); - await UpdateInMemoryIndexes(chunk, pendingTransaction.AssignedTxId!.Value); + await UpdateInMemoryIndexes(node, pendingTransaction.AssignedTxId!.Value); - _updatesSubject.OnNext((pendingTransaction.AssignedTxId.Value, chunk)); + _updatesSubject.OnNext((pendingTransaction.AssignedTxId.Value, node)); pendingTransaction.CompletionSource.SetResult(pendingTransaction.AssignedTxId.Value); _logger.LogDebug("Transaction {TxId} processed in {Elapsed}ms, new in-memory size is {Count} datoms", pendingTransaction.AssignedTxId!.Value, sw.ElapsedMilliseconds, _indexes.InMemorySize); } @@ -383,9 +383,9 @@ private IEnumerable ReverseLookupForIndex(TxId txId, EntityId private void Log(PendingTransaction pendingTransaction, out AppendableNode node) { - var newChunk = new AppendableNode(); + var newNode = new AppendableNode(); foreach (var datom in pendingTransaction.Data) - datom.Append(_registry, newChunk); + datom.Append(_registry, newNode); var nextTxBlock = _nodeStore.GetNextTx(); @@ -419,13 +419,13 @@ EntityId MaybeRemap(EntityId id) return id; } - newChunk.SetTx(nextTx); - newChunk.RemapEntities(MaybeRemap, _registry); + newNode.SetTx(nextTx); + newNode.RemapEntities(MaybeRemap, _registry); - newChunk.Sort(_comparatorTxLog); + newNode.Sort(_comparatorTxLog); - node = newChunk; - var newTxBlock = _nodeStore.LogTx(newChunk.Pack()); + node = newNode; + var newTxBlock = _nodeStore.LogTx(newNode.Pack()); Debug.Assert(newTxBlock.Value == nextTxBlock.Value, "newTxBlock == nextTxBlock"); pendingTransaction.AssignedTxId = nextTx; diff --git a/src/NexusMods.EventSourcing.Storage/FourCC.cs b/src/NexusMods.EventSourcing.Storage/FourCC.cs index f89f2dc9..7de63aad 100644 --- a/src/NexusMods.EventSourcing.Storage/FourCC.cs +++ b/src/NexusMods.EventSourcing.Storage/FourCC.cs @@ -29,7 +29,7 @@ public static FourCC From(ReadOnlySpan s) return result; } - #region Chunk Types + #region Node Types public static readonly FourCC PackedData = From("PDAT"); public static readonly FourCC PackedIndex = From("PIDX"); diff --git a/src/NexusMods.EventSourcing.Storage/NodeStore.cs b/src/NexusMods.EventSourcing.Storage/NodeStore.cs index c772dbdf..99f9e3bb 100644 --- a/src/NexusMods.EventSourcing.Storage/NodeStore.cs +++ b/src/NexusMods.EventSourcing.Storage/NodeStore.cs @@ -38,8 +38,8 @@ public IDataNode Flush(IDataNode node) { return node switch { - PackedNode packedChunk => Flush(packedChunk), - PackedIndexNode packedIndexChunk => Flush(packedIndexChunk), + PackedNode packedNode => Flush(packedNode), + PackedIndexNode packedIndexNode => Flush(packedIndexNode), _ => throw new NotImplementedException("Unknown node type. " + node.GetType().Name) }; } @@ -56,7 +56,7 @@ public IDataNode Flush(IIndexNode node) { return node switch { - PackedIndexNode packedIndexChunk => Flush(packedIndexChunk), + PackedIndexNode packedIndexNode => Flush(packedIndexNode), _ => throw new NotImplementedException("Unknown node type. " + node.GetType().Name) }; } diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/AppendableIndexNode.cs b/src/NexusMods.EventSourcing.Storage/Nodes/AppendableIndexNode.cs index 69308fef..a782f192 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/AppendableIndexNode.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/AppendableIndexNode.cs @@ -130,9 +130,9 @@ public override IDataNode Flush(INodeStore store) for (var i = 0; i < _children.Count; i++) { var child = _children[i]; - if (child is AppendableNode appendableChunk) + if (child is AppendableNode appendableNode) { - var packedChild = appendableChunk.Pack(); + var packedChild = appendableNode.Pack(); _children[i] = store.Flush(packedChild); } } @@ -198,17 +198,17 @@ public AppendableIndexNode Ingest(IDataNode node) var newChildren = new List(_children.Count); - void MaybeSplit(AppendableNode chunk) + void MaybeSplit(AppendableNode node) { - if (chunk.Length > Configuration.DataBlockSize * 2) + if (node.Length > Configuration.DataBlockSize * 2) { - var (a, b) = chunk.Split(); + var (a, b) = node.Split(); MaybeSplit(a); MaybeSplit(b); } else { - newChildren.Add(chunk); + newChildren.Add(node); } } @@ -246,8 +246,8 @@ void MaybeSplit(AppendableNode chunk) } - public AppendableNode Merge(TChunk child, TEnumerable datoms) - where TChunk : IDataNode + public AppendableNode Merge(TNode child, TEnumerable datoms) + where TNode : IDataNode where TEnumerable : IEnumerable { return AppendableNode.Initialize(child.Merge(datoms, _comparator)); diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/AppendableNode.cs b/src/NexusMods.EventSourcing.Storage/Nodes/AppendableNode.cs index 5ccf93cf..2b697754 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/AppendableNode.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/AppendableNode.cs @@ -224,12 +224,12 @@ public override IEnumerator GetEnumerator() /// public static AppendableNode Initialize(IEnumerable datoms) { - var chunk = new AppendableNode(); + var node = new AppendableNode(); foreach (var datom in datoms) { - chunk.Append(datom); + node.Append(datom); } - return chunk; + return node; } public (AppendableNode A, AppendableNode B) Split() diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/DataNodeExtensions.cs b/src/NexusMods.EventSourcing.Storage/Nodes/DataNodeExtensions.cs index 8fb4a215..8cdb0443 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/DataNodeExtensions.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/DataNodeExtensions.cs @@ -5,12 +5,12 @@ namespace NexusMods.EventSourcing.Storage.Nodes; public static class DataNodeExtensions { - public static IEnumerable Range(this TChunkA chunk, int start, int end) - where TChunkA : IDataNode + public static IEnumerable Range(this TNodeA node, int start, int end) + where TNodeA : IDataNode { for (var i = start; i < end; i++) { - yield return chunk[i]; + yield return node[i]; } } diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/PackedIndexNode.cs b/src/NexusMods.EventSourcing.Storage/Nodes/PackedIndexNode.cs index 65c94fb1..b9b0f232 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/PackedIndexNode.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/PackedIndexNode.cs @@ -90,15 +90,15 @@ public override void WriteTo(TWriter writer) writer.Write((byte)Comparator.SortOrder); foreach (var child in _children) { - if (child is ReferenceNode indexChunk) + if (child is ReferenceNode indexNode) { writer.WriteFourCC(FourCC.ReferenceIndex); - writer.Write((ulong)indexChunk.Key); + writer.Write((ulong)indexNode.Key); } - else if (child is ReferenceNode dataChunk) + else if (child is ReferenceNode dataNode) { writer.WriteFourCC(FourCC.ReferenceData); - writer.Write((ulong)dataChunk.Key); + writer.Write((ulong)dataNode.Key); } else { diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceIndexNode.cs b/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceIndexNode.cs index d2ea17dc..6efb04b8 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceIndexNode.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceIndexNode.cs @@ -11,20 +11,20 @@ public class ReferenceIndexNode : IIndexNode { private readonly NodeStore _store; private readonly StoreKey _key; - private WeakReference? _chunk; + private WeakReference? _node; - public ReferenceIndexNode(NodeStore store, StoreKey key, WeakReference? chunk) + public ReferenceIndexNode(NodeStore store, StoreKey key, WeakReference? node) { _store = store; _key = key; - _chunk = chunk; + _node = node; } public StoreKey Key => _key; public IIndexNode Resolve() { - if (_chunk?.TryGetTarget(out var target) == true) + if (_node?.TryGetTarget(out var target) == true) { return target; } @@ -34,14 +34,14 @@ public IIndexNode Resolve() private IIndexNode Load() { - var chunkData = (IIndexNode)_store.Load(_key); - _chunk = new WeakReference(chunkData); - if (chunkData.Length != Length) + var nodeData = (IIndexNode)_store.Load(_key); + _node = new WeakReference(nodeData); + if (nodeData.Length != Length) { throw new InvalidOperationException("Node length mismatch"); } - return chunkData; + return nodeData; } public IEnumerator GetEnumerator() diff --git a/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceNode.cs b/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceNode.cs index 501beaa9..3b898967 100644 --- a/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceNode.cs +++ b/src/NexusMods.EventSourcing.Storage/Nodes/ReferenceNode.cs @@ -7,11 +7,11 @@ namespace NexusMods.EventSourcing.Storage.Nodes; -public class ReferenceNode(NodeStore store, StoreKey key, WeakReference? chunk) : IDataNode +public class ReferenceNode(NodeStore store, StoreKey key, WeakReference? node) : IDataNode { private IDataNode Resolve() { - if (chunk?.TryGetTarget(out var target) == true) + if (node?.TryGetTarget(out var target) == true) { return target; } @@ -20,14 +20,14 @@ private IDataNode Resolve() private IDataNode LoadNode() { - var chunkData = store.Load(key); - chunk = new WeakReference(chunkData); - if (chunkData.Length != Length) + var nodeData = store.Load(key); + node = new WeakReference(nodeData); + if (nodeData.Length != Length) { throw new InvalidOperationException("Node length mismatch"); } - return chunkData; + return nodeData; } public IEnumerator GetEnumerator() @@ -42,7 +42,7 @@ IEnumerator IEnumerable.GetEnumerator() public StoreKey Key => key; - public bool IsResolved => chunk != null; + public bool IsResolved => node != null; public int Length => Resolve().Length; public IColumn EntityIds => Resolve().EntityIds; public IColumn AttributeIds => Resolve().AttributeIds; diff --git a/src/NexusMods.EventSourcing.Storage/RawDataChunk.cs b/src/NexusMods.EventSourcing.Storage/RawDataNode.cs similarity index 65% rename from src/NexusMods.EventSourcing.Storage/RawDataChunk.cs rename to src/NexusMods.EventSourcing.Storage/RawDataNode.cs index dc69a510..afc8c28d 100644 --- a/src/NexusMods.EventSourcing.Storage/RawDataChunk.cs +++ b/src/NexusMods.EventSourcing.Storage/RawDataNode.cs @@ -10,30 +10,30 @@ namespace NexusMods.EventSourcing.Storage.Abstractions; /// being a linear array of the same type, so something like (EntityId[], TxId[], ushort[], DatomFlags[]) etc. /// This allows data to be packed, and iteration to happen in a linear and even vectorized fashion. /// -[StructLayout(LayoutKind.Sequential, Pack = 1, Size = DataChunkSize)] -public unsafe struct RawDataChunk +[StructLayout(LayoutKind.Sequential, Pack = 1, Size = DataNodeSize)] +public unsafe struct RawDataNode { /// /// Size of the vector, must be a power of 16 for maximum vectorization support. /// - public const int DefaultChunkSize = 2048; + public const int DefaultNodeSize = 2048; /// /// The size of the DataChunk node in bytes. /// - public const int DataChunkSize = - (DefaultChunkSize * (sizeof(ulong) + sizeof(ulong) + sizeof(ushort) + sizeof(byte) + sizeof(ulong))) + - (DefaultChunkSize / 8); + public const int DataNodeSize = + (DefaultNodeSize * (sizeof(ulong) + sizeof(ulong) + sizeof(ushort) + sizeof(byte) + sizeof(ulong))) + + (DefaultNodeSize / 8); - public fixed ulong EntityIdVector[DefaultChunkSize]; + public fixed ulong EntityIdVector[DefaultNodeSize]; - public fixed ulong TxIdVector[DefaultChunkSize]; + public fixed ulong TxIdVector[DefaultNodeSize]; - public fixed ushort AttributeIdVector[DefaultChunkSize]; + public fixed ushort AttributeIdVector[DefaultNodeSize]; - public fixed byte FlagsVector[DefaultChunkSize]; + public fixed byte FlagsVector[DefaultNodeSize]; - public fixed ulong InlinedData[DefaultChunkSize]; + public fixed ulong InlinedData[DefaultNodeSize]; /// /// A reference to a block of memory that contains the outlined data for this node, this is data diff --git a/src/NexusMods.EventSourcing.Storage/Sorters/TxLog.cs b/src/NexusMods.EventSourcing.Storage/Sorters/TxLog.cs index b7feadcf..223e7526 100644 --- a/src/NexusMods.EventSourcing.Storage/Sorters/TxLog.cs +++ b/src/NexusMods.EventSourcing.Storage/Sorters/TxLog.cs @@ -31,11 +31,6 @@ public IComparer MakeComparer(MemoryDatom datoms) where TBlob { return new TxLogComparer(registry, datoms); } - - public int Compare(in MemoryDatom chunk, int a, int b) where T : IBlobColumn - { - throw new System.NotImplementedException(); - } } internal unsafe class TxLogComparer(AttributeRegistry registry, MemoryDatom datoms) : IComparer diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs b/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs index 73d413d6..a68f077a 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/AStorageTest.cs @@ -44,14 +44,14 @@ protected AStorageTest(IServiceProvider provider, KVStoreType storeType = KVStor } - public AppendableNode TestDatomChunk(int entityCount = 100) + public AppendableNode TestDatomNode(int entityCount = 100) { - var chunk = new AppendableNode(); + var node = new AppendableNode(); var emitters = new Action[] { - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, v), - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, "file " + v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, "file " + v), }; for (ulong e = 0; e < (ulong)entityCount; e++) @@ -64,7 +64,7 @@ public AppendableNode TestDatomChunk(int entityCount = 100) } } } - return chunk; + return node; } public IEnumerable<(IWriteDatom, ulong)> TestDatoms(int entityCount = 100) diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/AppendableNodeTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/AppendableNodeTests.cs index 6fb05d42..d2201577 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/AppendableNodeTests.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/AppendableNodeTests.cs @@ -189,7 +189,7 @@ public void CanReadAndWriteBlocks(uint count) var uncompressedSize = packed.Length * (8 + 8 + 1 + 8); Logger.LogInformation("Compression ratio: {0}%", (writer.WrittenMemory.Length * 100) / uncompressedSize); - var read = ChunkReader.ReadDataChunk(writer.WrittenMemory); + var read = NodeReader.ReadDataNode(writer.WrittenMemory); for (var i = 0; i < allDatoms.Length; i++) { diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/ComparatorTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/ComparatorTests.cs index 812cd8bb..73692ddc 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/ComparatorTests.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/ComparatorTests.cs @@ -22,12 +22,12 @@ public void EATVTests(Datom a, Datom b, int result) public IEnumerable ComparisonTestData() { - var chunk = new AppendableNode(); + var node = new AppendableNode(); var emitters = new Action[] { - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, v), - (e, tx, v) => _registry.Append(chunk, e, tx, DatomFlags.Added, "file " + v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, v), + (e, tx, v) => _registry.Append(node, e, tx, DatomFlags.Added, "file " + v), }; for (ulong e = 0; e < 2; e++) @@ -46,7 +46,7 @@ public IEnumerable ComparisonTestData() Datom? prev = null; - foreach (var datom in chunk) + foreach (var datom in node) { if (prev != null) { diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStorageTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStorageTests.cs index 4b311c05..6d743a04 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/DatomStorageTests.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/DatomStorageTests.cs @@ -28,14 +28,14 @@ public async Task CanFlushToDisk() DatomStoreSettings.MaxInMemoryDatoms = 128; - var chunks = TestDatoms(1024) + var nodes = TestDatoms(1024) .GroupBy(d => d.Item2, d => d.Item1) .OrderBy(d => d.Key) .ToArray(); - foreach (var chunk in chunks) + foreach (var node in nodes) { - await DatomStore.Transact(chunk); + await DatomStore.Transact(node); } } } diff --git a/tests/NexusMods.EventSourcing.Storage.Tests/IndexTests.cs b/tests/NexusMods.EventSourcing.Storage.Tests/IndexTests.cs index 35d46f6e..41276e49 100644 --- a/tests/NexusMods.EventSourcing.Storage.Tests/IndexTests.cs +++ b/tests/NexusMods.EventSourcing.Storage.Tests/IndexTests.cs @@ -22,7 +22,7 @@ public void CanIngestAndGetDatoms(int entityCount, SortOrders sortOrder, bool fl private void CanIngestAndGetDatomsInner(TType comparator, int entityCount, SortOrders sortOrder, bool flush) where TType : IDatomComparator { - var testData = TestDatomChunk(entityCount); + var testData = TestDatomNode(entityCount); var index = new AppendableIndexNode(comparator); @@ -36,11 +36,11 @@ private void CanIngestAndGetDatomsInner(TType comparator, int entityCount foreach (var group in grouped) { - var newChunk = AppendableNode.Initialize(group); - newChunk.Sort(comparator); + var newNode = AppendableNode.Initialize(group); + newNode.Sort(comparator); sw.Start(); - index = index.Ingest(newChunk); + index = index.Ingest(newNode); sw.Stop(); if (flush)