Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

IndexSegment improvements #86

Merged
merged 4 commits into from
Aug 8, 2024
Merged

IndexSegment improvements #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d54c292
Add custom enumerator to IndexSegment
erri120 Aug 8, 2024
2e3f271
IndexSegment implements `IReadOnlyList<Datom>`
erri120 Aug 8, 2024
3ae21d5
Improve logging performance
erri120 Aug 8, 2024
9b34aa9
Make ReprocessData static
erri120 Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 68 additions & 35 deletions src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using DynamicData;
using JetBrains.Annotations;
using NexusMods.MnemonicDB.Abstractions.DatomIterators;
using NexusMods.MnemonicDB.Abstractions.Internals;
using NexusMods.MnemonicDB.Abstractions.Models;
Expand All @@ -19,45 +19,49 @@ namespace NexusMods.MnemonicDB.Abstractions.IndexSegments;
/// A segment of an index, used most often as a cache. For example when an entity is read from the database,
/// the whole entity may be cached in one of these segments for fast access.
/// </summary>
public readonly struct IndexSegment : IEnumerable<Datom>
[PublicAPI]
public readonly struct IndexSegment : IReadOnlyList<Datom>
{
private readonly IAttributeRegistry _registry;
private readonly int _rowCount;
private readonly ReadOnlyMemory<byte> _data;


/// <summary>
/// Construct a new index segment from the given data and offsets
/// </summary>
public IndexSegment(ReadOnlySpan<byte> data, ReadOnlySpan<int> offsets, IAttributeRegistry registry)
{
_registry = registry;

if (data.Length == 0)
{
_rowCount = 0;
_data = ReadOnlyMemory<byte>.Empty;
return;
}

_rowCount = offsets.Length - 1;

var memory = new Memory<byte>(GC.AllocateUninitializedArray<byte>(data.Length + (_rowCount + 1) * sizeof(int)));
_data = memory;
ReprocessData(data, offsets, memory.Span);

ReprocessData(_rowCount, data, offsets, memory.Span);
}

/// <summary>
/// All the upper values
/// </summary>
private ReadOnlySpan<ulong> _uppers => _data.Span.SliceFast(0, _rowCount * sizeof(ulong)).CastFast<byte, ulong>();
private ReadOnlySpan<ulong> Uppers => _data.Span.SliceFast(0, _rowCount * sizeof(ulong)).CastFast<byte, ulong>();

/// <summary>
/// All the lower values
/// </summary>
private ReadOnlySpan<ulong> _lowers => _data.Span.SliceFast(_rowCount * sizeof(ulong), _rowCount * sizeof(ulong)).CastFast<byte, ulong>();
private ReadOnlySpan<ulong> Lowers => _data.Span.SliceFast(_rowCount * sizeof(ulong), _rowCount * sizeof(ulong)).CastFast<byte, ulong>();

/// <summary>
/// All the offsets
/// </summary>
private ReadOnlySpan<int> _offsets => _data.Span.SliceFast(_rowCount * sizeof(ulong) * 2, (_rowCount + 1) * sizeof(int)).CastFast<byte, int>();
private ReadOnlySpan<int> Offsets => _data.Span.SliceFast(_rowCount * sizeof(ulong) * 2, (_rowCount + 1) * sizeof(int)).CastFast<byte, int>();

/// <summary>
/// Pivots all the data into 4 columns:
Expand All @@ -66,21 +70,21 @@ public IndexSegment(ReadOnlySpan<byte> data, ReadOnlySpan<int> offsets, IAttribu
/// - (int) offsets for each row's value into the value blob
/// - (byte[]) value blob
/// </summary>
private void ReprocessData(ReadOnlySpan<byte> data, ReadOnlySpan<int> offsets, Span<byte> dataSpan)
private static void ReprocessData(int rowCount, ReadOnlySpan<byte> data, ReadOnlySpan<int> offsets, Span<byte> dataSpan)
{
var uppers = dataSpan.SliceFast(0, _rowCount * sizeof(ulong)).CastFast<byte, ulong>();
var lowers = dataSpan.SliceFast(_rowCount * sizeof(ulong), _rowCount * sizeof(ulong)).CastFast<byte, ulong>();
var uppers = dataSpan.SliceFast(0, rowCount * sizeof(ulong)).CastFast<byte, ulong>();
var lowers = dataSpan.SliceFast(rowCount * sizeof(ulong), rowCount * sizeof(ulong)).CastFast<byte, ulong>();

// Extra space for one int in the offsets so we can calculate the size of the last row
var valueOffsets = dataSpan.SliceFast(_rowCount * sizeof(ulong) * 2, (_rowCount + 1) * sizeof(int)).CastFast<byte, int>();
var values = dataSpan.SliceFast((_rowCount * (sizeof(ulong) * 2 + sizeof(int))) + sizeof(int));
// Extra space for one int in the offsets, so we can calculate the size of the last row
var valueOffsets = dataSpan.SliceFast(rowCount * sizeof(ulong) * 2, (rowCount + 1) * sizeof(int)).CastFast<byte, int>();
var values = dataSpan.SliceFast(rowCount * (sizeof(ulong) * 2 + sizeof(int)) + sizeof(int));

var relativeValueOffset = 0;

// The first row starts at the beginning of the value blob
var absoluteValueOffset = _rowCount * (sizeof(ulong) * 2 + sizeof(int)) + sizeof(int);
var absoluteValueOffset = rowCount * (sizeof(ulong) * 2 + sizeof(int)) + sizeof(int);

for (var i = 0; i < _rowCount; i++)
for (var i = 0; i < rowCount; i++)
{
var rowSegment = data.Slice(offsets[i], offsets[i + 1] - offsets[i]);
var prefix = MemoryMarshal.Read<KeyPrefix>(rowSegment);
Expand All @@ -96,11 +100,9 @@ private void ReprocessData(ReadOnlySpan<byte> data, ReadOnlySpan<int> offsets, S
}

// The last row's offset is the size of the value blob
valueOffsets[_rowCount] = absoluteValueOffset;
valueOffsets[rowCount] = absoluteValueOffset;
}



/// <summary>
/// Returns true if this segment is valid (contains data)
/// </summary>
Expand Down Expand Up @@ -128,13 +130,13 @@ public Datom this[int idx]
{
get
{
var offsets = _offsets;
var offsets = Offsets;
var fromOffset = offsets[idx];
var toOffset = offsets[idx + 1];

var valueSlice = _data.Slice(fromOffset, toOffset - fromOffset);

return new Datom(new KeyPrefix(_uppers[idx], _lowers[idx]), valueSlice, _registry);
return new Datom(new KeyPrefix(Uppers[idx], Lowers[idx]), valueSlice, _registry);
}
}

Expand All @@ -150,14 +152,16 @@ public bool Contains(IAttribute attribute)
return false;
}

/// <summary>
/// Returns the enumerator.
/// </summary>
public Enumerator GetEnumerator() => new(this);

/// <inheritdoc />
public IEnumerator<Datom> GetEnumerator()
{
for (var i = 0; i < _rowCount; i++)
{
yield return this[i];
}
}
IEnumerator<Datom> IEnumerable<Datom>.GetEnumerator() => GetEnumerator();

/// <inheritdoc />
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();

/// <summary>
/// Resolves all the datoms in this segment
Expand All @@ -167,10 +171,6 @@ public IEnumerable<IReadDatom> Resolved()
return this.Select(d => d.Resolved);
}

IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}

/// <summary>
/// Create a new index segment from the given datoms
Expand Down Expand Up @@ -206,7 +206,7 @@ public int FindFirst(EntityId find)
var mid = left + (right - left) / 2;


var lower = _lowers[mid];
var lower = Lowers[mid];
var e = EntityId.From((lower & 0xFF00000000000000) | ((lower >> 8) & 0x0000FFFFFFFFFFFF));

var comparison = e.CompareTo(find);
Expand Down Expand Up @@ -234,7 +234,7 @@ public int FindFirstAVX2(ulong find)
.WithElement(2, find)
.WithElement(3, find);

var casted = MemoryMarshal.Cast<ulong, Vector256<ulong>>(_lowers);
var casted = MemoryMarshal.Cast<ulong, Vector256<ulong>>(Lowers);

for (var idx = 0; idx < casted.Length; idx += 1)
{
Expand All @@ -255,14 +255,47 @@ public int FindFirstAVX2(ulong find)
}

// Handle remaining elements
for (int i = (casted.Length * Vector256<ulong>.Count); i < _lowers.Length; i++)
for (int i = (casted.Length * Vector256<ulong>.Count); i < Lowers.Length; i++)
{
if (_lowers[i] >= find)
if (Lowers[i] >= find)
{
return i;
}
}

return -1; // No value found that is greater than or equal to the target
}

/// <summary>
/// Enumerator.
/// </summary>
public struct Enumerator : IEnumerator<Datom>
{
private readonly IndexSegment _indexSegment;
private int _index;

internal Enumerator(IndexSegment indexSegment)
{
_indexSegment = indexSegment;
}

/// <inheritdoc/>
public Datom Current { get; private set; } = default!;

object IEnumerator.Current => Current;

/// <inheritdoc/>
public bool MoveNext()
{
if (_index >= _indexSegment.Count) return false;
Current = _indexSegment[_index++];
return true;
}

/// <inheritdoc/>
public void Reset() => _index = 0;

/// <inheritdoc/>
public void Dispose() { }
}
}
17 changes: 12 additions & 5 deletions src/NexusMods.MnemonicDB/Storage/DatomStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ public void Dispose()

private void ConsumeTransactions()
{
var debugEnabled = _logger.IsEnabled(LogLevel.Debug);

try
{
while (!_pendingTransactions.IsCompleted && !_shutdownToken.Token.IsCancellationRequested)
Expand All @@ -225,12 +227,17 @@ private void ConsumeTransactions()
}

Log(pendingTransaction, out var result);
var sw = Stopwatch.StartNew();
FinishTransaction(result, pendingTransaction);

if (_logger.IsEnabled(LogLevel.Debug))

if (debugEnabled)
{
var sw = Stopwatch.StartNew();
FinishTransaction(result, pendingTransaction);
_logger.LogDebug("Transaction {TxId} post-processed in {Elapsed}ms", result.AssignedTxId, sw.ElapsedMilliseconds);
}
else
{
FinishTransaction(result, pendingTransaction);
}
}
catch (Exception ex)
{
Expand Down
Loading