Merge pull request #63 from Nexus-Mods/observable-indexes

Observable indexes

docs/QueryDesign.md

@@ -0,0 +1,51 @@
+---
+hide:
+  - toc
+---
+
+## Query Design
+
+The architecture of MnemonicDB is fairly simple: tuples are stored in indexes sorted in various ways, the logging function
+(in `IDatomStore`) publishes a list of new databases and from those the updates in each transaction can be determined. This
+simple format allows for a wide variety of queries to be performed on the data and optimally a lot of performance can be gained
+in many parts of query.
+
+### Goals
+A few goals of what is desired in the query design:
+
+* **Performance**: The queries should not require O(n*m) operations as much as possible, while parts may be implemented
+simply and have higher complexity, options should be left in the design for optimization.
+* **Larger than memory**: The query results are expected to fit in memory, but the source datasets may not. This means that
+as much as possible only the minimal working set should be loaded into memory.
+* **System Sympathy**: The queries should be designed to work well the rest of the database, indexes store data pre-sorted,
+queries should be designed to take advantage of this.
+* **Live Queries**: C#'s DynamicData is a fantastic library for UI programming and is close (but not quite) to something
+that can be used by MnemonicDB. The queries should be designed to work well with this library, but also provide some sort
+of delta-update systems such as `IObservable<IChangeSet<T>>` or similar. This allows for small transactions to not require
+the entire query to be re-run. A delta update system fits very well with MnemonicDB's transactional publish queue, as each
+transaction can result in a delta of datoms added (or removed) from the database.
+
+
+### Concepts
+
+* **IConnection**: The connection is the primary interface for talking to the database, it can be "dereferenced" by calling
+`conn.Db` to get a immutable database. This interfaces also provides an `IObservable<IDb>` for subscribing to updates to the
+database. It also provides a `IObservable<IDb, IndexSlice>` for subscribing to updates along with the portion of the `TxLog`
+added in each transaction.
+* **IDb**: The database is a readonly view of all the data in the database as of a specific point in time.
+* **SliceDescriptor**: A description of a slice of an index in the database. It defines how a database value should be
+interpreted and which index to use. This can be thought of as a "view" on the database, consisting of a tuple of `[index, from, to]`.
+SliceDescriptors are immutable, and are compared by value. This means that they can be used as keys in dictionaries and sets, and makes
+them useful for caching.
+* **ObservableSlice**: A slice that can be subscribed to for updates. This is a `IObservable<IChangeSet<Datom>>` and is constructed
+by combining a `SliceDescriptor` and a `IConnection`.
+* **IndexSlice**: A loaded chunk from the database, made by combining a `SliceDescriptor` and a `IDb`.
+
+### Query Design
+
+Based on these simple primitives, a wide variety of queries can be constructed, two IndexSlices can be joined to filter each other,
+the datoms in the index can be grouped, sorted, and filtered in various ways. The `ObservableSlice` can be used to create live queries
+of the database.
+
+#### Example Queries
+

src/NexusMods.MnemonicDB.Abstractions/Attribute.cs

@@ -476,6 +476,24 @@ public override string ToString()
         {
             return $"({(IsRetract ? "-" : "+")}, {E.Value:x}, {A.Id.Name}, {V}, {T.Value:x})";
         }
+
+        /// <inheritdoc />
+        public bool EqualsByValue(IReadDatom other)
+        {
+            if (other is not ReadDatom o)
+                return false;
+            return A == o.A && E == o.E && V!.Equals(o.V);
+        }
+
+        /// <inheritdoc />
+        public int HashCodeByValue()
+        {
+            return HashCode.Combine(A, E, V);
+        }
+
+
     }
 
+
+
 }

src/NexusMods.MnemonicDB.Abstractions/AttributeId.cs

@@ -32,4 +32,12 @@ public override string ToString()
     {
         return "AId:" + Value.ToString("X");
     }
+
+    /// <summary>
+    /// Returns the next AttributeId after this one.
+    /// </summary>
+    public AttributeId Next()
+    {
+        return new((ushort)(Value + 1));
+    }
 }

src/NexusMods.MnemonicDB.Abstractions/Datom.cs

@@ -57,6 +57,11 @@ public TValue Resolve<TValue, TLowLevel>(Attribute<TValue, TLowLevel> attribute)
     /// </summary>
     public TxId T => Prefix.T;
 
+    /// <summary>
+    /// True if the datom is a retract
+    /// </summary>
+    public bool IsRetract => Prefix.IsRetract;
+
     /// <summary>
     /// Copies the data of this datom onto the heap so it's detached from the current iteration.
     /// </summary>
@@ -77,4 +82,31 @@ public override string ToString()
     {
         return Resolved.ToString()!;
     }
+
+    /// <summary>
+    /// Returns -1 if this datom is less than the other, 0 if they are equal, and 1 if this datom is greater than the other.
+    /// in relation to the given index type.
+    /// </summary>
+    public int Compare(Datom other, IndexType indexType)
+    {
+        switch (indexType)
+        {
+            case IndexType.TxLog:
+                return DatomComparators.TxLogComparator.Compare(RawSpan, other.RawSpan);
+            case IndexType.EAVTCurrent:
+            case IndexType.EAVTHistory:
+                return DatomComparators.EAVTComparator.Compare(RawSpan, other.RawSpan);
+            case IndexType.AEVTCurrent:
+            case IndexType.AEVTHistory:
+                return DatomComparators.AEVTComparator.Compare(RawSpan, other.RawSpan);
+            case IndexType.AVETCurrent:
+            case IndexType.AVETHistory:
+                return DatomComparators.AVETComparator.Compare(RawSpan, other.RawSpan);
+            case IndexType.VAETCurrent:
+            case IndexType.VAETHistory:
+                return DatomComparators.VAETComparator.Compare(RawSpan, other.RawSpan);
+            default:
+                throw new ArgumentOutOfRangeException(nameof(indexType), indexType, "Unknown index type");
+        }
+    }
 }

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/ADatomComparator.cs

@@ -1,4 +1,5 @@
 using System;
+using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 
@@ -31,6 +32,24 @@ public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen)
         return TE.Compare(aPtr, aLen, bPtr, bLen);
     }
 
+    /// <summary>
+    /// Compare two datom spans
+    /// </summary>
+    public static int Compare(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
+    {
+        fixed(byte* aPtr = a)
+        fixed(byte* bPtr = b)
+            return Compare(aPtr, a.Length, bPtr, b.Length);
+    }
+
+    /// <summary>
+    /// Compare two datoms
+    /// </summary>
+    public static int Compare(in Datom a, in Datom b)
+    {
+        return Compare(a.RawSpan, b.RawSpan);
+    }
+
     /// <inheritdoc />
     public int CompareInstance(byte* aPtr, int aLen, byte* bPtr, int bLen)
     {

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AEVComparator.cs

@@ -0,0 +1,8 @@
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.DatomComparators;
+
+/// <summary>
+/// AEV Comparator.
+/// </summary>
+public class AEVComparator : APartialDatomComparator<AComparer, EComparer, ValueComparer>;

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/APartialDatomComparator.cs

@@ -0,0 +1,50 @@
+using System;
+using NexusMods.MnemonicDB.Abstractions.DatomIterators;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.DatomComparators;
+
+/// <summary>
+/// A comparator that only considers the EAV portion of the datom, useful for in-memory sets that
+/// are not concerned with time, and only contain asserts
+/// </summary>
+public abstract unsafe class APartialDatomComparator<TA, TB, TC> : IDatomComparator
+    where TA : IElementComparer
+    where TB : IElementComparer
+    where TC : IElementComparer
+{
+    public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen)
+    {
+        var cmp = TA.Compare(aPtr, aLen, bPtr, bLen);
+        if (cmp != 0) return cmp;
+
+        cmp = TB.Compare(aPtr, aLen, bPtr, bLen);
+        if (cmp != 0) return cmp;
+
+        return TC.Compare(aPtr, aLen, bPtr, bLen);
+    }
+
+    /// <summary>
+    /// Compare two datom spans
+    /// </summary>
+    public static int Compare(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
+    {
+        fixed(byte* aPtr = a)
+        fixed(byte* bPtr = b)
+            return Compare(aPtr, a.Length, bPtr, b.Length);
+    }
+
+    /// <summary>
+    /// Compare two datoms
+    /// </summary>
+    public static int Compare(in Datom a, in Datom b)
+    {
+        return Compare(a.RawSpan, b.RawSpan);
+    }
+
+    /// <inheritdoc />
+    public int CompareInstance(byte* aPtr, int aLen, byte* bPtr, int bLen)
+    {
+        return Compare(aPtr, aLen, bPtr, bLen);
+    }
+}

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AVEComparator.cs

@@ -0,0 +1,8 @@
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.DatomComparators;
+
+/// <summary>
+/// AVE Comparator.
+/// </summary>
+public class AVEComparator : APartialDatomComparator<AComparer, ValueComparer, EComparer>;

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/EAVComparator.cs

@@ -0,0 +1,8 @@
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.DatomComparators;
+
+/// <summary>
+/// EAV Comparator.
+/// </summary>
+public class EAVComparator : APartialDatomComparator<EComparer, AComparer, ValueComparer>;

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/IDatomComparator.cs

@@ -23,7 +23,7 @@ public unsafe interface IDatomComparator
     public int CompareInstance(ReadOnlySpan<byte> a, ReadOnlySpan<byte> b)
     {
         fixed(byte* aPtr = a)
-            fixed(byte* bPtr = b)
-                return CompareInstance(aPtr, a.Length, bPtr, b.Length);
+        fixed(byte* bPtr = b)
+            return CompareInstance(aPtr, a.Length, bPtr, b.Length);
     }
 }

src/NexusMods.MnemonicDB.Abstractions/DatomComparators/VAEComparator.cs

@@ -0,0 +1,8 @@
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+
+namespace NexusMods.MnemonicDB.Abstractions.DatomComparators;
+
+/// <summary>
+/// VAE Comparator.
+/// </summary>
+public class VAEComparator : APartialDatomComparator<ValueComparer, AComparer, EComparer>;

src/NexusMods.MnemonicDB.Abstractions/DynamicCache.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
+using DynamicData;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.MnemonicDB.Abstractions;
+
+public class DynamicCache : IDynamicCache
+{
+    public IObservable<IChangeSet<Attribute<THighLevel, TLowLevel>.ReadDatom, EntityId>> Query<TModel, THighLevel, TLowLevel>(Attribute<THighLevel, TLowLevel> attr, THighLevel value)
+    {
+        throw new NotImplementedException();
+    }
+}

src/NexusMods.MnemonicDB.Abstractions/IAnalytics.cs

@@ -0,0 +1,15 @@
+using System.Collections.Frozen;
+
+namespace NexusMods.MnemonicDB.Abstractions;
+
+/// <summary>
+/// Database analytics, attached to each IDb instance but often calculated on-the fly
+/// and cached.
+/// </summary>
+public interface IAnalytics
+{
+    /// <summary>
+    /// All the entities referenced in the most recent transaction of the database.
+    /// </summary>
+    public FrozenSet<EntityId> LatestTxIds { get; }
+}

src/NexusMods.MnemonicDB.Abstractions/IConnection.cs

@@ -1,7 +1,25 @@
 using System;
+using NexusMods.MnemonicDB.Abstractions.IndexSegments;
+using NexusMods.MnemonicDB.Abstractions.Internals;
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
+/// <summary>
+/// A database revision, which includes a datom and the datoms added to it.
+/// </summary>
+public struct Revision
+{
+    /// <summary>
+    /// The database for the most recent transaction
+    /// </summary>
+    public IDb Database;
+
+    /// <summary>
+    /// The datoms that were added in the most recent transaction
+    /// </summary>
+    public IndexSegment AddedDatoms;
+}
+
 /// <summary>
 ///     Represents a connection to a database.
 /// </summary>
@@ -12,6 +30,11 @@ public interface IConnection
     /// </summary>
     public IDb Db { get; }
 
+    /// <summary>
+    /// The attribute registry for this connection
+    /// </summary>
+    public IAttributeRegistry Registry { get; }
+
     /// <summary>
     ///     Gets the most recent transaction id.
     /// </summary>
@@ -20,7 +43,7 @@ public interface IConnection
     /// <summary>
     ///     A sequential stream of database revisions.
     /// </summary>
-    public IObservable<IDb> Revisions { get; }
+    public IObservable<Revision> Revisions { get; }
 
     /// <summary>
     /// A service provider that entities can use to resolve their values

src/NexusMods.MnemonicDB.Abstractions/IDatomStore.cs

@@ -14,7 +14,8 @@ namespace NexusMods.MnemonicDB.Abstractions;
 public interface IDatomStore : IDisposable
 {
     /// <summary>
-    ///     An observable of the transaction log, for getting the latest changes to the store.
+    ///     An observable of the transaction log, for getting the latest changes to the store. This observable
+    /// will always start with the most recent value, so there is no reason to use `StartWith` or `Replay` on it.
     /// </summary>
     public IObservable<(TxId TxId, ISnapshot Snapshot)> TxLog { get; }
 

src/NexusMods.MnemonicDB.Abstractions/IDb.cs

@@ -5,6 +5,7 @@
 using NexusMods.MnemonicDB.Abstractions.IndexSegments;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 using NexusMods.MnemonicDB.Abstractions.Models;
+using NexusMods.MnemonicDB.Abstractions.Query;
 
 namespace NexusMods.MnemonicDB.Abstractions;
 
@@ -33,6 +34,11 @@ public interface IDb : IEquatable<IDb>
     /// </summary>
     IAttributeRegistry Registry { get; }
 
+    /// <summary>
+    /// Analytics for the database.
+    /// </summary>
+    IAnalytics Analytics { get; }
+
     /// <summary>
     ///     Gets a read model for the given entity id.
     /// </summary>
@@ -55,8 +61,16 @@ public TModel Get<TModel>(EntityId id)
     public Entities<EntityIds, TModel> GetReverse<TModel>(EntityId id, Attribute<EntityId, ulong> attribute)
         where TModel : IHasEntityIdAndDb;
 
+    /// <summary>
+    /// Get all the datoms for the given entity id.
+    /// </summary>
     public IEnumerable<IReadDatom> Datoms(EntityId id);
 
+    /// <summary>
+    /// Get all the datoms for the given slice descriptor.
+    /// </summary>
+    public IndexSegment Datoms(SliceDescriptor sliceDescriptor);
+
     /// <summary>
     ///     Gets the datoms for the given transaction id.
     /// </summary>