Merge pull request #868 from azist/wip

Wip #861 Facts, Type Hints, Bixon

src/Azos.Sky.Server/Chronicle/Server/ChronicleServerLogic.cs

@@ -208,6 +208,18 @@ public async Task WriteAsync(LogBatch data)
     public async Task<IEnumerable<Message>> GetAsync(LogChronicleFilter filter)
       => await m_Log.NonNull().GetAsync(filter.NonNull(nameof(filter))).ConfigureAwait(false);
 
+    //20230515 DKh #861
+    public async Task<IEnumerable<Fact>> GetFactsAsync(LogChronicleFactFilter filter)
+    {
+      var messages = await m_Log.NonNull().GetAsync(filter.NonNull(nameof(filter)).LogFilter.NonNull(nameof(filter.LogFilter))).ConfigureAwait(false);
+
+      //Server-side convert to facts
+      var facts = messages.Select(one => ArchiveConventions.LogMessageToFact(one))
+                          .ToArray();//materialize, as conversion is costly
+
+      return facts;
+    }
+
     public async Task WriteAsync(InstrumentationBatch data)
       => await m_Instrumentation.NonNull().WriteAsync(data.NonNull(nameof(data))).ConfigureAwait(false);
 

src/Azos.Sky.Server/Chronicle/Server/Web/ConsumptionControllers.cs

@@ -66,6 +66,19 @@ string esc(string s)
     [ChroniclePermission(ChronicleAccessLevel.Browse)]
     public async Task<object> Filter(LogChronicleFilter filter) => await ApplyFilterAsync(filter).ConfigureAwait(false);
 
+    [ApiEndpointDoc(Title = "Filter Facts",
+                    Uri = "filter-facts",
+                    Description = "Queries log chronicle by applying a structured filter `{@LogChronicleFactFilter}` and returning facts",
+                    Methods = new[] { "POST = post filter object for query execution" },
+                    RequestHeaders = new[] { API_DOC_HDR_ACCEPT_JSON },
+                    ResponseHeaders = new[] { API_DOC_HDR_NO_CACHE },
+                    RequestBody = "JSON representation of `{@LogChronicleFilter}`",
+                    ResponseContent = "JSON filter result - enumerable of `{@Fact}`",
+                    TypeSchemas = new[] { typeof(Fact) })]
+    [ActionOnPost(Name = "filter-facts"), AcceptsJson]
+    [ChroniclePermission(ChronicleAccessLevel.Browse)]
+    public async Task<object> Filter_Facts(LogChronicleFactFilter filter) => await ApplyFilterAsync(filter).ConfigureAwait(false);
+
     [ApiEndpointDoc(Title = "Batch",
                     Uri = "batch",
                     Description = "Uploads a batch of log messages to log chronicle using `{@LogBatch}`",

src/Azos.Sky/Chronicle/ChronicleWebClientLogic.cs

@@ -92,6 +92,11 @@ public async Task<IEnumerable<Message>> GetAsync(LogChronicleFilter filter)
      => await (filter.NonNull(nameof(filter)).CrossShard ? getCrossShard(filter)
                                                          : getOneShard(filter)).ConfigureAwait(false);
 
+    //20230515 DKh #861
+    public async Task<IEnumerable<Fact>> GetFactsAsync(LogChronicleFactFilter filter)
+         => await (filter.NonNull(nameof(filter)).LogFilter.NonNull(nameof(filter.LogFilter)).CrossShard ? getFactsCrossShard(filter)
+                                                             : getFactsOneShard(filter)).ConfigureAwait(false);
+
     private async Task<IEnumerable<Message>> getCrossShard(LogChronicleFilter filter)
     {
       filter.CrossShard = false; //stop recursion, each shard should return just its own data
@@ -134,6 +139,48 @@ private async Task<IEnumerable<Message>> getOneShard(LogChronicleFilter filter)
       return result;
     }
 
+    private async Task<IEnumerable<Fact>> getFactsCrossShard(LogChronicleFactFilter filter)
+    {
+      filter.LogFilter.CrossShard = false; //stop recursion, each shard should return just its own data
+      var shards = m_Server.GetEndpointsForAllShards(LogServiceAddress, nameof(ILogChronicle));
+
+      var calls = shards.Select(shard => shard.Call((http, ct) => http.Client.PostAndGetJsonMapAsync("filter-facts", new { filter = filter })));
+
+      var responses = await Task.WhenAll(calls.Select(async call => {
+        try
+        {
+          return await call.ConfigureAwait(false);
+        }
+        catch (Exception error)
+        {
+          WriteLog(MessageType.Warning, nameof(getCrossShard), "Shard fetch error: " + error.ToMessageWithType(), error);
+          return null;
+        }
+      })).ConfigureAwait(false);
+
+      var result = responses.SelectMany(response => response.UnwrapPayloadArray()
+                                                            .OfType<JsonDataMap>()
+                                                            .Select(imap => JsonReader.ToDoc<Fact>(imap)))
+                            .OrderBy(m => m.UtcTimestamp)
+                            .ToArray();
+
+      return result;
+    }
+
+    private async Task<IEnumerable<Fact>> getFactsOneShard(LogChronicleFactFilter filter)
+    {
+      var response = await m_Server.Call(LogServiceAddress,
+                                          nameof(ILogChronicle),
+                                          new ShardKey(0u),
+                                          (http, ct) => http.Client.PostAndGetJsonMapAsync("filter-facts", new { filter = filter })).ConfigureAwait(false);
+
+      var result = response.UnwrapPayloadArray()
+              .OfType<JsonDataMap>()
+              .Select(imap => JsonReader.ToDoc<Fact>(imap));
+
+      return result;
+    }
+
     public async Task WriteAsync(InstrumentationBatch data)
     {
       var response = await m_Server.Call(InstrumentationServiceAddress,

src/Azos.Sky/Chronicle/ILogChronicle.cs

@@ -27,6 +27,13 @@ public interface ILogChronicle
     /// Gets chronicle (a list) of messages satisfying the supplied LogChronicleFilter object
     /// </summary>
     Task<IEnumerable<Message>> GetAsync(LogChronicleFilter filter);
+
+    //20230515 DKh #861
+    /// <summary>
+    /// Gets chronicle (a list) of facts extracted from log messages on the server satisfying
+    /// the supplied LogChronicleFactFilter object
+    /// </summary>
+    Task<IEnumerable<Fact>> GetFactsAsync(LogChronicleFactFilter filter);
   }
 
   /// <summary>

src/Azos.Sky/Chronicle/LogChronicleFactFilter.cs

@@ -0,0 +1,32 @@
+/*<FILE_LICENSE>
+ * Azos (A to Z Application Operating System) Framework
+ * The A to Z Foundation (a.k.a. Azist) licenses this file to you under the MIT license.
+ * See the LICENSE file in the project root for more information.
+</FILE_LICENSE>*/
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+using Azos.Apps.Injection;
+using Azos.Data;
+using Azos.Data.Business;
+using Azos.Log;
+using Azos.Serialization.Bix;
+
+namespace Azos.Sky.Chronicle
+{
+  [Bix("a06fb712-7e64-435e-8325-f28d67a544e3")]
+  [Schema(Description = "Provides model for filtering log chronicles for facts")]
+  public sealed class LogChronicleFactFilter : FilterModel<IEnumerable<Fact>>
+  {
+    [Field(required: true, description: "Log message filter which facts are based on")]
+    public LogChronicleFilter LogFilter { get; set; }
+
+    [InjectModule] ILogChronicle m_Chronicle;
+
+    protected async override Task<SaveResult<IEnumerable<Fact>>> DoSaveAsync()
+     => new SaveResult<IEnumerable<Fact>>(await m_Chronicle.GetFactsAsync(this).ConfigureAwait(false));
+  }
+
+}

src/Azos/Apps/typeid/GuidTypeAttribute.cs

@@ -52,9 +52,7 @@ public static TAttribute TryGetGuidTypeAttribute<TDecorationTarget, TAttribute>(
     }
 
     private static FiniteSetLookup<Type, FiniteSetLookup<Type, GuidTypeAttribute>> s_Cache =
-      new FiniteSetLookup<Type, FiniteSetLookup<Type, GuidTypeAttribute>>(
-        ttarget => new FiniteSetLookup<Type, GuidTypeAttribute>( tattr => ttarget.GetCustomAttribute(tattr, false) as GuidTypeAttribute)
-    );
+      new (ttarget => new FiniteSetLookup<Type, GuidTypeAttribute>( tattr => ttarget.GetCustomAttribute(tattr, false) as GuidTypeAttribute));
 
 
     protected GuidTypeAttribute(string typeGuid)

src/Azos/Atom.cs

@@ -281,6 +281,9 @@ public string Value
       }
     }
 
+    /// <summary>Returns default if this is zero </summary>
+    public Atom Default(Atom dflt) => this.IsZero ? dflt : this;
+
     public bool Equals(Atom other) => this.ID == other.ID;
     public override bool Equals(object obj)
     {

src/Azos/CoreConsts.cs

@@ -164,6 +164,7 @@ public static class CoreConsts
       public const string TOPIC_ID_GEN = "idg";
 
       public static readonly Atom LOG_CHANNEL_SECURITY = Atom.Encode("sec");
+      public static readonly Atom LOG_CHANNEL_ANALYTICS = Atom.Encode("anl");
 
       public const string ISO_LANG_ENGLISH = "eng";
       public const string ISO_LANG_RUSSIAN = "rus";

src/Azos/CoreUtils.cs

@@ -15,6 +15,7 @@
 using Azos.Serialization.JSON;
 using System.Threading.Tasks;
 using Azos.Data.Idgen;
+using System.Linq;
 
 namespace Azos
 {
@@ -228,6 +229,22 @@ public static bool IsValidXMLName(this string name)
       return true;
     }
 
+    //#861
+    private static readonly Platform.FiniteSetLookup<Type, bool> ANONYMOUS_TYPES = new( t =>
+      t.BaseType == typeof(object) &&
+      t.Namespace.IsNullOrWhiteSpace() &&
+      t.GetCustomAttributes<CompilerGeneratedAttribute>().Any());
+
+    /// <summary>
+    /// Returns true if the type is anonymous akin to: `new {p=v}`
+    /// </summary>
+    public static bool IsAnonymousType(this Type t)
+    {
+      if (t==null) return false;
+      return ANONYMOUS_TYPES[t];
+    }
+
+
     /// <summary>
     /// Searches an Exception and its InnerException chain for the first instance of T or null.
     /// The original instance may be null itself in which case null is returned

src/Azos/Data/TypedDoc.cs

@@ -92,6 +92,11 @@ protected AmorphousTypedDoc()
     /// </summary>
     public virtual bool AmorphousDataEnabled => true;
 
+    /// <summary>
+    /// True if amorphous data is allocated (not null)
+    /// </summary>
+    public bool HasAmorphousData => m_AmorphousData != null;
+
     /// <summary>
     /// Returns data that does not comply with known schema (dynamic data). The field names are NOT case-sensitive
     /// </summary>
@@ -100,12 +105,17 @@ public IDictionary<string, object> AmorphousData
       get
       {
         if (m_AmorphousData == null)
+        {
           m_AmorphousData = new Dictionary<string, object>(StringComparer.OrdinalIgnoreCase);
+        }
 
         return m_AmorphousData;
       }
     }
 
+    /// <summary> Sets whole amorphous data object at once </summary>
+    public void SetAmorphousData(Dictionary<string, object> data) => m_AmorphousData = data;
+
     void IAmorphousData.BeforeSave(string targetName) => DoAmorphousDataBeforeSave(targetName);
     void IAmorphousData.AfterLoad(string targetName) => DoAmorphousDataAfterLoad(targetName);
 

src/Azos/IO/Archiving/accessors/FactArchiveAppender.cs

@@ -0,0 +1,51 @@
+/*<FILE_LICENSE>
+ * Azos (A to Z Application Operating System) Framework
+ * The A to Z Foundation (a.k.a. Azist) licenses this file to you under the MIT license.
+ * See the LICENSE file in the project root for more information.
+</FILE_LICENSE>*/
+
+using System;
+using Azos.Data;
+using Azos.Log;
+using Azos.Serialization.Bix;
+using Azos.Time;
+
+namespace Azos.IO.Archiving
+{
+  /// <summary>
+  /// Appends into Azos fact binary archives
+  /// </summary>
+  [ContentTypeSupport(CONTENT_TYPE_FACTS)]
+  public sealed class FactArchiveAppender : ArchiveBixAppender<Fact>
+  {
+    public const string CONTENT_TYPE_FACTS = "bix/azfacts";
+
+    public FactArchiveAppender(IVolume volume, ITimeSource time, Atom app, string host, Action<Fact, Bookmark> onPageCommit = null)
+     : base(volume, time, app, host, onPageCommit){ }
+
+    protected override void DoSerializeBix(BixWriter wri, Fact entry)
+    {
+      if (entry == null)
+      {
+        wri.Write(false); //NULL
+        return;
+      }
+
+      wri.Write(true); // NON-NULL
+      wri.Write(entry.FactType);
+      wri.Write(entry.Id);
+      wri.Write(entry.RelatedId == Guid.Empty ? (Guid?)null : entry.RelatedId);//nullable Guid takes 1 byte instead of 16
+      wri.Write(entry.Gdid.IsZero ? (GDID?)null : entry.Gdid);//nullable Gdid will consume 1 byte instead of 12 zeros
+      wri.Write(entry.Channel);
+      wri.Write(entry.Topic);
+      wri.Write(entry.Host);
+      wri.Write(entry.App);
+      wri.Write((int)entry.RecordType);
+      wri.Write(entry.Source);
+      wri.Write(entry.UtcTimestamp);
+      Bixon.WriteObject(wri, entry.HasAmorphousData ? entry.AmorphousData : null);
+      Bixon.WriteObject(wri, entry.Dimensions);
+      Bixon.WriteObject(wri, entry.Metrics);
+    }
+  }
+}

src/Azos/IO/Archiving/accessors/FactArchiveReader.cs

@@ -0,0 +1,81 @@
+/*<FILE_LICENSE>
+ * Azos (A to Z Application Operating System) Framework
+ * The A to Z Foundation (a.k.a. Azist) licenses this file to you under the MIT license.
+ * See the LICENSE file in the project root for more information.
+</FILE_LICENSE>*/
+
+using System;
+
+using Azos.Data;
+using Azos.Log;
+using Azos.Serialization.Bix;
+using Azos.Serialization.JSON;
+
+namespace Azos.IO.Archiving
+{
+  /// <summary>
+  /// Reads archives of Log.Fact items. The implementation is thread-safe
+  /// </summary>
+  [ContentTypeSupport(FactArchiveAppender.CONTENT_TYPE_FACTS)]
+  public sealed class FactArchiveReader : ArchiveBixReader<Fact>
+  {
+    public FactArchiveReader(IVolume volume, Func<Fact, Fact> factory = null) : base(volume)
+    {
+      Factory = factory;
+    }
+
+    public readonly Func<Fact, Fact> Factory;
+
+    [ThreadStatic]
+    private static Fact ts_FactCache;
+
+    public override Fact MaterializeBix(BixReader reader)
+    {
+      if (!reader.ReadBool()) return null;
+
+      Fact fact = null;
+
+      if (Factory != null)
+      {
+        fact = ts_FactCache;
+        if (fact == null)
+        {
+          fact = new Fact();
+        }
+      }
+      else
+      {
+        fact = new Fact();
+      }
+
+      fact.FactType = reader.ReadAtom();
+      fact.Id = reader.ReadGuid();
+
+      var guid = reader.ReadNullableGuid();
+      fact.RelatedId = guid.HasValue ? guid.Value : Guid.Empty;
+
+      var gdid = reader.ReadNullableGDID();
+      fact.Gdid = gdid.HasValue ? gdid.Value : GDID.ZERO;
+
+      fact.Channel = reader.ReadAtom();
+      fact.Topic = reader.ReadAtom();
+      fact.Host = reader.ReadString();
+      fact.App = reader.ReadAtom();
+      fact.RecordType = (MessageType)reader.ReadInt();
+      fact.Source = reader.ReadInt();
+      fact.UtcTimestamp = reader.ReadDateTime();
+      fact.SetAmorphousData(Bixon.ReadObject(reader) as JsonDataMap);
+      fact.Dimensions = Bixon.ReadObject(reader) as JsonDataMap;
+      fact.Metrics = Bixon.ReadObject(reader) as JsonDataMap;
+
+      if (Factory != null)
+      {
+        var result = Factory(fact);
+        ts_FactCache = object.ReferenceEquals(result, fact) ? null : fact;
+        return result;
+      }
+
+      return fact;
+    }
+  }
+}