From 06ef41dfee3a2bda1834d2785fe6844e514cc7dc Mon Sep 17 00:00:00 2001 From: Dmitry Katson Date: Tue, 26 Nov 2024 04:05:36 -0600 Subject: [PATCH] [Number Series Copilot] Adding Telemetry (#2254) #### Summary Adding telemetry to the number series copilot: - usage statistics - what tool was used - how many number series where generated - did user specified patterns - did user specified entities - how many retries were done - did the user applied the generated number series - areas for which user generated number series - entities for which user generated number series #### Work Item(s) Fixes #2252 Fixes [AB#558485](https://dynamicssmb2.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_workitems/edit/558485) --------- Co-authored-by: Jesper Schulz-Wedde --- .../Copilot/NoSeriesCopilotImpl.Codeunit.al | 18 +- .../src/Copilot/NoSeriesGeneration.Page.al | 7 + .../NoSeriesCopilotTelemetry.Codeunit.al | 219 ++++++++++++++++++ .../Tools/NoSeriesCopAddIntent.Codeunit.al | 2 + .../Tools/NoSeriesCopChangeIntent.Codeunit.al | 2 + 5 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al diff --git a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesCopilotImpl.Codeunit.al b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesCopilotImpl.Codeunit.al index 371bacae17..ac3c7f55fc 100644 --- a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesCopilotImpl.Codeunit.al +++ b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesCopilotImpl.Codeunit.al @@ -18,6 +18,7 @@ codeunit 324 "No. Series Copilot Impl." InherentEntitlements = X; var + NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry"; IncorrectCompletionErr: Label 'Incorrect completion. The property %1 is empty', Comment = '%1 = property name'; EmptyCompletionErr: Label 'Incorrect completion. The completion is empty.'; IncorrectCompletionNumberOfGeneratedNoSeriesErr: Label 'Incorrect completion. The number of generated number series is incorrect. Expected %1, but got %2', Comment = '%1 = Expected Number, %2 = Actual Number'; @@ -33,7 +34,6 @@ codeunit 324 "No. Series Copilot Impl." procedure GetNoSeriesSuggestions() var - FeatureTelemetry: Codeunit "Feature Telemetry"; NoSeriesCopilotRegister: Codeunit "No. Series Copilot Register"; AzureOpenAI: Codeunit "Azure OpenAI"; begin @@ -41,7 +41,7 @@ codeunit 324 "No. Series Copilot Impl." if not AzureOpenAI.IsEnabled(Enum::"Copilot Capability"::"No. Series Copilot") then exit; - FeatureTelemetry.LogUptake('0000LF4', FeatureName(), Enum::"Feature Uptake Status"::Discovered); + NoSeriesCopilotTelemetry.LogFeatureDiscovery(); Page.Run(Page::"No. Series Generation"); end; @@ -184,14 +184,19 @@ codeunit 324 "No. Series Copilot Impl." AOAIChatMessages.AddTool(ChangeNoSeriesIntent); AOAIChatMessages.AddTool(NextYearNoSeriesIntent); + NoSeriesCopilotTelemetry.ResetDurationTracking(); + NoSeriesCopilotTelemetry.StartDurationTracking(); AzureOpenAI.GenerateChatCompletion(AOAIChatMessages, AOAIChatCompletionParams, AOAIOperationResponse); + NoSeriesCopilotTelemetry.StopDurationTracking(); if not AOAIOperationResponse.IsSuccess() then Error(AOAIOperationResponse.GetError()); CompletionAnswerTxt := AOAIChatMessages.GetLastMessage(); // the model can answer to rephrase the question, if the user input is not clear if AOAIOperationResponse.IsFunctionCall() then - CompletionAnswerTxt := GenerateNoSeriesUsingToolResult(AzureOpenAI, InputText, AOAIOperationResponse, AddNoSeriesIntent.GetExistingNoSeries()); + CompletionAnswerTxt := GenerateNoSeriesUsingToolResult(AzureOpenAI, InputText, AOAIOperationResponse, AddNoSeriesIntent.GetExistingNoSeries()) + else + NoSeriesCopilotTelemetry.LogToolNotInvoked(AOAIOperationResponse); exit(CompletionAnswerTxt); end; @@ -260,7 +265,10 @@ codeunit 324 "No. Series Copilot Impl." begin MaxAttempts := 3; for Attempt := 1 to MaxAttempts do begin + NoSeriesCopilotTelemetry.ResetDurationTracking(); + NoSeriesCopilotTelemetry.StartDurationTracking(); AzureOpenAI.GenerateChatCompletion(AOAIChatMessages, AOAIChatCompletionParams, AOAIOperationResponse); + NoSeriesCopilotTelemetry.StopDurationTracking(); if not AOAIOperationResponse.IsSuccess() then Error(AOAIOperationResponse.GetError()); @@ -272,8 +280,10 @@ codeunit 324 "No. Series Copilot Impl." Error(AOAIFunctionResponse.GetError()); GeneratedNoSeriesArrayText := AOAIFunctionResponse.GetResult(); - if CheckIfValidResult(GeneratedNoSeriesArrayText, AOAIFunctionResponse.GetFunctionName(), ExpectedNoSeriesCount) then + if CheckIfValidResult(GeneratedNoSeriesArrayText, AOAIFunctionResponse.GetFunctionName(), ExpectedNoSeriesCount) then begin + NoSeriesCopilotTelemetry.LogGenerationCompletion(ReadGeneratedNumberSeriesJArray(GeneratedNoSeriesArrayText).Count, ExpectedNoSeriesCount, Attempt); exit(true); + end; AOAIChatMessages.DeleteMessage(AOAIChatMessages.GetHistory().Count); // remove the last message with wrong assistant response, as we need to regenerate the completion Sleep(500); diff --git a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesGeneration.Page.al b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesGeneration.Page.al index e4db90211b..ad226b6fed 100644 --- a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesGeneration.Page.al +++ b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/NoSeriesGeneration.Page.al @@ -171,6 +171,7 @@ page 332 "No. Series Generation" } var + NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry"; InputText: Text; PageCaptionLbl: text; IsGenerationDetailsVisible: Boolean; @@ -191,6 +192,8 @@ page 332 "No. Series Generation" begin if CloseAction = CloseAction::OK then ApplyGeneratedNoSeries(); + + NoSeriesCopilotTelemetry.LogFeatureUsage(); end; local procedure GenerateNoSeries() @@ -198,7 +201,10 @@ page 332 "No. Series Generation" GeneratedNoSeries: Record "No. Series Generation Detail"; NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; begin + NoSeriesCopilotTelemetry.StartDurationTracking(); NoSeriesCopilotImpl.Generate(Rec, GeneratedNoSeries, InputText); + NoSeriesCopilotTelemetry.StopDurationTracking(); + NoSeriesCopilotTelemetry.SaveTotalSuggestedLines(GeneratedNoSeries.Count()); CurrPage.GenerationDetails.Page.Load(GeneratedNoSeries); IsGenerationDetailsVisible := not GeneratedNoSeries.IsEmpty; end; @@ -210,5 +216,6 @@ page 332 "No. Series Generation" begin CurrPage.GenerationDetails.Page.GetTempRecord(Rec."No.", GeneratedNoSeries); NoSeriesCopilotImpl.ApplyGeneratedNoSeries(GeneratedNoSeries); + NoSeriesCopilotTelemetry.LogApply(GeneratedNoSeries); end; } \ No newline at end of file diff --git a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al new file mode 100644 index 0000000000..ca3774eae6 --- /dev/null +++ b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al @@ -0,0 +1,219 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace Microsoft.Foundation.NoSeries; +using System.Telemetry; +using System.AI; + +codeunit 389 "No. Series Copilot Telemetry" +{ + Access = Internal; + InherentPermissions = X; + InherentEntitlements = X; + + var + StartDateTime: DateTime; + Durations: List of [Duration]; // Generate action can be triggered multiple times + TotalSuggestedLines: List of [Integer]; // Generate action can be triggered multiple times + TotalAppliedLines: Integer; + + procedure LogFeatureDiscovery() + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + begin + FeatureTelemetry.LogUptake('0000LF4', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Discovered); + FeatureTelemetry.LogUptake('0000O9D', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::"Set up"); + end; + + procedure LogApply(GeneratedNoSeries: Record "No. Series Generation Detail") + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + begin + TotalAppliedLines := GeneratedNoSeries.Count(); + if TotalAppliedLines = 0 then + exit; + + FeatureTelemetry.LogUptake('0000O9E', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Used, GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries)); + end; + + procedure LogCreateNewNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer) + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + NoSeriesCopAddIntent: Codeunit "No. Series Cop. Add Intent"; + begin + FeatureTelemetry.LogUsage('0000O9F', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopAddIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); + end; + + procedure LogModifyExistingNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer; UpdateForNextYear: Boolean) + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + NoSeriesCopChangeIntent: Codeunit "No. Series Cop. Change Intent"; + NoSeriesCopNxtYrIntent: Codeunit "No. Series Cop. Nxt Yr. Intent"; + begin + if UpdateForNextYear then + FeatureTelemetry.LogUsage('0000O9G', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopNxtYrIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)) + else + FeatureTelemetry.LogUsage('0000O9H', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopChangeIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); + end; + + local procedure GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer) CustomDimension: Dictionary of [Text, Text] + begin + CustomDimension.Add('TotalUserSpecifiedEntities', Format(TotalUserSpecifiedEntities)); + CustomDimension.Add('CustomPatternsUsed', Format(CustomPatternsUsed)); + CustomDimension.Add('TotalBatches', Format(TotalBatches)); + CustomDimension.Add('TotalFoundTables', Format(TotalFoundTables)); + end; + + procedure LogFeatureUsage() + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + begin + // TotalAppliedLines will be zero in case none of the lines were inserted. + // We don't want to log telemetry in case the user did not generate any suggestions. + if Durations.Count() = 0 then + exit; + + FeatureTelemetry.LogUsage('0000O9I', NoSeriesCopilotImpl.FeatureName(), 'Statistics', GetFeatureTelemetryCustomDimensions()); + end; + + procedure LogGenerationCompletion(TotalGeneratedLines: Integer; TotalExpectedLines: Integer; Attempt: Integer) + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + TelemetryCD: Dictionary of [Text, Text]; + begin + TelemetryCD.Add('TotalGeneratedLines', Format(TotalGeneratedLines)); + TelemetryCD.Add('TotalExpectedLines', Format(TotalExpectedLines)); + TelemetryCD.Add('Attempt', Format(Attempt)); + TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations)); + + FeatureTelemetry.LogUsage('0000O9J', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', TelemetryCD); + end; + + + local procedure GetFeatureTelemetryCustomDimensions() CustomDimension: Dictionary of [Text, Text] + begin + CustomDimension.Add('Durations', ConvertListOfDurationToString(Durations)); + CustomDimension.Add('TotalSuggestedLines', ConvertListOfIntegerToString(TotalSuggestedLines)); + CustomDimension.Add('TotalAppliedLines', Format(TotalAppliedLines)); + end; + + local procedure GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries: Record "No. Series Generation Detail") CustomDimension: Dictionary of [Text, Text] + var + AppliedValues: Text; + TotalApplied: Integer; + begin + GetAppliedAreas(GeneratedNoSeries, AppliedValues, TotalApplied); + CustomDimension.Add('AppliedAreas', AppliedValues); + CustomDimension.Add('TotalAppliedAreas', Format(TotalApplied)); + + GetAppliedEntities(GeneratedNoSeries, AppliedValues, TotalApplied); + CustomDimension.Add('AppliedEntities', AppliedValues); + CustomDimension.Add('TotalAppliedEntities', Format(TotalApplied)); + end; + + local procedure GetAppliedAreas(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedAreas: Text; var TotalAppliedAreas: Integer) + var + AppliedArea: List of [Text]; + begin + Clear(AppliedArea); + Clear(TotalAppliedAreas); + if GeneratedNoSeries.FindSet() then + repeat + if not AppliedArea.Contains(GeneratedNoSeries."Setup Table Name") then + AppliedArea.Add(GeneratedNoSeries."Setup Table Name"); + until GeneratedNoSeries.Next() = 0; + + AppliedAreas := ConvertListOfTextToString(AppliedArea); + TotalAppliedAreas := AppliedArea.Count(); + end; + + local procedure GetAppliedEntities(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedEntities: Text; var TotalAppliedEntities: Integer) + var + AppliedEntity: List of [Text]; + begin + Clear(AppliedEntity); + Clear(TotalAppliedEntities); + if GeneratedNoSeries.FindSet() then + repeat + if not AppliedEntity.Contains(GeneratedNoSeries."Setup Field Name") then + AppliedEntity.Add(GeneratedNoSeries."Setup Field Name"); + until GeneratedNoSeries.Next() = 0; + + AppliedEntities := ConvertListOfTextToString(AppliedEntity); + TotalAppliedEntities := AppliedEntity.Count(); + end; + + local procedure ConvertListOfDurationToString(ListOfDuration: List of [Duration]) Result: Text + var + Dur: Duration; + DurationAsBigInt: BigInteger; + begin + foreach Dur in ListOfDuration do begin + DurationAsBigInt := Dur; + Result += Format(DurationAsBigInt) + ', '; + end; + Result := Result.TrimEnd(', '); + end; + + local procedure ConvertListOfIntegerToString(ListOfInteger: List of [Integer]) Result: Text + var + Int: Integer; + begin + foreach Int in ListOfInteger do + Result += Format(Int) + ', '; + Result := Result.TrimEnd(', '); + end; + + local procedure ConvertListOfTextToString(ListOfText: List of [Text]) Result: Text + var + Text: Text; + begin + foreach Text in ListOfText do + Result += Text + ', '; + Result := Result.TrimEnd(', '); + end; + + procedure LogToolNotInvoked(AOAIOperationResponse: Codeunit "AOAI Operation Response") + var + FeatureTelemetry: Codeunit "Feature Telemetry"; + NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + TelemetryCD: Dictionary of [Text, Text]; + begin + if Durations.Count() <> 0 then + TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations)); + + if AOAIOperationResponse.GetResult() = '' then + FeatureTelemetry.LogError('0000O9B', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', 'Completion answer is empty', '', TelemetryCD) + else + FeatureTelemetry.LogError('0000O9C', NoSeriesCopilotImpl.FeatureName(), 'Process function_call', 'function_call not found in the completion answer'); + end; + + procedure ResetDurationTracking() + begin + Clear(Durations); + end; + + procedure StartDurationTracking() + begin + StartDateTime := CurrentDateTime(); + end; + + procedure StopDurationTracking() Duration: Duration + begin + Durations.Add(CurrentDateTime() - StartDateTime); + end; + + procedure SaveTotalSuggestedLines(Total: Integer) + begin + TotalSuggestedLines.Add(Total); + end; + +} \ No newline at end of file diff --git a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopAddIntent.Codeunit.al b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopAddIntent.Codeunit.al index 164b896655..0ff0574629 100644 --- a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopAddIntent.Codeunit.al +++ b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopAddIntent.Codeunit.al @@ -60,6 +60,7 @@ codeunit 331 "No. Series Cop. Add Intent" implements "AOAI Function" var TempNoSeriesField: Record "Field" temporary; TempSetupTable: Record "Table Metadata" temporary; + NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry"; NewNoSeriesPrompt, CustomPatternsPromptList, TablesYamlList, EmptyList : List of [Text]; NumberOfToolResponses, i, ActualTablesChunkSize : Integer; NumberOfAddedTables: Integer; @@ -85,6 +86,7 @@ codeunit 331 "No. Series Cop. Add Intent" implements "AOAI Function" ToolResults.Add(ToolsImpl.ConvertListToText(NewNoSeriesPrompt), ActualTablesChunkSize); end; Progress.Close(); + NoSeriesCopilotTelemetry.LogCreateNewNumberSeriesToolUsage(ToolsImpl.GetEntities(Arguments).Count, CustomPatternsPromptList.Count > 0, NumberOfToolResponses, NumberOfAddedTables); end; local procedure GetTablesRequireNoSeries(var Arguments: JsonObject; var TempSetupTable: Record "Table Metadata" temporary; var TempNoSeriesField: Record "Field" temporary) diff --git a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopChangeIntent.Codeunit.al b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopChangeIntent.Codeunit.al index 4c78718f90..73be7c0b0e 100644 --- a/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopChangeIntent.Codeunit.al +++ b/src/Business Foundation/App/NoSeriesCopilot/src/Copilot/Tools/NoSeriesCopChangeIntent.Codeunit.al @@ -69,6 +69,7 @@ codeunit 334 "No. Series Cop. Change Intent" implements "AOAI Function" TempSetupTable: Record "Table Metadata" temporary; TempNoSeriesField: Record "Field" temporary; NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; + NoSeriesCopilotTelemetry: Codeunit "No. Series Copilot Telemetry"; ChangeNoSeriesPrompt, CustomPatternsPromptList, TablesYamlList, ExistingNoSeriesToChangeList : List of [Text]; NumberOfToolResponses, i, ActualTablesChunkSize : Integer; NumberOfChangedTables: Integer; @@ -102,6 +103,7 @@ codeunit 334 "No. Series Cop. Change Intent" implements "AOAI Function" ToolResults.Add(ToolsImpl.ConvertListToText(ChangeNoSeriesPrompt), ActualTablesChunkSize); end; Progress.Close(); + NoSeriesCopilotTelemetry.LogModifyExistingNumberSeriesToolUsage(ToolsImpl.GetEntities(Arguments).Count, CustomPatternsPromptList.Count > 0, NumberOfToolResponses, NumberOfChangedTables, UpdateForNextYear); end; [TryFunction]