-
Notifications
You must be signed in to change notification settings - Fork 172
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Number Series Copilot] Adding Telemetry (#2254)
<!-- Thank you for submitting a Pull Request. If you're new to contributing to BCApps please read our pull request guideline below * https://github.com/microsoft/BCApps/Contributing.md --> #### Summary <!-- Provide a general summary of your changes --> Adding telemetry to the number series copilot: - usage statistics - what tool was used - how many number series where generated - did user specified patterns - did user specified entities - how many retries were done - did the user applied the generated number series - areas for which user generated number series - entities for which user generated number series #### Work Item(s) <!-- Add the issue number here after the #. The issue needs to be open and approved. Submitting PRs with no linked issues or unapproved issues is highly discouraged. --> Fixes #2252 Fixes [AB#558485](https://dynamicssmb2.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_workitems/edit/558485) --------- Co-authored-by: Jesper Schulz-Wedde <[email protected]>
- Loading branch information
1 parent
073ef9a
commit 06ef41d
Showing
5 changed files
with
244 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
219 changes: 219 additions & 0 deletions
219
...Foundation/App/NoSeriesCopilot/src/Copilot/Telemetry/NoSeriesCopilotTelemetry.Codeunit.al
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,219 @@ | ||
| // ------------------------------------------------------------------------------------------------ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. See License.txt in the project root for license information. | ||
| // ------------------------------------------------------------------------------------------------ | ||
|
|
||
| namespace Microsoft.Foundation.NoSeries; | ||
| using System.Telemetry; | ||
| using System.AI; | ||
|
|
||
| codeunit 389 "No. Series Copilot Telemetry" | ||
| { | ||
| Access = Internal; | ||
| InherentPermissions = X; | ||
| InherentEntitlements = X; | ||
|
|
||
| var | ||
| StartDateTime: DateTime; | ||
| Durations: List of [Duration]; // Generate action can be triggered multiple times | ||
| TotalSuggestedLines: List of [Integer]; // Generate action can be triggered multiple times | ||
| TotalAppliedLines: Integer; | ||
|
|
||
| procedure LogFeatureDiscovery() | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| begin | ||
| FeatureTelemetry.LogUptake('0000LF4', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Discovered); | ||
| FeatureTelemetry.LogUptake('0000O9D', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::"Set up"); | ||
| end; | ||
|
|
||
| procedure LogApply(GeneratedNoSeries: Record "No. Series Generation Detail") | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| begin | ||
| TotalAppliedLines := GeneratedNoSeries.Count(); | ||
| if TotalAppliedLines = 0 then | ||
| exit; | ||
|
|
||
| FeatureTelemetry.LogUptake('0000O9E', NoSeriesCopilotImpl.FeatureName(), Enum::"Feature Uptake Status"::Used, GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries)); | ||
| end; | ||
|
|
||
| procedure LogCreateNewNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer) | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| NoSeriesCopAddIntent: Codeunit "No. Series Cop. Add Intent"; | ||
| begin | ||
| FeatureTelemetry.LogUsage('0000O9F', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopAddIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); | ||
| end; | ||
|
|
||
| procedure LogModifyExistingNumberSeriesToolUsage(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer; UpdateForNextYear: Boolean) | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| NoSeriesCopChangeIntent: Codeunit "No. Series Cop. Change Intent"; | ||
| NoSeriesCopNxtYrIntent: Codeunit "No. Series Cop. Nxt Yr. Intent"; | ||
| begin | ||
| if UpdateForNextYear then | ||
| FeatureTelemetry.LogUsage('0000O9G', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopNxtYrIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)) | ||
| else | ||
| FeatureTelemetry.LogUsage('0000O9H', NoSeriesCopilotImpl.FeatureName(), NoSeriesCopChangeIntent.GetName(), GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities, CustomPatternsUsed, TotalBatches, TotalFoundTables)); | ||
| end; | ||
|
|
||
| local procedure GetToolUsageTelemetryCustomDimensions(TotalUserSpecifiedEntities: Integer; CustomPatternsUsed: Boolean; TotalBatches: Integer; TotalFoundTables: Integer) CustomDimension: Dictionary of [Text, Text] | ||
| begin | ||
| CustomDimension.Add('TotalUserSpecifiedEntities', Format(TotalUserSpecifiedEntities)); | ||
| CustomDimension.Add('CustomPatternsUsed', Format(CustomPatternsUsed)); | ||
| CustomDimension.Add('TotalBatches', Format(TotalBatches)); | ||
| CustomDimension.Add('TotalFoundTables', Format(TotalFoundTables)); | ||
| end; | ||
|
|
||
| procedure LogFeatureUsage() | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| begin | ||
| // TotalAppliedLines will be zero in case none of the lines were inserted. | ||
| // We don't want to log telemetry in case the user did not generate any suggestions. | ||
| if Durations.Count() = 0 then | ||
| exit; | ||
|
|
||
| FeatureTelemetry.LogUsage('0000O9I', NoSeriesCopilotImpl.FeatureName(), 'Statistics', GetFeatureTelemetryCustomDimensions()); | ||
| end; | ||
|
|
||
| procedure LogGenerationCompletion(TotalGeneratedLines: Integer; TotalExpectedLines: Integer; Attempt: Integer) | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| TelemetryCD: Dictionary of [Text, Text]; | ||
| begin | ||
| TelemetryCD.Add('TotalGeneratedLines', Format(TotalGeneratedLines)); | ||
| TelemetryCD.Add('TotalExpectedLines', Format(TotalExpectedLines)); | ||
| TelemetryCD.Add('Attempt', Format(Attempt)); | ||
| TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations)); | ||
|
|
||
| FeatureTelemetry.LogUsage('0000O9J', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', TelemetryCD); | ||
| end; | ||
|
|
||
|
|
||
| local procedure GetFeatureTelemetryCustomDimensions() CustomDimension: Dictionary of [Text, Text] | ||
| begin | ||
| CustomDimension.Add('Durations', ConvertListOfDurationToString(Durations)); | ||
| CustomDimension.Add('TotalSuggestedLines', ConvertListOfIntegerToString(TotalSuggestedLines)); | ||
| CustomDimension.Add('TotalAppliedLines', Format(TotalAppliedLines)); | ||
| end; | ||
|
|
||
| local procedure GetFeatureUsedTelemetryCustomDimensions(GeneratedNoSeries: Record "No. Series Generation Detail") CustomDimension: Dictionary of [Text, Text] | ||
| var | ||
| AppliedValues: Text; | ||
| TotalApplied: Integer; | ||
| begin | ||
| GetAppliedAreas(GeneratedNoSeries, AppliedValues, TotalApplied); | ||
| CustomDimension.Add('AppliedAreas', AppliedValues); | ||
| CustomDimension.Add('TotalAppliedAreas', Format(TotalApplied)); | ||
|
|
||
| GetAppliedEntities(GeneratedNoSeries, AppliedValues, TotalApplied); | ||
| CustomDimension.Add('AppliedEntities', AppliedValues); | ||
| CustomDimension.Add('TotalAppliedEntities', Format(TotalApplied)); | ||
| end; | ||
|
|
||
| local procedure GetAppliedAreas(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedAreas: Text; var TotalAppliedAreas: Integer) | ||
| var | ||
| AppliedArea: List of [Text]; | ||
| begin | ||
| Clear(AppliedArea); | ||
| Clear(TotalAppliedAreas); | ||
| if GeneratedNoSeries.FindSet() then | ||
| repeat | ||
| if not AppliedArea.Contains(GeneratedNoSeries."Setup Table Name") then | ||
| AppliedArea.Add(GeneratedNoSeries."Setup Table Name"); | ||
| until GeneratedNoSeries.Next() = 0; | ||
|
|
||
| AppliedAreas := ConvertListOfTextToString(AppliedArea); | ||
| TotalAppliedAreas := AppliedArea.Count(); | ||
| end; | ||
|
|
||
| local procedure GetAppliedEntities(GeneratedNoSeries: Record "No. Series Generation Detail"; var AppliedEntities: Text; var TotalAppliedEntities: Integer) | ||
| var | ||
| AppliedEntity: List of [Text]; | ||
| begin | ||
| Clear(AppliedEntity); | ||
| Clear(TotalAppliedEntities); | ||
| if GeneratedNoSeries.FindSet() then | ||
| repeat | ||
| if not AppliedEntity.Contains(GeneratedNoSeries."Setup Field Name") then | ||
| AppliedEntity.Add(GeneratedNoSeries."Setup Field Name"); | ||
| until GeneratedNoSeries.Next() = 0; | ||
|
|
||
| AppliedEntities := ConvertListOfTextToString(AppliedEntity); | ||
| TotalAppliedEntities := AppliedEntity.Count(); | ||
| end; | ||
|
|
||
| local procedure ConvertListOfDurationToString(ListOfDuration: List of [Duration]) Result: Text | ||
| var | ||
| Dur: Duration; | ||
| DurationAsBigInt: BigInteger; | ||
| begin | ||
| foreach Dur in ListOfDuration do begin | ||
| DurationAsBigInt := Dur; | ||
| Result += Format(DurationAsBigInt) + ', '; | ||
| end; | ||
| Result := Result.TrimEnd(', '); | ||
| end; | ||
|
|
||
| local procedure ConvertListOfIntegerToString(ListOfInteger: List of [Integer]) Result: Text | ||
| var | ||
| Int: Integer; | ||
| begin | ||
| foreach Int in ListOfInteger do | ||
| Result += Format(Int) + ', '; | ||
| Result := Result.TrimEnd(', '); | ||
| end; | ||
|
|
||
| local procedure ConvertListOfTextToString(ListOfText: List of [Text]) Result: Text | ||
| var | ||
| Text: Text; | ||
| begin | ||
| foreach Text in ListOfText do | ||
| Result += Text + ', '; | ||
| Result := Result.TrimEnd(', '); | ||
| end; | ||
|
|
||
| procedure LogToolNotInvoked(AOAIOperationResponse: Codeunit "AOAI Operation Response") | ||
| var | ||
| FeatureTelemetry: Codeunit "Feature Telemetry"; | ||
| NoSeriesCopilotImpl: Codeunit "No. Series Copilot Impl."; | ||
| TelemetryCD: Dictionary of [Text, Text]; | ||
| begin | ||
| if Durations.Count() <> 0 then | ||
| TelemetryCD.Add('Response time', ConvertListOfDurationToString(Durations)); | ||
|
|
||
| if AOAIOperationResponse.GetResult() = '' then | ||
| FeatureTelemetry.LogError('0000O9B', NoSeriesCopilotImpl.FeatureName(), 'Call Chat Completion API', 'Completion answer is empty', '', TelemetryCD) | ||
| else | ||
| FeatureTelemetry.LogError('0000O9C', NoSeriesCopilotImpl.FeatureName(), 'Process function_call', 'function_call not found in the completion answer'); | ||
| end; | ||
|
|
||
| procedure ResetDurationTracking() | ||
| begin | ||
| Clear(Durations); | ||
| end; | ||
|
|
||
| procedure StartDurationTracking() | ||
| begin | ||
| StartDateTime := CurrentDateTime(); | ||
| end; | ||
|
|
||
| procedure StopDurationTracking() Duration: Duration | ||
| begin | ||
| Durations.Add(CurrentDateTime() - StartDateTime); | ||
| end; | ||
|
|
||
| procedure SaveTotalSuggestedLines(Total: Integer) | ||
| begin | ||
| TotalSuggestedLines.Add(Total); | ||
| end; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters