Merge branch 'main' into rysweet-python-xlang

.azure/pipelines/build.yaml

@@ -0,0 +1,90 @@
+trigger:
+  branches:
+    include:
+    - main
+  paths:
+    exclude:
+    - samples
+
+schedules:
+- cron: "0 0 * * *"
+  displayName: 'Daily midnight build (including CodeQL)'
+  branches:
+    include:
+    - main
+    - 3.x
+  always: true
+
+parameters:
+  - name: build_configuration
+    displayName: Build configuration
+    type: string
+    default: Release
+    values:
+    - Release
+    - Debug
+  - name: include_suffix
+    displayName: Append version suffix
+    type: boolean
+    default: true
+  - name: version_suffix
+    displayName: Version suffix
+    type: string
+    default: dev.$(Build.BuildNumber)
+  - name: codesign
+    displayName: Enable code signing
+    type: boolean
+    default: false
+  - name: skip_test
+    displayName: Skip tests
+    type: boolean
+    default: false
+  - name: publish_nuget
+    displayName: Publish to nuget.org
+    type: boolean
+    default: false
+  - name: publish_nightly
+    displayName: Publish to autogen-nightly
+    type: boolean
+    default: false
+  - name: runCodeQL3000
+    default: false
+    displayName: Run CodeQL3000 tasks
+    type: boolean
+
+variables:
+- template: templates/vars.yaml
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  ${{ if eq(variables['System.TeamProject'], 'GitHub - PR Builds') }}:
+    template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
+  ${{ else }}:
+    template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    settings:
+      skipBuildTagsForGitHubPullRequests: true
+    pool:
+      name: $(pool_name)
+      image: $(pool_image)
+      os: windows
+    stages:
+    - stage: build_test
+      displayName: Build and Tests
+      jobs:
+      - template: /.azure/pipelines/templates/build.yaml@self
+        parameters:
+          build_configuration: ${{ parameters.build_configuration }}
+          include_suffix: ${{ parameters.include_suffix }}
+          version_suffix: ${{ parameters.version_suffix }}
+          codesign: ${{ parameters.codesign }}
+          skip_test: ${{ parameters.skip_test }}
+          publish_nightly: ${{ parameters.publish_nightly }}
+          publish_nuget: ${{ parameters.publish_nuget }}
+          runCodeQL3000: ${{ parameters.runCodeQL3000 }}

.azure/pipelines/templates/build.yaml

@@ -0,0 +1,222 @@
+parameters:
+  - name: build_configuration
+    displayName: Build configuration
+    type: string
+    default: Release
+    values:
+    - Release
+    - Debug
+  - name: include_suffix
+    displayName: Append version suffix
+    type: boolean
+    default: true
+  - name: version_suffix
+    displayName: Version suffix
+    type: string
+    default: ci.$(Build.BuildNumber)
+  - name: codesign
+    displayName: Enable code signing
+    type: boolean
+    default: false
+  - name: skip_test
+    displayName: Skip tests
+    type: boolean
+    default: false
+  - name: publish_nightly
+    displayName: Publish to autogen-nightly
+    type: boolean
+    default: false
+  - name: publish_nuget
+    displayName: Publish to nuget.org
+    type: boolean
+    default: false
+  - name: runCodeQL3000
+    default: false
+    displayName: Run CodeQL3000 tasks
+    type: boolean
+
+jobs:
+
+# Build, sign dlls, build nuget pkgs, then sign them
+- job: Build
+  displayName: Build and create NuGet packages
+  variables:
+    publishVstsFeed: 'AGPublic/AutoGen-Nightly'
+    ${{ if eq(parameters.codesign, true) }}:
+      esrp_signing: true
+    ${{ else }}:
+      esrp_signing: false
+  ${{ if ne(variables['System.TeamProject'], 'GitHub - PR Builds') }}:
+    templateContext:
+      outputs:
+      - output: pipelineArtifact
+        targetPath: '$(build.sourcesdirectory)/dotnet/artifacts'
+        artifactName: artifacts folder
+      # Publish packages to nightly
+      - ${{ if eq(parameters.publish_nightly, true) }}: # TODO add eq(parameters.codesign, true)
+        - output: nuget
+          useDotNetTask: false
+          packageParentPath: $(Pipeline.Workspace)
+          packagesToPush: $(build.sourcesdirectory)/dotnet/artifacts/**/*.nupkg;$(build.sourcesdirectory)/dotnet/artifacts/**/*.snupkg
+          nuGetFeedType: internal
+          publishVstsFeed: $(publishVstsFeed)
+          allowPackageConflicts: true
+      - ${{ if and(eq(parameters.codesign, true), eq(parameters.publish_nuget, true)) }}:
+        - output: nuget
+          condition: succeeded()
+          useDotNetTask: false
+          packageParentPath: $(Pipeline.Workspace)
+          packagesToPush: $(build.sourcesdirectory)/dotnet/artifacts/**/*.nupkg;$(build.sourcesdirectory)/dotnet/artifacts/**/*.snupkg
+          nuGetFeedType: external
+          publishFeedCredentials: dotnet-orleans-nuget
+          publishPackageMetadata: true
+          allowPackageConflicts: true
+  steps:
+  - checkout: self
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    inputs:
+      useGlobalJson: true
+      workingDirectory: $(Build.SourcesDirectory)/dotnet
+  - task: Bash@3
+    displayName: Install .NET Aspire workload
+    inputs:
+      targetType: 'inline'
+      script: |
+        dotnet nuget locals all --clear 
+        dotnet workload install aspire
+  - ${{ if eq(variables.runCodeQL3000, 'true') }}:
+    - task: CodeQL3000Init@0
+      displayName: CodeQL Initialize
+    # This task only tags a build if it actually does CodeQL3000 work.
+    # Those tasks no-op while the analysis is considered up to date i.e. for runs w/in a few days of each other.
+    - script: "echo ##vso[build.addbuildtag]CodeQL3000"
+      displayName: 'Set CI CodeQL3000 tag'
+      condition: ne(variables.CODEQL_DIST,'')
+  - task: DotNetCoreCLI@2
+    displayName: Build
+    inputs:
+      command: build
+      arguments: '$(build_flags) /bl:${{parameters.build_configuration}}-Build.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)'
+      workingDirectory: $(Build.SourcesDirectory)/dotnet
+    env:
+      ${{ if and(eq(parameters.include_suffix, true), eq(parameters.publish_nuget, false)) }}:
+        VersionSuffix: ${{parameters.version_suffix}}
+      OfficialBuild: $(official_build)
+
+  - ${{ if eq(variables.runCodeQL3000, 'true') }}:
+    - task: CodeQL3000Finalize@0
+      displayName: CodeQL Finalize
+  # DLL code signing
+  - ${{ if eq(variables.esrp_signing, true) }}:
+    - task: UseDotNet@2
+      displayName: 'Codesign: Use .NET Core'
+      inputs:
+        packageType: runtime
+        version: $(codesign_runtime)
+    - task: CopyFiles@2
+      displayName: 'Codesign: Copy Files for signing'
+      inputs:
+        SourceFolder: '$(build.sourcesdirectory)'
+        Contents: |
+          src/**/bin/${{parameters.build_configuration}}/**/AutoGen*.dll
+          src/**/bin/${{parameters.build_configuration}}/**/Microsoft.AutoGen.*.dll
+        TargetFolder: '$(build.artifactstagingdirectory)\codesign'
+        CleanTargetFolder: true
+    - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+      displayName: 'Codesign: ESRP CodeSigning'
+      inputs:
+        ConnectedServiceName: 'CodeSign Service (NuGet)'
+        FolderPath: '$(build.artifactstagingdirectory)\codesign'
+        Pattern: '*'
+        signConfigType: inlineSignParams
+        inlineOperation: |
+          [
+            {
+              "keyCode": "CP-230012",
+              "operationSetCode": "SigntoolSign",
+              "parameters": [
+                {
+                  "parameterName": "OpusName",
+                  "parameterValue": "Microsoft"
+                },
+                {
+                  "parameterName": "OpusInfo",
+                  "parameterValue": "http://www.microsoft.com"
+                },
+                {
+                  "parameterName": "FileDigest",
+                  "parameterValue": "/fd \"SHA256\""
+                },
+                {
+                  "parameterName": "PageHash",
+                  "parameterValue": "/NPH"
+                },
+                {
+                  "parameterName": "TimeStamp",
+                  "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
+                }
+              ],
+                "toolName": "sign",
+                "toolVersion": "1.0"
+              },
+              {
+                "keyCode": "CP-230012",
+                "operationSetCode": "SigntoolVerify",
+                "parameters": [ ],
+                "toolName": "sign",
+                "toolVersion": "1.0"
+            }
+          ]
+        SessionTimeout: 180
+        VerboseLogin: true
+    - task: CopyFiles@2
+      displayName: 'Codesign: Copy Signed Files Back'
+      inputs:
+        SourceFolder: '$(build.artifactstagingdirectory)\codesign'
+        Contents: '**\*'
+        TargetFolder: '$(build.sourcesdirectory)'
+        OverWrite: true
+    # End DLL code signing
+  - task: CmdLine@2
+    displayName: Pack
+    inputs:
+      script: 'dotnet pack --no-build --no-restore $(build_flags) /bl:${{parameters.build_configuration}}-Pack.binlog /p:Configuration=${{parameters.build_configuration}} $(solution)'
+      workingDirectory: $(Build.SourcesDirectory)/dotnet
+    env:
+      ${{ if and(eq(parameters.include_suffix, true), eq(parameters.publish_nuget, false)) }}:
+        VersionSuffix: ${{parameters.version_suffix}}
+      OfficialBuild: $(official_build)
+  # NuGet code signing
+  - ${{ if eq(variables.esrp_signing, true) }}:
+    - task: UseDotNet@2
+      displayName: 'Codesign: Use .NET Core'
+      inputs:
+        packageType: runtime
+        version: $(codesign_runtime)
+    - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
+      displayName: 'Codesign: ESRP CodeSigning (nuget)'
+      inputs:
+        ConnectedServiceName: 'CodeSign Service (NuGet)'
+        FolderPath: '$(build.sourcesdirectory)/Artifacts/${{parameters.build_configuration}}'
+        Pattern: '*.nupkg'
+        signConfigType: inlineSignParams
+        inlineOperation: |
+          [
+            {
+              "keyCode": "CP-401405",
+              "operationSetCode": "NuGetSign",
+              "parameters": [],
+              "toolName": "sign",
+              "toolVersion": "1.0"
+            },
+            {
+              "keyCode": "CP-401405",
+              "operationSetCode": "NuGetVerify",
+              "parameters": [ ],
+              "toolName": "sign",
+              "toolVersion": "1.0"
+            }
+          ]
+        SessionTimeout: 180
+        VerboseLogin: true

.azure/pipelines/templates/vars.yaml

@@ -0,0 +1,34 @@
+# It seems that variables must be defined in their own file when using templates
+
+variables:
+  build_flags: ' /m /v:m'
+  solution: 'AutoGen.sln'
+  codesign_runtime: '2.1.x'
+  GDN_SUPPRESS_FORKED_BUILD_WARNING: true # Avoid warning "Guardian is not supported for builds from forked GitHub repositories"
+  MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+  # Auto-injection is not necessary because the tasks are explicitly included where they're enabled.
+  Codeql.SkipTaskAutoInjection: true
+  ${{ if eq(variables['System.TeamProject'], 'GitHub - PR Builds') }}:
+    pool_name: '1es-agpublish-pool'
+    pool_image: 'agpublish-agent-image'
+    official_build: false
+  ${{ else }}:
+    ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      pool_name: '1es-agpublish-pool'
+      pool_image: 'agpublish-agent-image'
+    ${{ else }}:
+      pool_name: '1es-agpublish-pool'
+      pool_image: 'agpublish-agent-image'
+    official_build: true
+    # Do not let CodeQL3000 Extension gate scan frequency.
+    Codeql.Cadence: 0
+    # Enable CodeQL3000 unconditionally so it may be run on any branch.
+    Codeql.Enabled: true
+    # Ignore test and infrastructure code.
+    Codeql.SourceRoot: src
+    # CodeQL3000 needs this plumbed along as a variable to enable TSA. Don't use TSA in manual builds.
+    Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }}
+    # Default expects tsaoptions.json under SourceRoot.
+    Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json'
+    # Do not slow builds down w/ the CodeQL3000 tasks unless this is a nightly build or it's requested.
+    runCodeQL3000: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }}

.gitignore

@@ -193,3 +193,6 @@ notebook/result.png
 samples/apps/autogen-studio/autogenstudio/models/test/
 
 notebook/coding
+
+# dotnet artifacts
+artifacts

dotnet/src/Microsoft.AutoGen/Agents/Services/Orleans/ISubscriptionsGrain.cs

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ISubscriptionsGrain.cs
+
+namespace Microsoft.AutoGen.Agents;
+public interface ISubscriptionsGrain : IGrainWithIntegerKey
+{
+    ValueTask Subscribe(string agentType, string topic);
+    ValueTask Unsubscribe(string agentType, string topic);
+    ValueTask<Dictionary<string, List<string>>> GetSubscriptions(string agentType);
+}

protos/agent_events.proto

@@ -3,7 +3,10 @@ syntax = "proto3";
 package agents;
 
 option csharp_namespace = "Microsoft.AutoGen.Abstractions";
-
+message TextMessage {
+    string textMessage = 1;
+    string source = 2;
+}
 message Input {
       string message = 1;
 }

python/.gitignore

@@ -172,3 +172,6 @@ docs/**/jupyter_execute
 
 # Temporary files
 tmp_code_*.py
+
+# .NET Development settings
+appsettings.Development.json