Test new Qdrant SK connector

microsoft · Aug 21, 2024 · 20ac9ae · 20ac9ae
1 parent cfefc49
commit 20ac9ae
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 36 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -3,27 +3,27 @@
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
   </PropertyGroup>
   <ItemGroup>
-    <PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.2" />
-    <PackageVersion Include="AWSSDK.S3" Version="3.7.310.4" />
+    <PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.1.0" />
+    <PackageVersion Include="AWSSDK.S3" Version="3.7.402" />
     <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0" />
-    <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.1" />
+    <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.2" />
     <PackageVersion Include="Azure.Identity" Version="1.12.0" />
-    <PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
-    <PackageVersion Include="Azure.Storage.Blobs" Version="12.20.0" />
-    <PackageVersion Include="Azure.Storage.Queues" Version="12.18.0" />
+    <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
+    <PackageVersion Include="Azure.Storage.Blobs" Version="12.21.2" />
+    <PackageVersion Include="Azure.Storage.Queues" Version="12.19.1" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.11.0" />
-    <PackageVersion Include="Discord.Net" Version="3.15.2" />
-    <PackageVersion Include="HtmlAgilityPack" Version="1.11.61" />
-    <PackageVersion Include="LLamaSharp" Version="0.13.0" />
-    <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.13.0" />
+    <PackageVersion Include="Discord.Net" Version="3.15.3" />
+    <PackageVersion Include="HtmlAgilityPack" Version="1.11.64" />
+    <PackageVersion Include="LLamaSharp" Version="0.15.0" />
+    <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.15.0" />
     <PackageVersion Include="LLamaSharp.Backend.Cuda11" Version="0.12.0" />
     <PackageVersion Include="LLamaSharp.Backend.Cuda12" Version="0.12.0" />
     <PackageVersion Include="LLamaSharp.Backend.OpenCL" Version="0.12.0" />
     <PackageVersion Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.1" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.Configuration.AzureAppConfiguration" Version="7.2.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.AzureAppConfiguration" Version="7.3.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
@@ -34,18 +34,19 @@
     <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Logging.TraceSource" Version="8.0.0" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24271.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.66.240709.1" />
-    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.66.240709.1" />
-    <PackageVersion Include="MongoDB.Driver.GridFS" Version="2.27.0" />
+    <PackageVersion Include="Microsoft.KernelMemory.Core" Version="0.70.240803.1" />
+    <PackageVersion Include="Microsoft.KernelMemory.Service.AspNetCore" Version="0.70.240803.1" />
+    <PackageVersion Include="MongoDB.Driver.GridFS" Version="2.28.0" />
     <PackageVersion Include="Moq" Version="4.20.70" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="PdfPig" Version="0.1.8" />
     <PackageVersion Include="Pgvector" Version="0.3.0" />
     <PackageVersion Include="Polly.Core" Version="8.4.1" />
+    <PackageVersion Include="Qdrant.Client" Version="1.11.0" />
     <PackageVersion Include="RabbitMQ.Client" Version="6.8.1" />
     <PackageVersion Include="NRedisStack" Version="0.12.0" />
     <PackageVersion Include="ReadLine" Version="2.0.1" />
-    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.6.2" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.7.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
     <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0" />
@@ -58,25 +59,19 @@
     <!-- see https://github.com/ClosedXML/ClosedXML/releases/tag/0.102.2 -->
     <PackageVersion Include="ClosedXML" Version="0.102.2" />
   </ItemGroup>
-  <!-- Semantic Kernel -->
-  <ItemGroup>
-<!--    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.15.1" />-->
-<!--    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.15.1" />-->
-<!--    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.OpenAI" Version="1.15.1" />-->
-  </ItemGroup>
   <!-- Documentation -->
   <ItemGroup>
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
   </ItemGroup>
   <!-- Code Analysis -->
   <ItemGroup>
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.10.0">
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.11.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
     <PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0" />
-    <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.10.48">
+    <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.11.20">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
@@ -95,7 +90,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
-    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
     <PackageVersion Include="xunit" Version="2.9.0" />
     <PackageVersion Include="xunit.abstractions" Version="2.0.3" />
     <PackageVersion Include="xunit.extensibility.core" Version="2.7.1" />

diff --git a/KernelMemory.sln b/KernelMemory.sln
@@ -297,7 +297,17 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernel.Abstractions
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.OpenAI", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj", "{397E4DFB-0318-4629-9BB1-52B6445232FD}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernel.Core", "_dev\semantic-kernel\dotnet\src\SemanticKernel.Core\SemanticKernel.Core.csproj", "{E8CC77FC-137A-4508-885F-346D90FC4169}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.AzureAISearch", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.AzureAISearch\Connectors.Memory.AzureAISearch.csproj", "{A91FD3F6-DF52-49A6-AF23-436D12EDA8E8}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.Qdrant", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj", "{764DBFC1-15B2-4E54-9920-F04F60B96BCE}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.Postgres", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj", "{FBA010D2-5FA2-47D4-9335-7F38B62BF504}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.Redis", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj", "{E2FCE5F7-3D0A-405D-8641-C38493CDFB89}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Memory.Pinecone", "_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj", "{3F469504-D722-4D20-BE96-DF85C3E58C6B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernel.Core", "_dev\semantic-kernel\dotnet\src\SemanticKernel.Core\SemanticKernel.Core.csproj", "{DA31F7BA-6F88-4E70-851E-C56648F27C11}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -560,15 +570,27 @@ Global
 		{0C2C849E-0BA0-4D55-B995-E81FCDBBF0DA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{0C2C849E-0BA0-4D55-B995-E81FCDBBF0DA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0C2C849E-0BA0-4D55-B995-E81FCDBBF0DA}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{0C2C849E-0BA0-4D55-B995-E81FCDBBF0DA}.Release|Any CPU.Build.0 = Release|Any CPU
 		{397E4DFB-0318-4629-9BB1-52B6445232FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{397E4DFB-0318-4629-9BB1-52B6445232FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{397E4DFB-0318-4629-9BB1-52B6445232FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{397E4DFB-0318-4629-9BB1-52B6445232FD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E8CC77FC-137A-4508-885F-346D90FC4169}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E8CC77FC-137A-4508-885F-346D90FC4169}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E8CC77FC-137A-4508-885F-346D90FC4169}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E8CC77FC-137A-4508-885F-346D90FC4169}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A91FD3F6-DF52-49A6-AF23-436D12EDA8E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A91FD3F6-DF52-49A6-AF23-436D12EDA8E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A91FD3F6-DF52-49A6-AF23-436D12EDA8E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{764DBFC1-15B2-4E54-9920-F04F60B96BCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{764DBFC1-15B2-4E54-9920-F04F60B96BCE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{764DBFC1-15B2-4E54-9920-F04F60B96BCE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FBA010D2-5FA2-47D4-9335-7F38B62BF504}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FBA010D2-5FA2-47D4-9335-7F38B62BF504}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FBA010D2-5FA2-47D4-9335-7F38B62BF504}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E2FCE5F7-3D0A-405D-8641-C38493CDFB89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E2FCE5F7-3D0A-405D-8641-C38493CDFB89}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E2FCE5F7-3D0A-405D-8641-C38493CDFB89}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3F469504-D722-4D20-BE96-DF85C3E58C6B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3F469504-D722-4D20-BE96-DF85C3E58C6B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3F469504-D722-4D20-BE96-DF85C3E58C6B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DA31F7BA-6F88-4E70-851E-C56648F27C11}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DA31F7BA-6F88-4E70-851E-C56648F27C11}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DA31F7BA-6F88-4E70-851E-C56648F27C11}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -660,7 +682,12 @@ Global
 		{5A14582B-C6D0-459E-BBB8-EA46CE8DC52E} = {155DA079-E267-49AF-973A-D1D44681970F}
 		{0C2C849E-0BA0-4D55-B995-E81FCDBBF0DA} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
 		{397E4DFB-0318-4629-9BB1-52B6445232FD} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
-		{E8CC77FC-137A-4508-885F-346D90FC4169} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{A91FD3F6-DF52-49A6-AF23-436D12EDA8E8} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{764DBFC1-15B2-4E54-9920-F04F60B96BCE} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{FBA010D2-5FA2-47D4-9335-7F38B62BF504} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{E2FCE5F7-3D0A-405D-8641-C38493CDFB89} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{3F469504-D722-4D20-BE96-DF85C3E58C6B} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
+		{DA31F7BA-6F88-4E70-851E-C56648F27C11} = {C72A2140-8D2F-48A9-B9D8-333E1B1D4ABE}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

diff --git a/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json b/extensions/Qdrant/Qdrant.FunctionalTests/appsettings.json
@@ -7,7 +7,8 @@
     "Services": {
       "Qdrant": {
         "Endpoint": "http://127.0.0.1:6333",
-        "APIKey": ""
+        "APIKey": "",
+        "UseSKConnector": true
       },
       "OpenAI": {
         // Name of the model used to generate text (text completion or chat completion)

diff --git a/extensions/Qdrant/Qdrant/Qdrant.csproj b/extensions/Qdrant/Qdrant/Qdrant.csproj
@@ -10,9 +10,11 @@
 
     <ItemGroup>
         <ProjectReference Include="..\..\..\service\Abstractions\Abstractions.csproj" />
+        <ProjectReference Include="..\..\..\_dev\semantic-kernel\dotnet\src\Connectors\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
     </ItemGroup>
 
     <ItemGroup>
+        <PackageReference Include="Qdrant.Client" />
         <PackageReference Include="System.Linq.Async" />
     </ItemGroup>
 

diff --git a/extensions/Qdrant/Qdrant/QdrantConfig.cs b/extensions/Qdrant/Qdrant/QdrantConfig.cs
@@ -19,4 +19,5 @@ public class QdrantConfig
 
     public string Endpoint { get; set; } = string.Empty;
     public string APIKey { get; set; } = string.Empty;
+    public bool UseSKConnector { get; set; } = false;
 }
diff --git a/extensions/Qdrant/Qdrant/QdrantMemory.cs b/extensions/Qdrant/Qdrant/QdrantMemory.cs
@@ -13,6 +13,8 @@
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.MemoryDb.Qdrant.Client;
 using Microsoft.KernelMemory.MemoryStorage;
+using Microsoft.SemanticKernel.Connectors.Qdrant;
+using Qdrant.Client;
 
 namespace Microsoft.KernelMemory.MemoryDb.Qdrant;
 
@@ -22,7 +24,7 @@ namespace Microsoft.KernelMemory.MemoryDb.Qdrant;
 /// * allow using more Qdrant specific filtering logic
 /// </summary>
 [Experimental("KMEXP03")]
-public sealed class QdrantMemory : IMemoryDb, IMemoryDbUpsertBatch
+public sealed class QdrantMemory : IMemoryDb, IMemoryDbUpsertBatch, IDisposable
 {
     private readonly ITextEmbeddingGenerator _embeddingGenerator;
     private readonly QdrantClient<DefaultQdrantPayload> _qdrantClient;
@@ -39,6 +41,15 @@ public QdrantMemory(
         ITextEmbeddingGenerator embeddingGenerator,
         ILoggerFactory? loggerFactory = null)
     {
+        // === MOVING TO SK connector
+        this._useSKConnector = config.UseSKConnector;
+        this._skQdrantClient = new QdrantClient(
+            address: new Uri(config.Endpoint),
+            apiKey: config.APIKey,
+            loggerFactory: loggerFactory ?? DefaultLogger.Factory);
+        this._skQdrantStore = new QdrantVectorStore(this._skQdrantClient, new QdrantVectorStoreOptions());
+        // ==========================
+
         this._embeddingGenerator = embeddingGenerator;
 
         if (this._embeddingGenerator == null)
@@ -62,6 +73,16 @@ public Task CreateIndexAsync(
     /// <inheritdoc />
     public async Task<IEnumerable<string>> GetIndexesAsync(CancellationToken cancellationToken = default)
     {
+        // === MOVING TO SK connector
+        if (this._useSKConnector)
+        {
+            IAsyncEnumerable<string> asyncList = this._skQdrantStore.ListCollectionNamesAsync(cancellationToken);
+            return await asyncList
+                .ToListAsync(cancellationToken: cancellationToken)
+                .ConfigureAwait(false);
+        }
+        // ==========================
+
         return await this._qdrantClient
             .GetCollectionsAsync(cancellationToken)
             .ToListAsync(cancellationToken: cancellationToken)
@@ -274,6 +295,15 @@ public async Task DeleteAsync(
     private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]");
     private const string ValidSeparator = "-";
 
+    private readonly QdrantClient _skQdrantClient;
+    private readonly QdrantVectorStore _skQdrantStore;
+    private readonly bool _useSKConnector;
+
+    public void Dispose()
+    {
+        this._skQdrantClient.Dispose();
+    }
+
     private static string NormalizeIndexName(string index)
     {
         ArgumentNullExceptionEx.ThrowIfNullOrWhiteSpace(index, nameof(index), "The index name is empty");

diff --git a/tools/InteractiveSetup/Main.cs b/tools/InteractiveSetup/Main.cs
@@ -53,7 +53,7 @@ public static void InteractiveSetup(string[] args)
             AzureAISearch.Setup(ctx);
             MongoDbAtlasMemoryDb.Setup(ctx);
             Services.Postgres.Setup(ctx);
-            Qdrant.Setup(ctx);
+            Services.Qdrant.Setup(ctx);
             Redis.Setup(ctx);
             SimpleVectorDb.Setup(ctx);
 
@@ -128,7 +128,7 @@ private static void ConfigureItem(Context ctx, string[] items)
                     break;
 
                 case string x when x.Equals("Qdrant", StringComparison.OrdinalIgnoreCase):
-                    Qdrant.Setup(ctx, true);
+                    Services.Qdrant.Setup(ctx, true);
                     break;
 
                 case string x when x.Equals("RabbitMQ", StringComparison.OrdinalIgnoreCase):