Merge pull request #184 from llm-tools/confluence_updates

Confluence updates
llm-tools · Dec 19, 2024 · 519c1c2 · 519c1c2
2 parents 84d6be8 + 5b7b3ba
commit 519c1c2
Show file tree

Hide file tree

Showing 35 changed files with 315 additions and 299 deletions.
diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
-        "@langchain/core": "^0.3.23",
+        "@langchain/core": "^0.3.25",
         "debug": "^4.4.0",
         "md5": "^2.3.0",
         "uuid": "^11.0.3"

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24"
+        "@llm-tools/embedjs-interfaces": "0.1.25"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,16 +1,16 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "debug": "^4.4.0",
         "langchain": "^0.3.7",
         "md5": "^2.3.0",
-        "mime": "^4.0.4",
+        "mime": "^4.0.6",
         "stream-mime-type": "^2.0.0"
     },
     "devDependencies": {

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "debug": "^4.4.0"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.2.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "debug": "^4.4.0"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "debug": "^4.4.0",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.14.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.4.0"
     },

diff --git a/databases/embedjs-libsql/package.json b/databases/embedjs-libsql/package.json
@@ -1,11 +1,11 @@
 {
     "name": "@llm-tools/embedjs-libsql",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add LibSQL support to embedjs",
     "dependencies": {
         "@libsql/client": "^0.14.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "debug": "^4.4.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "debug": "^4.4.0",
         "lmdb": "^3.2.0"
     },

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "debug": "^4.4.0",
         "mongodb": "^6.12.0"
     },

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "@pinecone-database/pinecone": "^4.0.0",
         "debug": "^4.4.0"
     },

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.4.0",
         "uuid": "^11.0.3"

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.4.0",
         "weaviate-ts-client": "^2.2.0"

diff --git a/examples/confluence/src/main.ts b/examples/confluence/src/main.ts
@@ -13,5 +13,5 @@ const llmApplication = await new RAGApplicationBuilder()
     .setEmbeddingModel(new OpenAiEmbeddings())
     .build();
 
-await llmApplication.addLoader(new ConfluenceLoader({ spaceNames: ['DEMO'] }));
+await llmApplication.addLoader(new ConfluenceLoader({ spaceName: 'DEMO' }));
 console.log(await llmApplication.query('Who founded Tesla?'));
diff --git a/loaders/embedjs-loader-confluence/package.json b/loaders/embedjs-loader-confluence/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-confluence",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Confluence loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-loader-web": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-loader-web": "0.1.25",
         "confluence.js": "^1.7.4",
         "debug": "^4.4.0",
         "md5": "^2.3.0"

diff --git a/loaders/embedjs-loader-confluence/src/confluence-loader.ts b/loaders/embedjs-loader-confluence/src/confluence-loader.ts
@@ -10,32 +10,34 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
 
     private readonly confluence: ConfluenceClient;
     private readonly confluenceBaseUrl: string;
-    private readonly spaceNames: string[];
+    private readonly spaceName: string;
+
+    private readonly lastUpdatedFilter?: Date;
 
     constructor({
-        spaceNames,
+        spaceName,
         confluenceBaseUrl,
         confluenceUsername,
         confluenceToken,
         chunkSize,
         chunkOverlap,
+        options,
     }: {
-        spaceNames: [string, ...string[]];
+        spaceName: string;
         confluenceBaseUrl?: string;
         confluenceUsername?: string;
         confluenceToken?: string;
         chunkSize?: number;
         chunkOverlap?: number;
+        options?: {
+            lastUpdatedFilter: Date;
+        };
     }) {
-        super(
-            `ConfluenceLoader_${md5(spaceNames.sort().join(','))}`,
-            { spaceNames },
-            chunkSize ?? 2000,
-            chunkOverlap ?? 200,
-        );
-
-        this.spaceNames = spaceNames;
+        super(`ConfluenceLoader_${md5(spaceName)}`, { spaceName }, chunkSize ?? 2000, chunkOverlap ?? 200);
+
+        this.spaceName = spaceName;
         this.confluenceBaseUrl = confluenceBaseUrl ?? process.env.CONFLUENCE_BASE_URL;
+        this.lastUpdatedFilter = options?.lastUpdatedFilter ?? null;
 
         this.confluence = new ConfluenceClient({
             host: this.confluenceBaseUrl,
@@ -49,16 +51,13 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
     }
 
     override async *getUnfilteredChunks() {
-        for (const spaceKey of this.spaceNames) {
-            let count = 0;
-
-            for await (const result of this.processSpace(spaceKey)) {
-                yield result;
-                count++;
-            }
-
-            this.debug(`Space '${spaceKey}' had ${count} new pages`);
+        let count = 0;
+        for await (const result of this.processSpace(this.spaceName)) {
+            yield result;
+            count++;
         }
+
+        this.debug(`Space '${this.spaceName}' had ${count} new pages`);
     }
 
     private async *processSpace(spaceKey: string) {
@@ -84,13 +83,26 @@ export class ConfluenceLoader extends BaseLoader<{ type: 'ConfluenceLoader' }, {
         let confluenceVersion = 0;
 
         try {
-            const spaceProperties = await this.confluence.content.getContentById({
+            const pageProperties = await this.confluence.content.getContentById({
                 id: pageId,
-                expand: ['version'],
+                expand: ['version', 'history'],
             });
 
-            if (!spaceProperties.version.number) throw new Error('Version number not found in space properties...');
-            confluenceVersion = spaceProperties.version.number;
+            if (this.lastUpdatedFilter) {
+                const pageLastEditDate = new Date(pageProperties.history.lastUpdated.when);
+
+                if (pageLastEditDate > this.lastUpdatedFilter) {
+                    this.debug(`Page '${title}' has last edit date ${pageLastEditDate}. Continuing...`);
+                } else {
+                    this.debug(
+                        `Page '${title}' has last edit date ${pageLastEditDate}, which is less than filter date. Skipping...`,
+                    );
+                    return;
+                }
+            }
+
+            if (!pageProperties.version.number) throw new Error('Version number not found in page properties...');
+            confluenceVersion = pageProperties.version.number;
         } catch (e) {
             this.debug('Could not get page properties. Page will be SKIPPED!', title, e.response);
             return;

diff --git a/loaders/embedjs-loader-csv/package.json b/loaders/embedjs-loader-csv/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-csv",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "CSV loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "csv-parse": "^5.6.0",
         "debug": "^4.4.0",
         "md5": "^2.3.0"

diff --git a/loaders/embedjs-loader-markdown/package.json b/loaders/embedjs-loader-markdown/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-markdown",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "XML loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-loader-web": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-loader-web": "0.1.25",
         "debug": "^4.4.0",
         "md5": "^2.3.0",
         "micromark": "^4.0.1",

diff --git a/loaders/embedjs-loader-msoffice/package.json b/loaders/embedjs-loader-msoffice/package.json
@@ -1,11 +1,11 @@
 {
     "name": "@llm-tools/embedjs-loader-msoffice",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Word, PPT and Excel loader for embedjs",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "md5": "^2.3.0",
         "office-text-extractor": "^3.0.3"
     },

diff --git a/loaders/embedjs-loader-pdf/package.json b/loaders/embedjs-loader-pdf/package.json
@@ -1,11 +1,11 @@
 {
     "name": "@llm-tools/embedjs-loader-pdf",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "PDF loader for embedjs",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "md5": "^2.3.0",
         "office-text-extractor": "^3.0.3"
     },

diff --git a/loaders/embedjs-loader-sitemap/package.json b/loaders/embedjs-loader-sitemap/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-sitemap",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Sitemap recursive loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-loader-web": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-loader-web": "0.1.25",
         "debug": "^4.4.0",
         "md5": "^2.3.0",
         "sitemapper": "^3.2.18"

diff --git a/loaders/embedjs-loader-web/package.json b/loaders/embedjs-loader-web/package.json
@@ -1,11 +1,11 @@
 {
     "name": "@llm-tools/embedjs-loader-web",
-    "version": "0.1.24",
+    "version": "0.1.25",
     "description": "Web page loader for embedjs",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.24",
-        "@llm-tools/embedjs-utils": "0.1.24",
+        "@llm-tools/embedjs-interfaces": "0.1.25",
+        "@llm-tools/embedjs-utils": "0.1.25",
         "debug": "^4.4.0",
         "html-to-text": "^9.0.5",
         "md5": "^2.3.0"