Merge pull request #133 from llm-tools/huggingface

Improved confluence loader
llm-tools · Oct 21, 2024 · c1d405b · c1d405b
2 parents 3097386 + 74477d1
commit c1d405b
Show file tree

Hide file tree

Showing 57 changed files with 278 additions and 105 deletions.
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.9",

diff --git a/core/embedjs-interfaces/src/interfaces/base-loader.ts b/core/embedjs-interfaces/src/interfaces/base-loader.ts
@@ -6,8 +6,8 @@ import { BaseCache } from './base-cache.js';
 import { LoaderList, LoaderChunk, UnfilteredLoaderChunk } from '../types.js';
 
 export abstract class BaseLoader<
-    T extends Record<string, string | number | boolean> = Record<string, string | number | boolean>,
-    M extends Record<string, unknown> = Record<string, null>,
+    MetadataTemplate extends Record<string, string | number | boolean> = Record<string, string | number | boolean>,
+    CacheTemplate extends Record<string, unknown> = Record<string, null>,
 > extends EventEmitter {
     private static cache: Pick<
         BaseCache,
@@ -107,12 +107,12 @@ export abstract class BaseLoader<
         return BaseLoader.cache.loaderCustomHas(this.getCustomCacheKey(key));
     }
 
-    protected async getFromCache(key: string) {
+    protected async getFromCache(key: string): Promise<CacheTemplate> {
         if (!BaseLoader.cache) return null;
         return BaseLoader.cache.loaderCustomGet(this.getCustomCacheKey(key));
     }
 
-    protected async saveToCache(key: string, value: M) {
+    protected async saveToCache(key: string, value: CacheTemplate) {
         if (!BaseLoader.cache) return;
         await BaseLoader.cache.loaderCustomSet(this.getCustomCacheKey(key), value);
     }
@@ -122,15 +122,17 @@ export abstract class BaseLoader<
         return BaseLoader.cache.loaderCustomDelete(this.getCustomCacheKey(key));
     }
 
-    protected async loadIncrementalChunk(incrementalGenerator: AsyncGenerator<LoaderChunk<T>, void, void>) {
+    protected async loadIncrementalChunk(
+        incrementalGenerator: AsyncGenerator<LoaderChunk<MetadataTemplate>, void, void>,
+    ) {
         this.emit('incrementalChunkAvailable', incrementalGenerator);
     }
 
     /**
      * This TypeScript function asynchronously processes chunks of data, cleans up the content,
      * calculates a content hash, and yields the modified chunks.
      */
-    public async *getChunks(): AsyncGenerator<LoaderChunk<T>, void, void> {
+    public async *getChunks(): AsyncGenerator<LoaderChunk<MetadataTemplate>, void, void> {
         const chunks = await this.getUnfilteredChunks();
 
         for await (const chunk of chunks) {
@@ -148,5 +150,5 @@ export abstract class BaseLoader<
         }
     }
 
-    abstract getUnfilteredChunks(): AsyncGenerator<UnfilteredLoaderChunk<T>, void, void>;
+    abstract getUnfilteredChunks(): AsyncGenerator<UnfilteredLoaderChunk<MetadataTemplate>, void, void>;
 }
diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7"
+        "@llm-tools/embedjs-interfaces": "0.1.8"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs to align it with other projects, there were no code changes.

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,11 +1,11 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
-        "@llm-tools/embedjs-utils": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
+        "@llm-tools/embedjs-utils": "0.1.8",
         "axios": "^1.7.7",
         "debug": "^4.3.7",
         "langchain": "^0.3.2",

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.11.0",
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "lmdb": "^3.1.3"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "debug": "^4.3.7",
         "mongodb": "^6.9.0"
     },

diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "@pinecone-database/pinecone": "^3.0.3",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.3.7",
         "uuid": "^10.0.0"

diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7",
         "weaviate-ts-client": "^2.2.0"

diff --git a/loaders/embedjs-loader-confluence/CHANGELOG.md b/loaders/embedjs-loader-confluence/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.8 (2024-10-21)
+
+Updated conflunece loader to leverage better caching and skip processing pages with no updates
+
 ## 0.1.7 (2024-10-15)
 
 This was a version bump only for embedjs-loader-confluence to align it with other projects, there were no code changes.

diff --git a/loaders/embedjs-loader-confluence/package.json b/loaders/embedjs-loader-confluence/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-confluence",
-    "version": "0.1.7",
+    "version": "0.1.8",
     "description": "Confluence loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.7",
-        "@llm-tools/embedjs-loader-web": "0.1.7",
+        "@llm-tools/embedjs-interfaces": "0.1.8",
+        "@llm-tools/embedjs-loader-web": "0.1.8",
         "confluence.js": "^1.7.4",
         "debug": "^4.3.7",
         "md5": "^2.3.0"