Merge pull request #143 from llm-tools/loaders

Dynamic loader fix
llm-tools · Oct 27, 2024 · 3f2a1e5 · 3f2a1e5
2 parents f5964cb + 9f39bfe
commit 3f2a1e5
Show file tree

Hide file tree

Showing 76 changed files with 1,253 additions and 1,347 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -26,4 +26,4 @@ jobs:
             - name: Install dependencies
               run: npm ci
             - name: Test build
-              run: npm run build:esm
+              run: npm run build:ci
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.15",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10"
+        "@llm-tools/embedjs-interfaces": "0.1.11"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs-utils/src/util/stream.ts b/core/embedjs-utils/src/util/stream.ts
@@ -9,3 +9,9 @@ export async function stream2buffer(stream: Stream): Promise<Buffer> {
         stream.on('error', (err) => reject(`error converting stream - ${err}`));
     });
 }
+
+export function contentTypeToMimeType(contentType: string) {
+    if (!contentType) return contentType;
+    if (contentType.includes(';')) return contentType.split(';')[0];
+    else return contentType;
+}
diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,8 @@
+## 0.1.11 (2024-10-27)
+
+-   Improved URL Loader to better detect dynamic content mime type
+-   Url and Local path loaders do not throw an exception when a loader is missing. Only logs will be emitted.
+
 ## 0.1.10 (2024-10-25)
 
 -   The library now shows more informative errors when embedding model is not set

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
-        "@llm-tools/embedjs-utils": "0.1.10",
-        "axios": "^1.7.7",
+        "@langchain/textsplitters": "^0.1.0",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
+        "@llm-tools/embedjs-utils": "0.1.11",
         "debug": "^4.3.7",
         "langchain": "^0.3.4",
         "md5": "^2.3.0",
@@ -16,7 +16,7 @@
     "devDependencies": {
         "@types/debug": "^4.1.12",
         "@types/md5": "^2.3.5",
-        "@types/node": "^22.8.0"
+        "@types/node": "^22.8.1"
     },
     "main": "./src/index.js",
     "license": "Apache-2.0",

diff --git a/core/embedjs/src/loaders/local-path-loader.ts b/core/embedjs/src/loaders/local-path-loader.ts
@@ -39,14 +39,18 @@ export class LocalPathLoader extends BaseLoader<{ type: 'LocalPathLoader' }> {
             this.debug(`File '${this.path}' has mime type '${mime}'`);
             stream.destroy();
 
-            const loader = await createLoaderFromMimeType(currentPath, mime);
-            for await (const result of await loader.getUnfilteredChunks()) {
-                yield {
-                    pageContent: result.pageContent,
-                    metadata: {
-                        source: currentPath,
-                    },
-                };
+            try {
+                const loader = await createLoaderFromMimeType(currentPath, mime);
+                for await (const result of await loader.getUnfilteredChunks()) {
+                    yield {
+                        pageContent: result.pageContent,
+                        metadata: {
+                            source: currentPath,
+                        },
+                    };
+                }
+            } catch (err) {
+                this.debug(`Error creating loader for mime type '${mime}'`, err);
             }
         } else {
             const files = fs.readdirSync(currentPath);

diff --git a/core/embedjs/src/loaders/text-loader.ts b/core/embedjs/src/loaders/text-loader.ts
@@ -1,4 +1,4 @@
-import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import md5 from 'md5';
 
 import { BaseLoader } from '@llm-tools/embedjs-interfaces';

diff --git a/core/embedjs/src/loaders/url-loader.ts b/core/embedjs/src/loaders/url-loader.ts
@@ -1,37 +1,45 @@
 import { getMimeType } from 'stream-mime-type';
 import createDebugMessages from 'debug';
-import axios from 'axios';
 import md5 from 'md5';
 
+import { contentTypeToMimeType, truncateCenterString } from '@llm-tools/embedjs-utils';
 import { BaseLoader } from '@llm-tools/embedjs-interfaces';
-import { truncateCenterString } from '@llm-tools/embedjs-utils';
 import { createLoaderFromMimeType } from '../util/mime.js';
 
 export class UrlLoader extends BaseLoader<{ type: 'UrlLoader' }> {
     private readonly debug = createDebugMessages('embedjs:loader:UrlLoader');
-    private readonly url: string;
+    private readonly url: URL;
 
     constructor({ url }: { url: string }) {
         super(`UrlLoader_${md5(url)}`, { url: truncateCenterString(url, 50) });
-        this.url = url;
+        this.url = new URL(url);
+        this.debug(`UrlLoader verified '${url}' is a valid URL!`);
     }
 
     override async *getUnfilteredChunks() {
-        this.debug('Loader is a valid URL!');
-        const stream = (await axios.get(this.url, { responseType: 'stream' })).data;
-        const { mime } = await getMimeType(stream);
-        this.debug(`Loader type detected as '${mime}'`);
-        stream.destroy();
+        const response = await fetch(this.url, { headers: { 'Accept-Encoding': '' } });
+        const stream = response.body as unknown as NodeJS.ReadableStream;
+        let { mime } = await getMimeType(stream, { strict: true });
+        this.debug(`Loader stream detected type '${mime}'`);
 
-        const loader = await createLoaderFromMimeType(this.url, mime);
-        for await (const result of await loader.getUnfilteredChunks()) {
-            yield {
-                pageContent: result.pageContent,
-                metadata: {
-                    type: <const>'UrlLoader',
-                    source: this.url,
-                },
-            };
+        if (!mime) {
+            mime = contentTypeToMimeType(response.headers.get('content-type'));
+            this.debug(`Using type '${mime}' from content-type header`);
+        }
+
+        try {
+            const loader = await createLoaderFromMimeType(this.url.href, mime);
+            for await (const result of await loader.getUnfilteredChunks()) {
+                yield {
+                    pageContent: result.pageContent,
+                    metadata: {
+                        type: <const>'UrlLoader',
+                        source: this.url.href,
+                    },
+                };
+            }
+        } catch (err) {
+            this.debug(`Error creating loader for mime type '${mime}'`, err);
         }
     }
 }
diff --git a/core/embedjs/src/util/mime.ts b/core/embedjs/src/util/mime.ts
@@ -78,8 +78,10 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string)
             if (await SitemapLoader.test(loader)) {
                 return new SitemapLoader({ url: loader });
             }
-            throw new Error(`No processor found for generic xml`);
+            throw new Error(`No loader supported for generic xml`);
         }
+        case undefined:
+            throw new Error(`MIME type could not be detected. Please file an issue if you think this is a bug.`);
         default:
             throw new Error(`Unknown mime type '${mimeType}'`);
     }

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.11.0",
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "lmdb": "^3.1.3"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "debug": "^4.3.7",
         "mongodb": "^6.10.0"
     },

diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 -   bumped pinecone library version to 4.x

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "@pinecone-database/pinecone": "^4.0.0",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.3.7",
         "uuid": "^10.0.0"

diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.11 (2024-10-27)
+
+This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.
+
 ## 0.1.10 (2024-10-25)
 
 This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.10",
+    "version": "0.1.11",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.10",
+        "@llm-tools/embedjs-interfaces": "0.1.11",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7",
         "weaviate-ts-client": "^2.2.0"