added await in examples so loaders can complete

llm-tools · May 11, 2024 · 4d5f874 · 4d5f874
1 parent 2fd0335
commit 4d5f874
Show file tree

Hide file tree

Showing 8 changed files with 43 additions and 36 deletions.
diff --git a/examples/chroma/src/index.ts b/examples/chroma/src/index.ts
@@ -8,9 +8,9 @@ const llmApplication = await new RAGApplicationBuilder()
     .setVectorDb(new ChromaDb({ url: 'http://localhost:8000' }))
     .build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 console.log((await llmApplication.query('Who founded Tesla?')).result);
 // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink.

diff --git a/examples/cohere/src/index.ts b/examples/cohere/src/index.ts
@@ -10,9 +10,9 @@ const llmApplication = await new RAGApplicationBuilder()
     .setVectorDb(new HNSWDb())
     .build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 console.log((await llmApplication.query('Who founded Tesla?')).result);
 // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink.

diff --git a/examples/ollama/src/index.ts b/examples/ollama/src/index.ts
@@ -1,21 +1,30 @@
-import { RAGApplicationBuilder, WebLoader, YoutubeLoader, SitemapLoader, Ollama, AdaEmbeddings } from '../../../src/index.js';
+import {
+    RAGApplicationBuilder,
+    WebLoader,
+    YoutubeLoader,
+    SitemapLoader,
+    Ollama,
+    AdaEmbeddings,
+} from '../../../src/index.js';
 import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js';
 
 const modelName = process.argv[2] || 'llama3';
 
 const llmApplication = await new RAGApplicationBuilder()
     .setEmbeddingModel(new AdaEmbeddings())
-    .setModel(new Ollama({
-        modelName: modelName,
-        baseUrl: 'http://localhost:11434'
-    }))
+    .setModel(
+        new Ollama({
+            modelName: modelName,
+            baseUrl: 'http://localhost:11434',
+        }),
+    )
     .setSearchResultCount(30)
     .setVectorDb(new HNSWDb())
     .build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 console.log((await llmApplication.query('Who founded Tesla?')).result);
 // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink.

diff --git a/examples/pinecone/src/index.ts b/examples/pinecone/src/index.ts
@@ -23,9 +23,9 @@ const llmApplication = await new RAGApplicationBuilder()
     )
     .build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 console.log((await llmApplication.query('Who founded Tesla?')).result);
 // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink.

diff --git a/examples/simple/package.json b/examples/simple/package.json
@@ -4,7 +4,7 @@
     "type": "module",
     "private": true,
     "scripts": {
-        "start": "tsc && node dist/examples/simple/src/index.js"
+        "start": "tsc && DEBUG=embedjs:* node dist/examples/simple/src/index.js"
     },
     "author": "",
     "license": "ISC",
@@ -13,6 +13,6 @@
         "ts-node": "^10.9.2"
     },
     "devDependencies": {
-        "@types/node": "^20.11.24"
+        "@types/node": "^20.12.11"
     }
 }
diff --git a/examples/simple/src/index.ts b/examples/simple/src/index.ts
@@ -5,9 +5,9 @@ import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js';
 
 const llmApplication = await new RAGApplicationBuilder().setSearchResultCount(30).setVectorDb(new HNSWDb()).build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 console.log((await llmApplication.query('Who founded Tesla?')).result);
 // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink.

diff --git a/examples/vertexai/src/index.ts b/examples/vertexai/src/index.ts
@@ -2,26 +2,27 @@ import 'dotenv/config';
 
 import { RAGApplicationBuilder, WebLoader, YoutubeLoader, SitemapLoader } from '../../../src/index.js';
 import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js';
-import { VertexAI,GeckoEmbedding } from '../../../src/index.js';
+import { VertexAI, GeckoEmbedding } from '../../../src/index.js';
 
 const llmApplication = await new RAGApplicationBuilder()
-    .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409'}))
+    .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409' }))
     .setEmbeddingModel(new GeckoEmbedding())
     .setSearchResultCount(30)
-    .setVectorDb(new HNSWDb()).build();
+    .setVectorDb(new HNSWDb())
+    .build();
 
-llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
-llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
-llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' }));
+await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' }));
+await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
 
 let question = 'Who founded Tesla?';
-console.log("[QUESTION]", question);
+console.log('[QUESTION]', question);
 console.log((await llmApplication.query(question)).result);
 
 question = 'Tell me about the history of Tesla?';
-console.log("[QUESTION]", question);
+console.log('[QUESTION]', question);
 console.log((await llmApplication.query(question)).result);
 
 question = 'What cars does Tesla have';
-console.log("[QUESTION]", question);
+console.log('[QUESTION]', question);
 console.log((await llmApplication.query(question)).result);
diff --git a/src/vectorDb/hnswlib-db.ts b/src/vectorDb/hnswlib-db.ts
@@ -11,14 +11,11 @@ export class HNSWDb implements BaseDb {
     private docCount: number;
     private docMap: Map<number, { pageContent: string; metadata: Metadata<Record<string, string | number | boolean>> }>;
 
-    constructor() {
-        this.docCount = 0;
-        this.docMap = new Map();
-    }
-
     async init({ dimensions }: { dimensions: number }) {
         this.index = await new HNSWLib.HierarchicalNSW('cosine', dimensions);
         this.index.initIndex(0);
+        this.docMap = new Map();
+        this.docCount = 0;
     }
 
     async insertChunks(chunks: EmbeddedChunk[]): Promise<number> {