From 4d5f8748313641b63890cb0b9024d7fd3614fc85 Mon Sep 17 00:00:00 2001 From: Adhityan K V Date: Sat, 11 May 2024 16:21:48 +0200 Subject: [PATCH] added await in examples so loaders can complete --- examples/chroma/src/index.ts | 6 +++--- examples/cohere/src/index.ts | 6 +++--- examples/ollama/src/index.ts | 25 +++++++++++++++++-------- examples/pinecone/src/index.ts | 6 +++--- examples/simple/package.json | 4 ++-- examples/simple/src/index.ts | 6 +++--- examples/vertexai/src/index.ts | 19 ++++++++++--------- src/vectorDb/hnswlib-db.ts | 7 ++----- 8 files changed, 43 insertions(+), 36 deletions(-) diff --git a/examples/chroma/src/index.ts b/examples/chroma/src/index.ts index bcca0d00..2bad016a 100644 --- a/examples/chroma/src/index.ts +++ b/examples/chroma/src/index.ts @@ -8,9 +8,9 @@ const llmApplication = await new RAGApplicationBuilder() .setVectorDb(new ChromaDb({ url: 'http://localhost:8000' })) .build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); console.log((await llmApplication.query('Who founded Tesla?')).result); // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink. diff --git a/examples/cohere/src/index.ts b/examples/cohere/src/index.ts index 94f8f698..a428c1f4 100644 --- a/examples/cohere/src/index.ts +++ b/examples/cohere/src/index.ts @@ -10,9 +10,9 @@ const llmApplication = await new RAGApplicationBuilder() .setVectorDb(new HNSWDb()) .build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); console.log((await llmApplication.query('Who founded Tesla?')).result); // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink. diff --git a/examples/ollama/src/index.ts b/examples/ollama/src/index.ts index 12c06efc..ef884603 100644 --- a/examples/ollama/src/index.ts +++ b/examples/ollama/src/index.ts @@ -1,21 +1,30 @@ -import { RAGApplicationBuilder, WebLoader, YoutubeLoader, SitemapLoader, Ollama, AdaEmbeddings } from '../../../src/index.js'; +import { + RAGApplicationBuilder, + WebLoader, + YoutubeLoader, + SitemapLoader, + Ollama, + AdaEmbeddings, +} from '../../../src/index.js'; import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js'; const modelName = process.argv[2] || 'llama3'; const llmApplication = await new RAGApplicationBuilder() .setEmbeddingModel(new AdaEmbeddings()) - .setModel(new Ollama({ - modelName: modelName, - baseUrl: 'http://localhost:11434' - })) + .setModel( + new Ollama({ + modelName: modelName, + baseUrl: 'http://localhost:11434', + }), + ) .setSearchResultCount(30) .setVectorDb(new HNSWDb()) .build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); console.log((await llmApplication.query('Who founded Tesla?')).result); // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink. diff --git a/examples/pinecone/src/index.ts b/examples/pinecone/src/index.ts index 679cb04a..513cf3ee 100644 --- a/examples/pinecone/src/index.ts +++ b/examples/pinecone/src/index.ts @@ -23,9 +23,9 @@ const llmApplication = await new RAGApplicationBuilder() ) .build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); console.log((await llmApplication.query('Who founded Tesla?')).result); // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink. diff --git a/examples/simple/package.json b/examples/simple/package.json index 53cf3c62..0c906e8d 100644 --- a/examples/simple/package.json +++ b/examples/simple/package.json @@ -4,7 +4,7 @@ "type": "module", "private": true, "scripts": { - "start": "tsc && node dist/examples/simple/src/index.js" + "start": "tsc && DEBUG=embedjs:* node dist/examples/simple/src/index.js" }, "author": "", "license": "ISC", @@ -13,6 +13,6 @@ "ts-node": "^10.9.2" }, "devDependencies": { - "@types/node": "^20.11.24" + "@types/node": "^20.12.11" } } diff --git a/examples/simple/src/index.ts b/examples/simple/src/index.ts index 71119e3c..f79821fa 100644 --- a/examples/simple/src/index.ts +++ b/examples/simple/src/index.ts @@ -5,9 +5,9 @@ import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js'; const llmApplication = await new RAGApplicationBuilder().setSearchResultCount(30).setVectorDb(new HNSWDb()).build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); console.log((await llmApplication.query('Who founded Tesla?')).result); // The founder of Tesla is Elon Musk. He co-founded the company with JB Straubel, Martin Eberhard, Marc Tarpenning, and Ian Wright in 2003. Elon Musk is also the CEO of SpaceX and Neuralink. diff --git a/examples/vertexai/src/index.ts b/examples/vertexai/src/index.ts index 5afa182f..08250bba 100644 --- a/examples/vertexai/src/index.ts +++ b/examples/vertexai/src/index.ts @@ -2,26 +2,27 @@ import 'dotenv/config'; import { RAGApplicationBuilder, WebLoader, YoutubeLoader, SitemapLoader } from '../../../src/index.js'; import { HNSWDb } from '../../../src/vectorDb/hnswlib-db.js'; -import { VertexAI,GeckoEmbedding } from '../../../src/index.js'; +import { VertexAI, GeckoEmbedding } from '../../../src/index.js'; const llmApplication = await new RAGApplicationBuilder() - .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409'})) + .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409' })) .setEmbeddingModel(new GeckoEmbedding()) .setSearchResultCount(30) - .setVectorDb(new HNSWDb()).build(); + .setVectorDb(new HNSWDb()) + .build(); -llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); -llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); +await llmApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); +await llmApplication.addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })); +await llmApplication.addLoader(new WebLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); let question = 'Who founded Tesla?'; -console.log("[QUESTION]", question); +console.log('[QUESTION]', question); console.log((await llmApplication.query(question)).result); question = 'Tell me about the history of Tesla?'; -console.log("[QUESTION]", question); +console.log('[QUESTION]', question); console.log((await llmApplication.query(question)).result); question = 'What cars does Tesla have'; -console.log("[QUESTION]", question); +console.log('[QUESTION]', question); console.log((await llmApplication.query(question)).result); diff --git a/src/vectorDb/hnswlib-db.ts b/src/vectorDb/hnswlib-db.ts index 33e8ad5b..a229269b 100644 --- a/src/vectorDb/hnswlib-db.ts +++ b/src/vectorDb/hnswlib-db.ts @@ -11,14 +11,11 @@ export class HNSWDb implements BaseDb { private docCount: number; private docMap: Map> }>; - constructor() { - this.docCount = 0; - this.docMap = new Map(); - } - async init({ dimensions }: { dimensions: number }) { this.index = await new HNSWLib.HierarchicalNSW('cosine', dimensions); this.index.initIndex(0); + this.docMap = new Map(); + this.docCount = 0; } async insertChunks(chunks: EmbeddedChunk[]): Promise {