Merge pull request #130 from llm-tools/huggingface

OpenAI refactor
llm-tools · Oct 15, 2024 · 02b1ce1 · 02b1ce1
2 parents daeb63c + b3b06f5
commit 02b1ce1
Show file tree

Hide file tree

Showing 99 changed files with 517 additions and 345 deletions.
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.9",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6"
+        "@llm-tools/embedjs-interfaces": "0.1.7"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs to align it with other projects, there were no code changes.

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,11 +1,11 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
-        "@llm-tools/embedjs-utils": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
+        "@llm-tools/embedjs-utils": "0.1.7",
         "axios": "^1.7.7",
         "debug": "^4.3.7",
         "langchain": "^0.3.2",

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.11.0",
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "lmdb": "^3.1.3"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "debug": "^4.3.7",
         "mongodb": "^6.9.0"
     },

diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "@pinecone-database/pinecone": "^3.0.3",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.3.7",
         "uuid": "^10.0.0"

diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.7 (2024-10-15)
+
+This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.
+
 ## 0.1.6 (2024-10-13)
 
 This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.6",
+    "version": "0.1.7",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.6",
+        "@llm-tools/embedjs-interfaces": "0.1.7",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7",
         "weaviate-ts-client": "^2.2.0"

diff --git a/docs/api-reference/methods/add-loader.mdx b/docs/api-reference/methods/add-loader.mdx
@@ -16,13 +16,13 @@ The `addLoader()` method is used to load the data sources from different data so
 
 ```ts Code example
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();
 
@@ -34,13 +34,13 @@ await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profil
 
 ```ts Code example
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();
 

diff --git a/docs/api-reference/methods/delete-loader.mdx b/docs/api-reference/methods/delete-loader.mdx
@@ -14,15 +14,15 @@ The `deleteLoader()` method allows you to delete the data loaded by a loader pre
 
 ```ts
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { RedisCache } from '@llm-tools/embedjs-redis';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .setCache(
     new RedisCache({

diff --git a/docs/api-reference/methods/get-loaders.mdx b/docs/api-reference/methods/get-loaders.mdx
@@ -10,15 +10,15 @@ This data comes from the cache and if the cache is inMemory, then the list will
 
 ```ts
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { RedisCache } from '@llm-tools/embedjs-redis';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .setCache(
     new RedisCache({

diff --git a/docs/api-reference/methods/reset.mdx b/docs/api-reference/methods/reset.mdx
@@ -8,13 +8,13 @@ The `reset()` method allows you to wipe the data from your RAG application and s
 
 ```ts Code example
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();
 

diff --git a/docs/api-reference/overview.mdx b/docs/api-reference/overview.mdx
@@ -43,13 +43,13 @@ Create a EmbedJs `RAGApplication` using `RAGApplicationBuilder`. `RAGApplication
 
 ```ts Code Example
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 
 //app is of type RAGApplication
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();
 ```
diff --git a/docs/components/data-sources/csv.mdx b/docs/components/data-sources/csv.mdx
@@ -17,13 +17,13 @@ npm install @llm-tools/embedjs-loader-csv
 
 ```ts
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { DocxLoader } from '@llm-tools/embedjs-loader-csv';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();
 
@@ -34,13 +34,13 @@ app.addLoader(new CsvLoader({ filePathOrUrl: '/path/to/file.csv' }))
 
 ```ts
 import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 import { DocxLoader } from '@llm-tools/embedjs-loader-csv';
 
 const app = await new RAGApplicationBuilder()
 .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
-.setEmbeddingModel(new OpenAi3SmallEmbeddings())
+.setEmbeddingModel(new OpenAiEmbeddings())
 .setVectorDb(new HNSWDb())
 .build();