diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md index c1c7bbe4..397f8fe3 100644 --- a/core/embedjs-interfaces/CHANGELOG.md +++ b/core/embedjs-interfaces/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json index e613911c..e278364a 100644 --- a/core/embedjs-interfaces/package.json +++ b/core/embedjs-interfaces/package.json @@ -1,6 +1,6 @@ { "name": "@llm-tools/embedjs-interfaces", - "version": "0.1.16", + "version": "0.1.17", "description": "Interfaces for extending the embedjs ecosystem", "dependencies": { "@langchain/core": "^0.3.17", diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md index d177e1be..aed2d430 100644 --- a/core/embedjs-utils/CHANGELOG.md +++ b/core/embedjs-utils/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json index 8057f628..ceb33f2f 100644 --- a/core/embedjs-utils/package.json +++ b/core/embedjs-utils/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-utils", - "version": "0.1.16", + "version": "0.1.17", "description": "Useful util functions when extending the embedjs ecosystem", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16" + "@llm-tools/embedjs-interfaces": "0.1.17" }, "type": "module", "main": "./src/index.js", diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md index df33ea85..b64198ef 100644 --- a/core/embedjs/CHANGELOG.md +++ b/core/embedjs/CHANGELOG.md @@ -1,13 +1,19 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - -### ๐Ÿฉน Fixes - -- renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + +### ๐Ÿฉน Fixes + +- renamed remaining instances if vectorDb to vectorDatabase ([ca79586](https://github.com/llm-tools/embedJs/commit/ca79586)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/core/embedjs/package.json b/core/embedjs/package.json index 15d45478..e04ebf90 100644 --- a/core/embedjs/package.json +++ b/core/embedjs/package.json @@ -1,12 +1,12 @@ { "type": "module", "name": "@llm-tools/embedjs", - "version": "0.1.16", + "version": "0.1.17", "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "langchain": "^0.3.5", "md5": "^2.3.0", diff --git a/core/embedjs/src/core/rag-application-builder.ts b/core/embedjs/src/core/rag-application-builder.ts index 52790a2b..6c2f6ff7 100644 --- a/core/embedjs/src/core/rag-application-builder.ts +++ b/core/embedjs/src/core/rag-application-builder.ts @@ -24,7 +24,7 @@ export class RAGApplicationBuilder { constructor() { this.loaders = []; this.temperature = 0.1; - this.searchResultCount = 7; + this.searchResultCount = 30; this.model = SIMPLE_MODELS.OPENAI_GPT4_TURBO; this.systemMessage = `You are a helpful human like chat bot. Use relevant provided context and chat history to answer the query at the end. Answer in full. diff --git a/core/embedjs/src/loaders/url-loader.ts b/core/embedjs/src/loaders/url-loader.ts index e3871776..2106100d 100644 --- a/core/embedjs/src/loaders/url-loader.ts +++ b/core/embedjs/src/loaders/url-loader.ts @@ -17,7 +17,7 @@ export class UrlLoader extends BaseLoader<{ type: 'UrlLoader' }> { } override async *getUnfilteredChunks() { - const response = await getSafe(this.url.toString(), { headers: { 'Accept-Encoding': '' } }); + const response = await getSafe(this.url.href, { headers: { 'Accept-Encoding': '' } }); const stream = response.body as unknown as NodeJS.ReadableStream; let { mime } = await getMimeType(stream, { strict: true }); this.debug(`Loader stream detected type '${mime}'`); diff --git a/core/embedjs/src/util/mime.ts b/core/embedjs/src/util/mime.ts index 36d8001d..221af0a4 100644 --- a/core/embedjs/src/util/mime.ts +++ b/core/embedjs/src/util/mime.ts @@ -3,7 +3,7 @@ import createDebugMessages from 'debug'; import { BaseLoader } from '@llm-tools/embedjs-interfaces'; import { TextLoader } from '../loaders/text-loader.js'; -export async function createLoaderFromMimeType(loader: string, mimeType: string): Promise { +export async function createLoaderFromMimeType(loaderData: string, mimeType: string): Promise { createDebugMessages('embedjs:util:createLoaderFromMimeType')(`Incoming mime type '${mimeType}'`); switch (mimeType) { @@ -15,7 +15,7 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string) ); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported DocxLoader'); - return new DocxLoader({ filePathOrUrl: loader }); + return new DocxLoader({ filePathOrUrl: loaderData }); } case 'application/vnd.ms-excel': case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': { @@ -25,14 +25,14 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string) ); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported ExcelLoader'); - return new ExcelLoader({ filePathOrUrl: loader }); + return new ExcelLoader({ filePathOrUrl: loaderData }); } case 'application/pdf': { const { PdfLoader } = await import('@llm-tools/embedjs-loader-pdf').catch(() => { throw new Error('Package `@llm-tools/embedjs-loader-pdf` needs to be installed to load PDF files'); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported PdfLoader'); - return new PdfLoader({ filePathOrUrl: loader }); + return new PdfLoader({ filePathOrUrl: loaderData }); } case 'application/vnd.openxmlformats-officedocument.presentationml.presentation': { const { PptLoader } = await import('@llm-tools/embedjs-loader-msoffice').catch(() => { @@ -41,33 +41,35 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string) ); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported PptLoader'); - return new PptLoader({ filePathOrUrl: loader }); + return new PptLoader({ filePathOrUrl: loaderData }); } case 'text/plain': { - const fineType = mime.getType(loader); - createDebugMessages('embedjs:util:createLoaderFromMimeType')(`Fine type for '${loader}' is '${fineType}'`); + const fineType = mime.getType(loaderData); + createDebugMessages('embedjs:util:createLoaderFromMimeType')( + `Fine type for '${loaderData}' is '${fineType}'`, + ); if (fineType === 'text/csv') { const { CsvLoader } = await import('@llm-tools/embedjs-loader-csv').catch(() => { - throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load csv files'); + throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load CSV files'); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported CsvLoader'); - return new CsvLoader({ filePathOrUrl: loader }); - } else return new TextLoader({ text: loader }); + return new CsvLoader({ filePathOrUrl: loaderData }); + } else return new TextLoader({ text: loaderData }); } case 'application/csv': { const { CsvLoader } = await import('@llm-tools/embedjs-loader-csv').catch(() => { - throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load csv files'); + throw new Error('Package `@llm-tools/embedjs-loader-csv` needs to be installed to load CSV files'); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported CsvLoader'); - return new CsvLoader({ filePathOrUrl: loader }); + return new CsvLoader({ filePathOrUrl: loaderData }); } case 'text/html': { const { WebLoader } = await import('@llm-tools/embedjs-loader-web').catch(() => { throw new Error('Package `@llm-tools/embedjs-loader-web` needs to be installed to load web documents'); }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported WebLoader'); - return new WebLoader({ urlOrContent: loader }); + return new WebLoader({ urlOrContent: loaderData }); } case 'text/xml': { const { SitemapLoader } = await import('@llm-tools/embedjs-loader-sitemap').catch(() => { @@ -75,10 +77,16 @@ export async function createLoaderFromMimeType(loader: string, mimeType: string) }); createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported SitemapLoader'); - if (await SitemapLoader.test(loader)) { - return new SitemapLoader({ url: loader }); + if (await SitemapLoader.test(loaderData)) { + return new SitemapLoader({ url: loaderData }); } - throw new Error(`No loader supported for generic xml`); + + //This is not a Sitemap but is still XML + const { XmlLoader } = await import('@llm-tools/embedjs-loader-xml').catch(() => { + throw new Error('Package `@llm-tools/embedjs-loader-xml` needs to be installed to load XML documents'); + }); + createDebugMessages('embedjs:util:createLoaderFromMimeType')('Dynamically imported XmlLoader'); + return new XmlLoader({ filePathOrUrl: loaderData }); } case undefined: throw new Error(`MIME type could not be detected. Please file an issue if you think this is a bug.`); diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md index a66804a5..92b6f921 100644 --- a/databases/embedjs-astra/CHANGELOG.md +++ b/databases/embedjs-astra/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json index 82c5cb7e..046a144e 100644 --- a/databases/embedjs-astra/package.json +++ b/databases/embedjs-astra/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-astradb", - "version": "0.1.16", + "version": "0.1.17", "description": "Add AstraDB support to embedjs", "dependencies": { "@datastax/astra-db-ts": "^1.5.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md index 53205d14..e5ef80ae 100644 --- a/databases/embedjs-cosmos/CHANGELOG.md +++ b/databases/embedjs-cosmos/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json index 12a5eac7..f9613d30 100644 --- a/databases/embedjs-cosmos/package.json +++ b/databases/embedjs-cosmos/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-cosmos", - "version": "0.1.16", + "version": "0.1.17", "description": "Add CosmosDB support to embedjs", "dependencies": { "@azure/cosmos": "^4.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md index 2c719d94..214ba47e 100644 --- a/databases/embedjs-hnswlib/CHANGELOG.md +++ b/databases/embedjs-hnswlib/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json index c8d9ec9a..771df944 100644 --- a/databases/embedjs-hnswlib/package.json +++ b/databases/embedjs-hnswlib/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-hnswlib", - "version": "0.1.16", + "version": "0.1.17", "description": "Add HNSWLib support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "hnswlib-node": "^3.0.0" }, diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md index 39a1c586..4c941542 100644 --- a/databases/embedjs-lancedb/CHANGELOG.md +++ b/databases/embedjs-lancedb/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json index f91e4e68..3ea51c55 100644 --- a/databases/embedjs-lancedb/package.json +++ b/databases/embedjs-lancedb/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-lancedb", - "version": "0.1.16", + "version": "0.1.17", "description": "Add LanceDb support to embedjs", "dependencies": { "@lancedb/lancedb": "^0.12.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "compute-cosine-similarity": "^1.1.0" }, "type": "module", diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md index b56a1fd1..d8adf562 100644 --- a/databases/embedjs-lmdb/CHANGELOG.md +++ b/databases/embedjs-lmdb/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json index 3be6081d..6d720b6b 100644 --- a/databases/embedjs-lmdb/package.json +++ b/databases/embedjs-lmdb/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-lmdb", - "version": "0.1.16", + "version": "0.1.17", "description": "Add LMDB support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "lmdb": "^3.1.4" }, "type": "module", diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md index 750e622a..7fa12d58 100644 --- a/databases/embedjs-mongodb/CHANGELOG.md +++ b/databases/embedjs-mongodb/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json index 9208260d..c068e13d 100644 --- a/databases/embedjs-mongodb/package.json +++ b/databases/embedjs-mongodb/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-mongodb", - "version": "0.1.16", + "version": "0.1.17", "description": "Add MongoDB support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "mongodb": "^6.10.0" }, diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md index 6cedb453..78fb0b0c 100644 --- a/databases/embedjs-pinecone/CHANGELOG.md +++ b/databases/embedjs-pinecone/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json index 3a073cfc..d411a8ac 100644 --- a/databases/embedjs-pinecone/package.json +++ b/databases/embedjs-pinecone/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-pinecone", - "version": "0.1.16", + "version": "0.1.17", "description": "Add Pinecone support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "@pinecone-database/pinecone": "^4.0.0", "debug": "^4.3.7" }, diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md index 3314cc63..99b7dcb7 100644 --- a/databases/embedjs-qdrant/CHANGELOG.md +++ b/databases/embedjs-qdrant/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json index fb818421..86c57abc 100644 --- a/databases/embedjs-qdrant/package.json +++ b/databases/embedjs-qdrant/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-qdrant", - "version": "0.1.16", + "version": "0.1.17", "description": "Add Qdrant support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "@qdrant/js-client-rest": "^1.12.0", "debug": "^4.3.7", "uuid": "^11.0.2" diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md index bdbd7325..1a362bcc 100644 --- a/databases/embedjs-redis/CHANGELOG.md +++ b/databases/embedjs-redis/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json index 1b7932d1..e7d33491 100644 --- a/databases/embedjs-redis/package.json +++ b/databases/embedjs-redis/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-redis", - "version": "0.1.16", + "version": "0.1.17", "description": "Add Redis support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "ioredis": "^5.4.1" }, "type": "module", diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md index 29092de4..ebc8510b 100644 --- a/databases/embedjs-weaviate/CHANGELOG.md +++ b/databases/embedjs-weaviate/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json index 40d8ffe9..587bff71 100644 --- a/databases/embedjs-weaviate/package.json +++ b/databases/embedjs-weaviate/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-weaviate", - "version": "0.1.16", + "version": "0.1.17", "description": "Add Weaviate support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7", "weaviate-ts-client": "^2.2.0" diff --git a/docs/components/data-sources/markdown.mdx b/docs/components/data-sources/markdown.mdx new file mode 100644 index 00000000..49cf7057 --- /dev/null +++ b/docs/components/data-sources/markdown.mdx @@ -0,0 +1,28 @@ +--- +title: '๐Ÿ“ Markdown / MDX file' +--- + +You can load any Markdown(MD) or MDX file from your local file system or through a URL. + +## Install Markdown addon + +```bash +npm install @llm-tools/embedjs-loader-markdown +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAiEmbeddings()) +.setVectorDatabase(new HNSWDb()) +.build(); + +app.addLoader(new MarkdownLoader({ filePathOrUrl: '...' })) +``` diff --git a/docs/components/data-sources/overview.mdx b/docs/components/data-sources/overview.mdx index 6f8c93b3..74fdd4d7 100644 --- a/docs/components/data-sources/overview.mdx +++ b/docs/components/data-sources/overview.mdx @@ -21,6 +21,7 @@ We handle the complexity of loading unstructured data from these data sources, a + diff --git a/docs/components/data-sources/xml.mdx b/docs/components/data-sources/xml.mdx index d1fdb800..74871196 100644 --- a/docs/components/data-sources/xml.mdx +++ b/docs/components/data-sources/xml.mdx @@ -1,5 +1,5 @@ --- -title: '๐Ÿ“Š XML' +title: '๐Ÿงพ XML' --- You can load any XML file from your local file system or through a URL. diff --git a/docs/components/vector-databases/astradb.mdx b/docs/components/vector-databases/astradb.mdx index f49f5140..2258326e 100644 --- a/docs/components/vector-databases/astradb.mdx +++ b/docs/components/vector-databases/astradb.mdx @@ -22,6 +22,7 @@ npm install @llm-tools/embedjs-astradb import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { AstraDb } from '@llm-tools/embedjs-astradb'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/hnswlib.mdx b/docs/components/vector-databases/hnswlib.mdx index d58b31b3..2ecdf905 100644 --- a/docs/components/vector-databases/hnswlib.mdx +++ b/docs/components/vector-databases/hnswlib.mdx @@ -17,6 +17,7 @@ npm install @llm-tools/embedjs-hnswlib import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/lancedb.mdx b/docs/components/vector-databases/lancedb.mdx index 928efe82..3e068e54 100644 --- a/docs/components/vector-databases/lancedb.mdx +++ b/docs/components/vector-databases/lancedb.mdx @@ -18,6 +18,7 @@ npm install @llm-tools/embedjs-lancedb import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { LanceDb } from '@llm-tools/embedjs-lancedb'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/mongodb.mdx b/docs/components/vector-databases/mongodb.mdx index 54270375..8681651a 100644 --- a/docs/components/vector-databases/mongodb.mdx +++ b/docs/components/vector-databases/mongodb.mdx @@ -32,6 +32,7 @@ npm install @llm-tools/embedjs-mongodb import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { MongoDb } from '@llm-tools/embedjs-mongodb'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/pinecone.mdx b/docs/components/vector-databases/pinecone.mdx index 9f73b52a..922043bc 100644 --- a/docs/components/vector-databases/pinecone.mdx +++ b/docs/components/vector-databases/pinecone.mdx @@ -23,6 +23,7 @@ npm install @llm-tools/embedjs-pinecone import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { PineconeDb } from '@llm-tools/embedjs-pinecone'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/qdrant.mdx b/docs/components/vector-databases/qdrant.mdx index b623e52a..05d3b1c7 100644 --- a/docs/components/vector-databases/qdrant.mdx +++ b/docs/components/vector-databases/qdrant.mdx @@ -16,6 +16,7 @@ npm install @llm-tools/embedjs-pinecone import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { QdrantDb } from '@llm-tools/embedjs-qdrant'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/components/vector-databases/weaviate.mdx b/docs/components/vector-databases/weaviate.mdx index 89cb1c94..c4a6130f 100644 --- a/docs/components/vector-databases/weaviate.mdx +++ b/docs/components/vector-databases/weaviate.mdx @@ -16,6 +16,7 @@ npm install @llm-tools/embedjs-weaviate import { RAGApplicationBuilder } from '@llm-tools/embedjs'; import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; import { WeaviateDb } from '@llm-tools/embedjs-weaviate'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; // set OPENAI_API_KEY in your env process.env.OPENAI_API_KEY = "sk-xxx"; diff --git a/docs/mint.json b/docs/mint.json index 1575c197..3c812cef 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -82,6 +82,7 @@ "components/data-sources/ppt", "components/data-sources/excel", "components/data-sources/sitemap", + "components/data-sources/markdown", "components/data-sources/directory", "components/data-sources/custom" ] diff --git a/examples/confluence/src/main.ts b/examples/confluence/src/main.ts index 5133de6e..637629ec 100644 --- a/examples/confluence/src/main.ts +++ b/examples/confluence/src/main.ts @@ -8,7 +8,6 @@ const llmApplication = await new RAGApplicationBuilder() .setModel(new OpenAi({ modelName: 'gpt-4o' })) .setEmbeddingModel(new OpenAiEmbeddings()) .setVectorDatabase(new HNSWDb()) - .setSearchResultCount(30) .build(); await llmApplication.addLoader(new ConfluenceLoader({ spaceNames: ['DEMO'] })); diff --git a/examples/dynamic/src/main.ts b/examples/dynamic/src/main.ts index d29372fc..c9ab3d33 100644 --- a/examples/dynamic/src/main.ts +++ b/examples/dynamic/src/main.ts @@ -7,7 +7,6 @@ const llmApplication = await new RAGApplicationBuilder() .setModel(new OpenAi({ modelName: 'gpt-4o' })) .setEmbeddingModel(new OpenAiEmbeddings()) .setVectorDatabase(new HNSWDb()) - .setSearchResultCount(30) .build(); await llmApplication.addLoader(new UrlLoader({ url: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); diff --git a/examples/markdown/eslint.config.js b/examples/markdown/eslint.config.js new file mode 100644 index 00000000..e2a15f2a --- /dev/null +++ b/examples/markdown/eslint.config.js @@ -0,0 +1,3 @@ +import baseConfig from '../../eslint.config.js'; + +export default [...baseConfig]; diff --git a/examples/markdown/package-lock.json b/examples/markdown/package-lock.json new file mode 100644 index 00000000..bddb0203 --- /dev/null +++ b/examples/markdown/package-lock.json @@ -0,0 +1,27 @@ +{ + "name": "@llm-tools/embedjs-examples-markdown", + "version": "0.1.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@llm-tools/embedjs-examples-markdown", + "version": "0.1.1", + "dependencies": { + "dotenv": "^16.4.5" + } + }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + } + } +} diff --git a/examples/markdown/package.json b/examples/markdown/package.json new file mode 100644 index 00000000..a852a0f2 --- /dev/null +++ b/examples/markdown/package.json @@ -0,0 +1,11 @@ +{ + "name": "@llm-tools/embedjs-examples-markdown", + "version": "0.1.1", + "type": "module", + "dependencies": { + "dotenv": "^16.4.5" + }, + "scripts": { + "start": "nx run examples-markdown:serve" + } +} diff --git a/examples/markdown/project.json b/examples/markdown/project.json new file mode 100644 index 00000000..41fe2006 --- /dev/null +++ b/examples/markdown/project.json @@ -0,0 +1,57 @@ +{ + "name": "examples-markdown", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "examples/markdown/src", + "projectType": "application", + "tags": [], + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "development", + "options": { + "platform": "node", + "outputPath": "dist/examples/markdown", + "format": ["esm"], + "bundle": true, + "main": "examples/markdown/src/main.ts", + "tsConfig": "examples/markdown/tsconfig.app.json", + "generatePackageJson": false, + "esbuildOptions": { + "sourcemap": true, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": {}, + "production": { + "esbuildOptions": { + "sourcemap": false, + "outExtension": { + ".js": ".js" + } + } + } + } + }, + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "examples-markdown:build", + "runBuildTargetDependencies": true + }, + "configurations": { + "development": { + "buildTarget": "examples-markdown:build:development" + }, + "production": { + "buildTarget": "examples-markdown:build:production" + } + } + } + } +} diff --git a/examples/markdown/src/main.ts b/examples/markdown/src/main.ts new file mode 100644 index 00000000..6866ace5 --- /dev/null +++ b/examples/markdown/src/main.ts @@ -0,0 +1,14 @@ +import 'dotenv/config'; +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi, OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { MarkdownLoader } from '@llm-tools/embedjs-loader-markdown'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const llmApplication = await new RAGApplicationBuilder() + .setModel(new OpenAi({ modelName: 'gpt-4o' })) + .setEmbeddingModel(new OpenAiEmbeddings()) + .setVectorDatabase(new HNSWDb()) + .build(); + +await llmApplication.addLoader(new MarkdownLoader({ filePathOrUrl: './docs/get-started/quickstart.mdx' })); +console.log(await llmApplication.query('How do you create an embedJs application?')); diff --git a/examples/markdown/tsconfig.app.json b/examples/markdown/tsconfig.app.json new file mode 100644 index 00000000..5a8c35da --- /dev/null +++ b/examples/markdown/tsconfig.app.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "types": ["node"] + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/examples/markdown/tsconfig.json b/examples/markdown/tsconfig.json new file mode 100644 index 00000000..c60cf5c2 --- /dev/null +++ b/examples/markdown/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + } + ], + "compilerOptions": { + "esModuleInterop": true, + "target": "ES2022", + "lib": ["ES2022", "ES2022.Object"], + "module": "NodeNext", + "moduleResolution": "nodenext" + } +} diff --git a/examples/simple/src/main.ts b/examples/simple/src/main.ts index a299b52c..25671612 100644 --- a/examples/simple/src/main.ts +++ b/examples/simple/src/main.ts @@ -8,7 +8,6 @@ const llmApplication = await new RAGApplicationBuilder() .setModel(new OpenAi({ modelName: 'gpt-4o' })) .setEmbeddingModel(new OpenAiEmbeddings()) .setVectorDatabase(new HNSWDb()) - .setSearchResultCount(30) .build(); await llmApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); diff --git a/loaders/embedjs-loader-confluence/CHANGELOG.md b/loaders/embedjs-loader-confluence/CHANGELOG.md index ed0cf058..18b76e5c 100644 --- a/loaders/embedjs-loader-confluence/CHANGELOG.md +++ b/loaders/embedjs-loader-confluence/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-confluence/package.json b/loaders/embedjs-loader-confluence/package.json index 3b6e097c..c0448f37 100644 --- a/loaders/embedjs-loader-confluence/package.json +++ b/loaders/embedjs-loader-confluence/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-confluence", - "version": "0.1.16", + "version": "0.1.17", "description": "Confluence loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-loader-web": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", "confluence.js": "^1.7.4", "debug": "^4.3.7", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-csv/CHANGELOG.md b/loaders/embedjs-loader-csv/CHANGELOG.md index 384831fc..69c04a47 100644 --- a/loaders/embedjs-loader-csv/CHANGELOG.md +++ b/loaders/embedjs-loader-csv/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-csv/package.json b/loaders/embedjs-loader-csv/package.json index 28ae6aea..6ec80bc0 100644 --- a/loaders/embedjs-loader-csv/package.json +++ b/loaders/embedjs-loader-csv/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-csv", - "version": "0.1.16", + "version": "0.1.17", "description": "CSV loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "csv-parse": "^5.5.6", "debug": "^4.3.7", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-markdown/README.md b/loaders/embedjs-loader-markdown/README.md new file mode 100644 index 00000000..01785c47 --- /dev/null +++ b/loaders/embedjs-loader-markdown/README.md @@ -0,0 +1,8 @@ +# embedjs-loader-markdown + +

+NPM Version +License +

+ +This package extends and offers additional functionality to [embedJs](https://www.npmjs.com/package/@llm-tools/embedjs). Refer to the documentation there for more details. diff --git a/loaders/embedjs-loader-markdown/eslint.config.js b/loaders/embedjs-loader-markdown/eslint.config.js new file mode 100644 index 00000000..4c3c47f6 --- /dev/null +++ b/loaders/embedjs-loader-markdown/eslint.config.js @@ -0,0 +1,20 @@ +import baseConfig from '../../eslint.config.js'; +import parser from '@nx/eslint-plugin'; + +export default [ + ...baseConfig, + { + files: ['**/*.json'], + rules: { + '@nx/dependency-checks': [ + 'error', + { + ignoredFiles: ['{projectRoot}/eslint.config.{js,cjs,mjs}'], + }, + ], + }, + languageOptions: { + parser, + }, + }, +]; diff --git a/loaders/embedjs-loader-markdown/package.json b/loaders/embedjs-loader-markdown/package.json new file mode 100644 index 00000000..16d69cc4 --- /dev/null +++ b/loaders/embedjs-loader-markdown/package.json @@ -0,0 +1,43 @@ +{ + "name": "@llm-tools/embedjs-loader-markdown", + "version": "0.1.17", + "description": "XML loader for embedjs", + "dependencies": { + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", + "debug": "^4.3.7", + "md5": "^2.3.0", + "micromark": "^4.0.0", + "micromark-extension-gfm": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.1" + }, + "type": "module", + "main": "./src/index.js", + "license": "Apache-2.0", + "publishConfig": { + "access": "public" + }, + "keywords": [ + "llm", + "ai", + "gpt3", + "chain", + "prompt", + "prompt engineering", + "chatgpt", + "machine learning", + "ml", + "anthropic", + "embeddings", + "vectorstores" + ], + "author": "K V Adhityan", + "bugs": { + "url": "https://github.com/llm-tools/embedjs/issues" + }, + "homepage": "https://github.com/llm-tools/embedjs#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/llm-tools/embedjs.git" + } +} diff --git a/loaders/embedjs-loader-markdown/project.json b/loaders/embedjs-loader-markdown/project.json new file mode 100644 index 00000000..fa87e11f --- /dev/null +++ b/loaders/embedjs-loader-markdown/project.json @@ -0,0 +1,29 @@ +{ + "name": "embedjs-loader-markdown", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "loaders/embedjs-loader-markdown/src", + "projectType": "library", + "tags": [], + "targets": { + "build": { + "executor": "@nx/js:tsc", + "outputs": ["{options.outputPath}"], + "options": { + "outputPath": "dist/esm/embedjs-loader-markdown", + "main": "loaders/embedjs-loader-markdown/src/index.ts", + "tsConfig": "loaders/embedjs-loader-markdown/tsconfig.lib.json", + "assets": ["loaders/embedjs-loader-markdown/*.md"] + } + }, + "build-cjs": { + "executor": "@nx/js:tsc", + "outputs": ["{options.outputPath}"], + "dependsOn": ["^build-cjs"], + "options": { + "outputPath": "dist/cjs/embedjs-loader-markdown", + "main": "loaders/embedjs-loader-markdown/src/index.ts", + "tsConfig": "loaders/embedjs-loader-markdown/tsconfig.cjs.json" + } + } + } +} diff --git a/loaders/embedjs-loader-markdown/src/index.ts b/loaders/embedjs-loader-markdown/src/index.ts new file mode 100644 index 00000000..145df741 --- /dev/null +++ b/loaders/embedjs-loader-markdown/src/index.ts @@ -0,0 +1 @@ +export { MarkdownLoader } from './markdown-loader.js'; diff --git a/loaders/embedjs-loader-markdown/src/markdown-loader.ts b/loaders/embedjs-loader-markdown/src/markdown-loader.ts new file mode 100644 index 00000000..f77b801b --- /dev/null +++ b/loaders/embedjs-loader-markdown/src/markdown-loader.ts @@ -0,0 +1,61 @@ +import { micromark } from 'micromark'; +import { mdxJsx } from 'micromark-extension-mdx-jsx'; +import { gfmHtml, gfm } from 'micromark-extension-gfm'; +import createDebugMessages from 'debug'; +import fs from 'node:fs'; +import md5 from 'md5'; + +import { BaseLoader } from '@llm-tools/embedjs-interfaces'; +import { getSafe, isValidURL, stream2buffer } from '@llm-tools/embedjs-utils'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +export class MarkdownLoader extends BaseLoader<{ type: 'MarkdownLoader' }> { + private readonly debug = createDebugMessages('embedjs:loader:MarkdownLoader'); + private readonly filePathOrUrl: string; + private readonly isUrl: boolean; + + constructor({ + filePathOrUrl, + chunkOverlap, + chunkSize, + }: { + filePathOrUrl: string; + chunkSize?: number; + chunkOverlap?: number; + }) { + super(`MarkdownLoader_${md5(filePathOrUrl)}`, { filePathOrUrl }, chunkSize ?? 1000, chunkOverlap ?? 0); + + this.filePathOrUrl = filePathOrUrl; + this.isUrl = isValidURL(filePathOrUrl) ? true : false; + } + + override async *getUnfilteredChunks() { + const buffer = this.isUrl + ? (await getSafe(this.filePathOrUrl, { format: 'buffer' })).body + : await stream2buffer(fs.createReadStream(this.filePathOrUrl)); + + this.debug('MarkdownLoader stream created'); + const result = micromark(buffer, { extensions: [gfm(), mdxJsx()], htmlExtensions: [gfmHtml()] }); + this.debug('Markdown parsed...'); + + const webLoader = new WebLoader({ + urlOrContent: result, + chunkSize: this.chunkSize, + chunkOverlap: this.chunkOverlap, + }); + + for await (const result of await webLoader.getUnfilteredChunks()) { + result.pageContent = result.pageContent.replace(/[\[\]\(\)\{\}]/g, ''); + + yield { + pageContent: result.pageContent, + metadata: { + type: 'MarkdownLoader' as const, + source: this.filePathOrUrl, + }, + }; + } + + this.debug(`MarkdownLoader for filePathOrUrl '${this.filePathOrUrl}' finished`); + } +} diff --git a/loaders/embedjs-loader-markdown/tsconfig.cjs.json b/loaders/embedjs-loader-markdown/tsconfig.cjs.json new file mode 100644 index 00000000..1be21d0d --- /dev/null +++ b/loaders/embedjs-loader-markdown/tsconfig.cjs.json @@ -0,0 +1,7 @@ +{ + "extends": "./tsconfig.lib.json", + "compilerOptions": { + "module": "commonjs", + "moduleResolution": "Node10" + } +} diff --git a/loaders/embedjs-loader-markdown/tsconfig.json b/loaders/embedjs-loader-markdown/tsconfig.json new file mode 100644 index 00000000..eeb778bc --- /dev/null +++ b/loaders/embedjs-loader-markdown/tsconfig.json @@ -0,0 +1,26 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "ES2022.Object"], + "module": "NodeNext", + "moduleResolution": "nodenext", + "esModuleInterop": true, + "declaration": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "useDefineForClassFields": true, + "strictPropertyInitialization": false, + "allowJs": false, + "strict": false + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/loaders/embedjs-loader-markdown/tsconfig.lib.json b/loaders/embedjs-loader-markdown/tsconfig.lib.json new file mode 100644 index 00000000..bdeb03cf --- /dev/null +++ b/loaders/embedjs-loader-markdown/tsconfig.lib.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/loaders/embedjs-loader-msoffice/CHANGELOG.md b/loaders/embedjs-loader-msoffice/CHANGELOG.md index 8d5618c8..8135da97 100644 --- a/loaders/embedjs-loader-msoffice/CHANGELOG.md +++ b/loaders/embedjs-loader-msoffice/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-msoffice/package.json b/loaders/embedjs-loader-msoffice/package.json index afba1d32..9ff3ce3d 100644 --- a/loaders/embedjs-loader-msoffice/package.json +++ b/loaders/embedjs-loader-msoffice/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-msoffice", - "version": "0.1.16", + "version": "0.1.17", "description": "Word, PPT and Excel loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" }, diff --git a/loaders/embedjs-loader-pdf/CHANGELOG.md b/loaders/embedjs-loader-pdf/CHANGELOG.md index feca10ae..03c97689 100644 --- a/loaders/embedjs-loader-pdf/CHANGELOG.md +++ b/loaders/embedjs-loader-pdf/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-pdf/package.json b/loaders/embedjs-loader-pdf/package.json index f8a66ed8..03fb1f7d 100644 --- a/loaders/embedjs-loader-pdf/package.json +++ b/loaders/embedjs-loader-pdf/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-pdf", - "version": "0.1.16", + "version": "0.1.17", "description": "PDF loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" }, diff --git a/loaders/embedjs-loader-sitemap/CHANGELOG.md b/loaders/embedjs-loader-sitemap/CHANGELOG.md index fb423e36..ac67a99c 100644 --- a/loaders/embedjs-loader-sitemap/CHANGELOG.md +++ b/loaders/embedjs-loader-sitemap/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-sitemap/package.json b/loaders/embedjs-loader-sitemap/package.json index a96bb533..a31638cc 100644 --- a/loaders/embedjs-loader-sitemap/package.json +++ b/loaders/embedjs-loader-sitemap/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-sitemap", - "version": "0.1.16", + "version": "0.1.17", "description": "Sitemap recursive loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-loader-web": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", "debug": "^4.3.7", "md5": "^2.3.0", "sitemapper": "^3.2.14" diff --git a/loaders/embedjs-loader-web/CHANGELOG.md b/loaders/embedjs-loader-web/CHANGELOG.md index 727df39f..d54d4213 100644 --- a/loaders/embedjs-loader-web/CHANGELOG.md +++ b/loaders/embedjs-loader-web/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-web/package.json b/loaders/embedjs-loader-web/package.json index f70f433a..5022b053 100644 --- a/loaders/embedjs-loader-web/package.json +++ b/loaders/embedjs-loader-web/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-web", - "version": "0.1.16", + "version": "0.1.17", "description": "Web page loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "html-to-text": "^9.0.5", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-web/src/web-loader.ts b/loaders/embedjs-loader-web/src/web-loader.ts index 9926dc59..64f71455 100644 --- a/loaders/embedjs-loader-web/src/web-loader.ts +++ b/loaders/embedjs-loader-web/src/web-loader.ts @@ -34,7 +34,6 @@ export class WebLoader extends BaseLoader<{ type: 'WebLoader' }> { try { const data = this.isUrl ? (await getSafe(this.urlOrContent, { format: 'text' })).body : this.urlOrContent; - console.log('WTF', data); const text = convert(data, { wordwrap: false, diff --git a/loaders/embedjs-loader-xml/package.json b/loaders/embedjs-loader-xml/package.json index dd0409a8..56726a86 100644 --- a/loaders/embedjs-loader-xml/package.json +++ b/loaders/embedjs-loader-xml/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-loader-xml", - "version": "0.1.16", + "version": "0.1.17", "description": "XML loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "fast-xml-parser": "^4.5.0", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-youtube/CHANGELOG.md b/loaders/embedjs-loader-youtube/CHANGELOG.md index 637ef94b..bced13d9 100644 --- a/loaders/embedjs-loader-youtube/CHANGELOG.md +++ b/loaders/embedjs-loader-youtube/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/loaders/embedjs-loader-youtube/package.json b/loaders/embedjs-loader-youtube/package.json index c5cd4e8b..2a767e24 100644 --- a/loaders/embedjs-loader-youtube/package.json +++ b/loaders/embedjs-loader-youtube/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-youtube", - "version": "0.1.16", + "version": "0.1.17", "description": "Youtube transcript and channel recursive loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "md5": "^2.3.0", "usetube": "^2.2.7", diff --git a/models/embedjs-anthropic/CHANGELOG.md b/models/embedjs-anthropic/CHANGELOG.md index 2cb7f63b..58e93b08 100644 --- a/models/embedjs-anthropic/CHANGELOG.md +++ b/models/embedjs-anthropic/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-anthropic/package.json b/models/embedjs-anthropic/package.json index b5f491c7..b9005bdd 100644 --- a/models/embedjs-anthropic/package.json +++ b/models/embedjs-anthropic/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-anthropic", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of Anthropic models with embedjs", "dependencies": { "@langchain/anthropic": "^0.3.7", "@langchain/core": "^0.3.17", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-cohere/CHANGELOG.md b/models/embedjs-cohere/CHANGELOG.md index a3ce0a72..1cc2df88 100644 --- a/models/embedjs-cohere/CHANGELOG.md +++ b/models/embedjs-cohere/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-cohere/package.json b/models/embedjs-cohere/package.json index 3ce0b6f5..4099d405 100644 --- a/models/embedjs-cohere/package.json +++ b/models/embedjs-cohere/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-cohere", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of Cohere models with embedjs", "dependencies": { "@langchain/cohere": "^0.3.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "cohere-ai": "^7.14.0" }, "type": "module", diff --git a/models/embedjs-huggingface/CHANGELOG.md b/models/embedjs-huggingface/CHANGELOG.md index 0d7661f4..cbbb2fc2 100644 --- a/models/embedjs-huggingface/CHANGELOG.md +++ b/models/embedjs-huggingface/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-huggingface/package.json b/models/embedjs-huggingface/package.json index b0d1e612..73720f84 100644 --- a/models/embedjs-huggingface/package.json +++ b/models/embedjs-huggingface/package.json @@ -1,12 +1,12 @@ { "name": "@llm-tools/embedjs-huggingface", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of HuggingFace models with embedjs", "dependencies": { "@huggingface/inference": "^2.8.1", "@langchain/community": "^0.3.11", "@langchain/core": "^0.3.17", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-mistral/CHANGELOG.md b/models/embedjs-mistral/CHANGELOG.md index 9f3b9fa9..2d772e38 100644 --- a/models/embedjs-mistral/CHANGELOG.md +++ b/models/embedjs-mistral/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-mistral/package.json b/models/embedjs-mistral/package.json index 85f4cbfc..b77773e5 100644 --- a/models/embedjs-mistral/package.json +++ b/models/embedjs-mistral/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-mistral", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of Mistral models with embedjs", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/mistralai": "^0.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-ollama/CHANGELOG.md b/models/embedjs-ollama/CHANGELOG.md index 3cf348db..c93af75b 100644 --- a/models/embedjs-ollama/CHANGELOG.md +++ b/models/embedjs-ollama/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-ollama/package.json b/models/embedjs-ollama/package.json index e18089b3..627dd1df 100644 --- a/models/embedjs-ollama/package.json +++ b/models/embedjs-ollama/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-ollama", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of Ollama with embedjs", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/ollama": "^0.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-openai/CHANGELOG.md b/models/embedjs-openai/CHANGELOG.md index 24b3eeac..a28ea032 100644 --- a/models/embedjs-openai/CHANGELOG.md +++ b/models/embedjs-openai/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-openai/package.json b/models/embedjs-openai/package.json index eeff4386..12b71049 100644 --- a/models/embedjs-openai/package.json +++ b/models/embedjs-openai/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-openai", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of OpenAI models with embedjs", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/openai": "^0.3.11", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-vertexai/CHANGELOG.md b/models/embedjs-vertexai/CHANGELOG.md index 8883874f..913a4637 100644 --- a/models/embedjs-vertexai/CHANGELOG.md +++ b/models/embedjs-vertexai/CHANGELOG.md @@ -1,9 +1,15 @@ -## 0.1.16 (2024-11-04) - -### ๐Ÿš€ Features - -- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) - +## 0.1.17 (2024-11-05) + +### ๐Ÿš€ Features + +- added markdown/mdx loader ([847947d](https://github.com/llm-tools/embedJs/commit/847947d) + +## 0.1.16 (2024-11-04) + +### ๐Ÿš€ Features + +- added xml loader ([9172511](https://github.com/llm-tools/embedJs/commit/9172511)) + ## 0.1.15 and 0.1.14 (2024-11-01) ### ๐Ÿš€ Features diff --git a/models/embedjs-vertexai/package.json b/models/embedjs-vertexai/package.json index 67124488..ab8c4fce 100644 --- a/models/embedjs-vertexai/package.json +++ b/models/embedjs-vertexai/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-vertexai", - "version": "0.1.16", + "version": "0.1.17", "description": "Enable usage of VertexAI models with embedjs", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/google-vertexai": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" }, "type": "module", diff --git a/package-lock.json b/package-lock.json index 9cf23a25..8ea2f95c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,12 +48,12 @@ }, "core/embedjs": { "name": "@llm-tools/embedjs", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "langchain": "^0.3.5", "md5": "^2.3.0", @@ -68,7 +68,7 @@ }, "core/embedjs-interfaces": { "name": "@llm-tools/embedjs-interfaces", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.17", @@ -79,87 +79,87 @@ }, "core/embedjs-utils": { "name": "@llm-tools/embedjs-utils", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16" + "@llm-tools/embedjs-interfaces": "0.1.17" } }, "databases/embedjs-astra": { "name": "@llm-tools/embedjs-astradb", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@datastax/astra-db-ts": "^1.5.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "databases/embedjs-cosmos": { "name": "@llm-tools/embedjs-cosmos", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@azure/cosmos": "^4.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "databases/embedjs-hnswlib": { "name": "@llm-tools/embedjs-hnswlib", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "hnswlib-node": "^3.0.0" } }, "databases/embedjs-lancedb": { "name": "@llm-tools/embedjs-lancedb", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@lancedb/lancedb": "^0.12.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "compute-cosine-similarity": "^1.1.0" } }, "databases/embedjs-lmdb": { "name": "@llm-tools/embedjs-lmdb", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "lmdb": "^3.1.4" } }, "databases/embedjs-mongodb": { "name": "@llm-tools/embedjs-mongodb", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "mongodb": "^6.10.0" } }, "databases/embedjs-pinecone": { "name": "@llm-tools/embedjs-pinecone", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "@pinecone-database/pinecone": "^4.0.0", "debug": "^4.3.7" } }, "databases/embedjs-qdrant": { "name": "@llm-tools/embedjs-qdrant", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "@qdrant/js-client-rest": "^1.12.0", "debug": "^4.3.7", "uuid": "^11.0.2" @@ -167,19 +167,19 @@ }, "databases/embedjs-redis": { "name": "@llm-tools/embedjs-redis", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "ioredis": "^5.4.1" } }, "databases/embedjs-weaviate": { "name": "@llm-tools/embedjs-weaviate", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7", "weaviate-ts-client": "^2.2.0" @@ -187,11 +187,11 @@ }, "loaders/embedjs-loader-confluence": { "name": "@llm-tools/embedjs-loader-confluence", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-loader-web": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", "confluence.js": "^1.7.4", "debug": "^4.3.7", "md5": "^2.3.0" @@ -199,47 +199,61 @@ }, "loaders/embedjs-loader-csv": { "name": "@llm-tools/embedjs-loader-csv", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "csv-parse": "^5.5.6", "debug": "^4.3.7", "md5": "^2.3.0" } }, + "loaders/embedjs-loader-markdown": { + "name": "@llm-tools/embedjs-loader-markdown", + "version": "0.1.17", + "license": "Apache-2.0", + "dependencies": { + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", + "debug": "^4.3.7", + "md5": "^2.3.0", + "micromark": "^4.0.0", + "micromark-extension-gfm": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.1" + } + }, "loaders/embedjs-loader-msoffice": { "name": "@llm-tools/embedjs-loader-msoffice", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" } }, "loaders/embedjs-loader-pdf": { "name": "@llm-tools/embedjs-loader-pdf", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" } }, "loaders/embedjs-loader-sitemap": { "name": "@llm-tools/embedjs-loader-sitemap", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-loader-web": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-loader-web": "0.1.17", "debug": "^4.3.7", "md5": "^2.3.0", "sitemapper": "^3.2.14" @@ -247,12 +261,12 @@ }, "loaders/embedjs-loader-web": { "name": "@llm-tools/embedjs-loader-web", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "html-to-text": "^9.0.5", "md5": "^2.3.0" @@ -263,10 +277,10 @@ }, "loaders/embedjs-loader-xml": { "name": "@llm-tools/embedjs-loader-xml", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7", "fast-xml-parser": "^4.5.0", "md5": "^2.3.0" @@ -274,12 +288,12 @@ }, "loaders/embedjs-loader-youtube": { "name": "@llm-tools/embedjs-loader-youtube", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", - "@llm-tools/embedjs-utils": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", + "@llm-tools/embedjs-utils": "0.1.17", "debug": "^4.3.7", "md5": "^2.3.0", "usetube": "^2.2.7", @@ -291,34 +305,34 @@ }, "models/embedjs-anthropic": { "name": "@llm-tools/embedjs-anthropic", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/anthropic": "^0.3.7", "@langchain/core": "^0.3.17", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "models/embedjs-cohere": { "name": "@llm-tools/embedjs-cohere", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/cohere": "^0.3.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "cohere-ai": "^7.14.0" } }, "models/embedjs-huggingface": { "name": "@llm-tools/embedjs-huggingface", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@huggingface/inference": "^2.8.1", "@langchain/community": "^0.3.11", "@langchain/core": "^0.3.17", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, @@ -845,45 +859,45 @@ }, "models/embedjs-mistral": { "name": "@llm-tools/embedjs-mistral", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/mistralai": "^0.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "models/embedjs-ollama": { "name": "@llm-tools/embedjs-ollama", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/ollama": "^0.1.1", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "models/embedjs-openai": { "name": "@llm-tools/embedjs-openai", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/openai": "^0.3.11", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, "models/embedjs-vertexai": { "name": "@llm-tools/embedjs-vertexai", - "version": "0.1.16", + "version": "0.1.17", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.17", "@langchain/google-vertexai": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.16", + "@llm-tools/embedjs-interfaces": "0.1.17", "debug": "^4.3.7" } }, @@ -5936,6 +5950,10 @@ "resolved": "loaders/embedjs-loader-csv", "link": true }, + "node_modules/@llm-tools/embedjs-loader-markdown": { + "resolved": "loaders/embedjs-loader-markdown", + "link": true + }, "node_modules/@llm-tools/embedjs-loader-msoffice": { "resolved": "loaders/embedjs-loader-msoffice", "link": true @@ -8206,6 +8224,15 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/acorn": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz", + "integrity": "sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -8277,7 +8304,6 @@ "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "dev": true, "license": "MIT", "dependencies": { "@types/ms": "*" @@ -8287,9 +8313,17 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "dev": true, "license": "MIT" }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", @@ -8367,7 +8401,6 @@ "version": "0.7.34", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", - "dev": true, "license": "MIT" }, "node_modules/@types/node": { @@ -8424,6 +8457,12 @@ "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", "license": "MIT" }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, "node_modules/@types/usetube": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/@types/usetube/-/usetube-2.1.2.tgz", @@ -9574,6 +9613,16 @@ "node": ">=10" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", @@ -10142,6 +10191,19 @@ "node": ">=0.10.0" } }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/decompress-response": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", @@ -10252,6 +10314,15 @@ "node": ">=0.10" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/detect-libc": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", @@ -10289,6 +10360,19 @@ "node": ">= 4.0.0" } }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -10915,6 +10999,30 @@ "node": ">=4.0" } }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -14179,6 +14287,623 @@ "node": ">= 8" } }, + "node_modules/micromark": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.0.tgz", + "integrity": "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.1.tgz", + "integrity": "sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.0.tgz", + "integrity": "sha512-Ub2ncQv+fwD70/l4ou27b4YzfNaCJOvyX4HxXU15m7mpYY+rjuWzsLIPZHJL253Z643RpbcP1oeIJlQ/SKW67g==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.1.tgz", + "integrity": "sha512-vNuFb9czP8QCtAQcEJn0UJQJZA8Dk6DXKBqx+bg/w0WGuSxDxNr7hErW89tHUY31dUW4NqEOWwmEUNhjTFmHkg==", + "license": "MIT", + "dependencies": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", + "integrity": "sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.0.tgz", + "integrity": "sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.2.tgz", + "integrity": "sha512-5E5I2pFzJyg2CtemqAbcyCktpHXuJbABnsb32wX2U8IQKhhVFBqkcZR5LRm1WVoFqa4kTueZK4abep7wdo9nrw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.0.tgz", + "integrity": "sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.0.tgz", + "integrity": "sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.0.tgz", + "integrity": "sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.0.tgz", + "integrity": "sha512-KvOVV+X1yLBfs9dCBSopq/+G1PcgT3lAK07mC4BzXi5E7ahzMAF8oIupDDJ6mievI6F+lAATkbQQlQixJfT3aQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.0.tgz", + "integrity": "sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.0.tgz", + "integrity": "sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.0.tgz", + "integrity": "sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.1.tgz", + "integrity": "sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.0.tgz", + "integrity": "sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.2.tgz", + "integrity": "sha512-Fk+xmBrOv9QZnEDguL9OI9/NQQp6Hz4FuQ4YmCb/5V7+9eAh1s6AYSvL20kHkD67YIg7EpE54TiSlcsf3vyZgA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.0.tgz", + "integrity": "sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.0.tgz", + "integrity": "sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.0.tgz", + "integrity": "sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.0.tgz", + "integrity": "sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.1.tgz", + "integrity": "sha512-jZNtiFl/1aY73yS3UGQkutD0UbhTt68qnRpw2Pifmz5wV9h8gOVsN70v+Lq/f1rKaU/W8pxRe8y8Q9FX1AOe1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.0.tgz", + "integrity": "sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.0.tgz", + "integrity": "sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", @@ -14983,9 +15708,9 @@ } }, "node_modules/openai": { - "version": "4.70.3", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.70.3.tgz", - "integrity": "sha512-N2XOWjuT5yKIdLgjZkQt9i5+cAXI7qKM7E5PpIsmVfnTi/Y812omr3rozgKwxXJC6aga8nl2BWos4HRdlZllFA==", + "version": "4.71.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.71.0.tgz", + "integrity": "sha512-jeJ7+6cZvj+ZbIsbX/Ag8+pug2+vjKbrD/v3Hwp6uv3KZyWjSkZa5MdUshzpNC3jsFzakfbUhEEFQXsKWNgm/g==", "license": "Apache-2.0", "dependencies": { "@types/node": "^18.11.18", @@ -17085,6 +17810,32 @@ "node": ">=4" } }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/universalify": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", @@ -17253,6 +18004,20 @@ "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==" }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", diff --git a/tsconfig.base.json b/tsconfig.base.json index 490167d2..59188f4c 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -28,6 +28,7 @@ "@llm-tools/embedjs-lmdb": ["databases/embedjs-lmdb/src/index.ts"], "@llm-tools/embedjs-loader-confluence": ["loaders/embedjs-loader-confluence/src/index.ts"], "@llm-tools/embedjs-loader-csv": ["loaders/embedjs-loader-csv/src/index.ts"], + "@llm-tools/embedjs-loader-markdown": ["loaders/embedjs-loader-markdown/src/index.ts"], "@llm-tools/embedjs-loader-msoffice": ["loaders/embedjs-loader-msoffice/src/index.ts"], "@llm-tools/embedjs-loader-pdf": ["loaders/embedjs-loader-pdf/src/index.ts"], "@llm-tools/embedjs-loader-sitemap": ["loaders/embedjs-loader-sitemap/src/index.ts"],