diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..c4cc3641 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,33 @@ +name: πŸ› Bug Report +description: Create a report to help us reproduce and fix the bug + +body: + - type: markdown + attributes: + value: > + #### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/llm-tools/embedJs/issues?q=is%3Aissue+sort%3Acreated-desc+). + - type: textarea + attributes: + label: πŸ› Describe the bug + description: | + Please provide a clear and concise description of what the bug is. + + If relevant, add a minimal example so that we can reproduce the error by running the code. It is very important for the snippet to be as succinct (minimal) as possible, so please take time to trim down any irrelevant code to help us debug efficiently. We are going to copy-paste your code and we expect to get the same result as you did: avoid any external data, and include the relevant imports, etc. + + Please also paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````. + placeholder: | + A clear and concise description of what the bug is. + + ```ts + Sample code to reproduce the problem + ``` + + ``` + The error message you got, with the full traceback. + ```` + validations: + required: true + - type: markdown + attributes: + value: > + Thanks for contributing πŸŽ‰! diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..407d17fc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Urgent issue + url: https://www.linkedin.com/in/adhityan + about: If your issue is urgent and needs immidiately attention, please contact the author directly on LinkedIn. diff --git a/.github/ISSUE_TEMPLATE/documentation_issue.yml b/.github/ISSUE_TEMPLATE/documentation_issue.yml new file mode 100644 index 00000000..3a162f2f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_issue.yml @@ -0,0 +1,11 @@ +name: Documentation +description: Report an issue related to the docs. +title: "DOC: " + +body: + - type: textarea + attributes: + label: 'Issue with current documentation:' + description: > + Please make sure to leave a reference to the document/code you're + referring to. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..1fe30721 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,23 @@ +name: πŸš€ Feature request +description: Submit a proposal/request for a new EmbedJs feature + +body: + - type: textarea + id: feature-request + attributes: + label: πŸš€ The feature + description: > + A clear and concise description of the feature proposal + validations: + required: true + - type: textarea + attributes: + label: Motivation, pitch + description: > + Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., *"I'm working on X and would like Y to be possible"*. If this is related to another GitHub issue, please link here too. + validations: + required: true + - type: markdown + attributes: + value: > + Thanks for contributing πŸŽ‰! diff --git a/.prettierignore b/.prettierignore index e26f0b3f..a3a57dca 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,4 +2,5 @@ /dist /coverage /.nx/cache -/.nx/workspace-data \ No newline at end of file +/.nx/workspace-data +*.mdx diff --git a/README.md b/README.md index 0bf37efe..89d58035 100644 --- a/README.md +++ b/README.md @@ -1,1308 +1,30 @@ -# EmbedJs - -

-License -NPM Version +

+ EmbedJs Logo

-EmbedJs is an Open Source Framework for personalizing LLM responses. An ultimate toolkit for building powerful Retrieval-Augmented Generation (RAG) and Large Language Model (LLM) applications with ease in Node.js. - -It segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. It enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data. - -Here's an example of how easy it is to get started - - -```TS -const ragApplication = await new RAGApplicationBuilder() - .addLoader({ type: 'YoutubeSearch', youtubeSearchString: 'Tesla cars' }) - .addLoader('https://en.wikipedia.org/wiki/Tesla,_Inc.') - .addLoader('https://tesla-info.com/sitemap.xml') - .setVectorDb(new LanceDb({ path: '.db' })) - .build(); -``` - -That's it. Now you can ask questions - - -```TS -console.log(await ragApplication.query('Give me the history of Tesla?')); -``` - -## Features - -- Supports all popular large language models - paid and open source - -- Supports many vector databases including self-hosted and cloud variants. - -- Load different kinds of unstructured data. Comes built in with several loaders that makes this easy. - -- Supports several cache options that can greatly improve the performance of your RAG applications in production. - -- Exposes a simple and highly configureable API allows both quick launch and deep customizabilty. - -- Use just as an embedding engine or a full blown chat API with history - -## Quick note - -The author(s) are looking to add core maintainers for this opensource project. Reach out on [Linkedin](https://www.linkedin.com/in/adhityan/) if you are interested. If you want to contribute in general - create issues on GitHub or send in PRs. - -# Contents - -- [EmbedJs](#embedjs) - - [Features](#features) - - [Quick note](#quick-note) -- [Contents](#contents) -- [Getting started](#getting-started) - - [Installation](#installation) - - [Usage](#usage) - - [Temperature](#temperature) - - [Search results count](#search-results-count) - - [Customize the prompt](#customize-the-prompt) - - [Get context (dry run)](#get-context-dry-run) - - [Delete loader](#delete-loader) - - [Get count of embedded chunks](#get-count-of-embedded-chunks) - - [Remove all embeddings / reset](#remove-all-embeddings--reset) - - [Set cut-off for relevance](#set-cut-off-for-relevance) - - [Add new loaders later](#add-new-loaders-later) - - [Loader inference](#loader-inference) -- [Loaders supported](#loaders-supported) - - [Youtube video](#youtube-video) - - [Youtube channel](#youtube-channel) - - [Youtube search](#youtube-search) - - [PDF file](#pdf-file) - - [Docx file](#docx-file) - - [Excel file](#excel-file) - - [Powerpoint file](#powerpoint-file) - - [Web page](#web-page) - - [Confluence](#confluence) - - [Sitemap](#sitemap) - - [Text](#text) - - [Json](#json) - - [Csv](#csv) - - [Add a custom loader](#add-a-custom-loader) - - [More loaders coming soon](#more-loaders-coming-soon) -- [LLMs](#llms) - - [OpenAI](#openai) - - [Azure OpenAI](#azure-openai) - - [Mistral](#mistral) - - [Hugging Face](#hugging-face) - - [Anthropic](#anthropic) - - [Vertex AI](#vertex-ai) - - [Ollama](#ollama) - - [Use custom LLM model](#use-custom-llm-model) - - [More LLMs coming soon](#more-llms-coming-soon) -- [Embedding models](#embedding-models) - - [OpenAI v3 Small](#openai-v3-small) - - [OpenAI v3 Large](#openai-v3-large) - - [Ada](#ada) - - [Cohere](#cohere) - - [Gecko Embedding](#gecko-embedding) - - [Ollama local embedding](#ollama-local-embedding) - - [Use custom embedding model](#use-custom-embedding-model) - - [More embedding models coming soon](#more-embedding-models-coming-soon) -- [Vector databases supported](#vector-databases-supported) - - [Pinecone](#pinecone) - - [LanceDB](#lancedb) - - [Chroma](#chroma) - - [HNSWLib](#hnswlib) - - [Weaviate](#weaviate) - - [Qdrant](#qdrant) - - [MongoDB (vector database)](#mongodb-vector-database) - - [Astra DB](#astra-db) - - [Bring your own database](#bring-your-own-database) - - [More databases coming soon](#more-databases-coming-soon) -- [Caches](#caches) - - [LMDB](#lmdb) - - [InMemory (cache)](#inmemory-cache) - - [Redis](#redis) - - [MongoDB (cache)](#mongodb-cache) - - [Bring your own cache](#bring-your-own-cache) - - [More caches coming soon](#more-caches-coming-soon) -- [Conversation history](#conversation-history) - - [InMemory (conversation)](#inmemory-conversation) - - [MongoDB (conversation)](#mongodb-conversation) -- [Langsmith Integration](#langsmith-integration) -- [Sample projects](#sample-projects) -- [Contributing](#contributing) - -# Getting started - -## Installation - -You can install the library via NPM or Yarn - -```bash -npm i @llm-tools/embedjs -``` - -## Usage - -To configure a new EmbedJs application, you need to do three steps - - -> **1. Pick an LLM**
-> The library supports several LLMs. Activate one by allowing the instructions in the [LLM](#llms) section. - -```TS -const ragApplication = await new RAGApplicationBuilder() - .setModel(new HuggingFace({ modelName: 'mistralai/Mixtral-8x7B-v0.1' })) - ... -``` - -**Note:** To use the library only for embeddings and not instantiate a LLM, you can pass the string `NO_MODEL` to the setModel function here. This will disable the option to call the `query` function but you can still get the embeddings with the [`getContext`](#get-context-dry-run) method. - -> **2. Pick a Vector database**
-> The library supports several vector databases. Enable one by allowing the instructions in the [Vector Databases](#vector-databases-supported) section. - -```TS - .setVectorDb(new PineconeDb({ projectName: 'test', namespace: 'dev' })) -``` - -> **3. Load some data**
-> The library supports several kinds of loaders. You can use zero, one or many kinds of loaders together to import custom knowledge. Read the [loaders](#loaders-supported) section to learn more about the different supported loaders. - -```TS - .addLoader(new YoutubeSearchLoader({ searchString: 'Tesla cars' })) - .addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })) - .build(); -``` - -That's it! Now that you have your instance of `RAGApplication`, you can use it to query against the loaded data sets, like so - - -```TS -await ragApplication.query('What is Tesla?'); -``` - -## Temperature - -The temperature is a number between 0 and 1. It governs the randomness and creativity of the LLM responses. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. You can alter it by - - -```TS -await new RAGApplicationBuilder() -.setTemperature(0.1) -``` - -**NOTE:** The default value is 0.1, which makes the GPT responses very precise. - -## Search results count - -This is the number of documents to aim for when retrieving results from the vector database. A high number of results might mean there is more non-relevant data in the context. A low number might mean none of the relevant documents are retrieved. You need to set the number that works best for you. The parameter can be altered by - - -```TS -await new RAGApplicationBuilder() -.setSearchResultCount(10) -``` - -**NOTE:** The default value is 7. - -It is important to note that the library does not simply dump all contextual document chunks into the prompt. It sends them to the model marking them as context documents. The number of documents still counts toward the token limit. - -When the number of documents fetched leads to a request above the token limit, the library uses the following strategy - - -> It runs a preprocessing step to select relevant sections from each document until the total number of tokens is less than the maximum number of tokens allowed by the model. It then uses the transformed documents as context to answer the question. - -## Customize the prompt - -LLM models need some care. The models are notorious for inventing responses when they don't know the answer. Keeping this in mind, the library auto adds a wrapper to all user queries. The default prompt is - - -> Use all the provided context to answer the query at the end. Answer in full. If you don't know the answer, just say that you don't know, don't try to make up an answer. Query: {0} - -The placeholder `{0}` is replaced with the input query. In some cases, you may want to customize this prompt. This can be done with ease by - - -```TS -await new RAGApplicationBuilder() -.setQueryTemplate('My own query template') -``` - -## Get context (dry run) - -During development, you may want to test the performance and quality of the `Loaders` you have enabled without making any LLM calls. You can do this by using the `getContext` method - - -```TS -await ragApplication.getContext('What is Steve Jobs?') -``` - -## Delete loader - -You can remove the embeddings added from a specific loader by calling the `deleteLoader` method with the uniqueId of the loader. - -```TS -await ragApplication.deleteLoader('uniqueId...', true) -``` - -## Get count of embedded chunks - -You can fetch the count of embeddedings stored in your vector database at any time by calling the `getEmbeddingsCount` method - - -```TS -await ragApplication.getEmbeddingsCount() -``` - -## Remove all embeddings / reset - -You can remove all stored embeddings in the vectorDb using the `deleteAllEmbeddings` method - - -```TS -await ragApplication.deleteAllEmbeddings(true) -``` - -## Set cut-off for relevance - -The library can filter the embeddings returned from a vector store that have a low relevance score to the query being asked. To do this, set the cut-off value using the `setEmbeddingRelevanceCutOff` method - - -```TS -await ragApplication.setEmbeddingRelevanceCutOff(0.23) -``` - -## Add new loaders later - -You can add new loaders at any point dynamically (even after calling the `build` function on `RAGApplicationBuilder`). To do this, simply call the `addLoader` method - - -```TS -await ragApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -``` - -**Note:** Do not forget to await the dynamically added loaders to ensure you wait for the load to complete before making queries on it. - -## Loader inference - -You can add most loaders by passing a string to the `addLoader` or the `addLoaders` methods. The value can be a URL, path, JSON or youtube video id. The library will infer the type of content and invoke the appropirate loader automatically. - -```TS -await ragApplication.addLoader('pQiT2U5E9tI'); //invokes youtube URL -await ragApplication.addLoader('https://lamport.azurewebsites.net/pubs/paxos-simple.pdf'); //invokes PDF loader -``` - -**Note:** If you pass the path to a local directory, every file in that directory is recursively added (including subfolders)! - -# Loaders supported - -Loaders take a specific format, process the input and create chunks of the data. You can import all the loaders from the path `@llm-tools/embedjs`. Currently, the library supports the following formats - - -## Youtube video - -To add any youtube video to your app, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeLoader` - -```TS -.addLoader(new YoutubeLoader({ videoIdOrUrl: 'w2KbwC-s7pY' })) -``` - -## Youtube channel - -To add all videos in a youtube channel, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeChannelLoader` - -```TS -.addLoader(new YoutubeChannelLoader({ youtubeChannelId: '...' })) -``` - -## Youtube search - -To do a general youtube search and add the popular search results, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeSearchLoader` - -```TS -.addLoader(new YoutubeSearchLoader({ youtubeSearchString: '...' })) -``` - -## PDF file - -To add a pdf file, use `PdfLoader`. To get started, install the PdfLoader addon - - -```bash -npm install @llm-tools/embedjs-loader-pdf -``` - -Now, you can add a local file - - -```TS -.addLoader(new PdfLoader({ filePathOrUrl: path.resolve('paxos-simple.pdf') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new PdfLoader({ url: 'https://lamport.azurewebsites.net/pubs/paxos-simple.pdf' })) -``` - -**Note:** Currently there is no support for PDF forms and password protected documents - -## Docx file - -To add a docx file, use `DocxLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new DocxLoader({ filePathOrUrl: path.resolve('paxos.docx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new DocxLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Excel file - -To add an excel xlsx file, use `ExcelLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new ExcelLoader({ filePathOrUrl: path.resolve('numbers.xlsx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new ExcelLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Powerpoint file - -To add an powerpoint / pptx file, use `PptLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new PptLoader({ filePathOrUrl: path.resolve('wow.pptx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new PptLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Web page - -To add a web page, follow these steps - - -- Install Web loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-web -``` - -- Now you can use the `WebLoader` - -```TS -.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Formula_One' })) -``` - -## Confluence - -To add a confluence space, follow these steps - - -- Install the Confluence loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-confluence -``` - -- set the following environment variables - - -```bash -CONFLUENCE_BASE_URL= -CONFLUENCE_USER_NAME= -CONFLUENCE_API_TOKEN= -``` - -**Note:** The confluence space name is the value you see in the url in the space overview page `/wiki/spaces/{{ space name }}/overview`. - -- Now you can use the `ConfluenceLoader` - -```TS -.addLoader(new ConfluenceLoader({ spaceNames: ['...'] })) -``` - -## Sitemap - -To add a XML sitemap, follow these steps - - -- Install the Sitemap loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-sitemap -``` - -- Now you can use the `SitemapLoader` - -```TS -.addLoader(new SitemapLoader({ url: '...' })) -``` - -This will load all URLs in a sitemap via the WebLoader. - -## Text - -To supply your own text, use `TextLoader`. - -```TS -.addLoader(new TextLoader({ text: 'The best company name for a company making colorful socks is MrSocks' })) -``` - -**Note:** Feel free to add your custom text without worrying about duplication. The library will chuck, cache and update the vector databases without duplication. - -## Json - -To add a parsed Javascript object to your embeddings, use `JsonLoader`. The library will not parse a string to JSON on its own but once this is done, it can be injested easily. - -```TS -.addLoader(new JsonLoader({ object: { key: value, ... } })) -``` - -**Note:** if you want to restrict the keys that get added to the vectorDb in a dynamically obtained object, you can use the `pickKeysForEmbedding` optional parameter in the `JsonLoader` constructor. - -## Csv - -To add a Csv file (or URL) to your embeddings, use `CsvLoader`. The library will parse the Csv and add each row to its vector database. To get started, follow these steps - - -- Install the Csv loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-csv -``` - -- Now you can use the `CsvLoader` like so - - -```TS -.addLoader(new CsvLoader({ filePathOrUrl: '...' })) -``` - -**Note:** You can control how the `CsvLoader` parses the file in great detail by passing in the optional `csvParseOptions` constructor parameter. - -## Add a custom loader - -You can pass along a custom loader to the `addLoader` method by extending and implementing the abstract class `BaseLoader`. Here's how that would look like - - -```TS -class CustomLoader extends BaseLoader<{ customChunkMetadata: string }> { - constructor() { - super('uniqueId'); - } - - async *getChunks() { - throw new Error('Method not implemented.'); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a common loader pattern, so the community can benefit from it. - -## More loaders coming soon - -If you want to add any other format, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported formats. All PRs are welcome. - -# LLMs - -It's relatively easy to switch between different LLMs using the library. You can import any of the LLMs from the path `@llm-tools/embedjs`. We support the following LLMs today - - -## OpenAI - -To use the OpenAI LLM models, you need a API key from OpenAI. You can alternatively use Azure OpenAI to run these models. Read the [Azure OpenAI](#azure-openai) section below to learn more about this. In this section, we will cover how to use OpenAI provided LLMs. - -The first step is to obtain an API Key from OpenAI. You can do this by visiting their [API Portal](https://platform.openai.com/api-keys). Once you obtain a key, set it as an environment variable, like so - - -```bash -OPENAI_API_KEY="" -``` - -Next, you need to install the OpenAI addon - - -```bash -npm install @llm-tools/embedjs-openai -``` - -Once these two steps are done, it is relatively easy to run OpenAI LLMs. All you need is to indicate the model type you want to run. - -- For GPT 3.5 Turbo - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(SIMPLE_MODELS.OPENAI_GPT3_TURBO) -``` - -- For GPT 4 - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(SIMPLE_MODELS.OPENAI_GPT4) -``` - -- To use a custom model name - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new OpenAi({ modelName: 'gpt-4' })) -``` - -**Note:** GPT 3.5 Turbo is used as the default model if you do not specifiy one. - -## Azure OpenAI - -In order to be able to use an OpenAI model on Azure, it first needs to be deployed. Please refer to [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/) on how to deploy a model on Azure. To run this library, you will need to deploy two models - - -- text-embedding-ada (or any of the newer models) -- GPT-3.5-turbo (or the 4 series) - -Once these models are deployed, using Azure OpenAI instead of the regular OpenAI is easy to do. Just follow these steps - - -- Remove the `OPENAI_API_KEY` environment variable if you have set it already. - -- Set the following environment variables - - -```bash -# Set this to `azure` -OPENAI_API_TYPE=azure -# The API version you want to use -AZURE_OPENAI_API_VERSION=2023-05-15 -# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. -export AZURE_OPENAI_BASE_PATH=https://your-resource-name.openai.azure.com/openai/deployments -# The API key1 or key2 for your Azure OpenAI resource -export AZURE_OPENAI_API_KEY= -# The deployment name you used for your embedding model -AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=text-embedding-ada-002 -# The deployment name you used for your llm -AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-35-turbo -``` - -You are now all set and can now run the Azure OpenAI LLMs using the [`OpenAi` model](#openai) steps detailed above. - -## Mistral - -To use Mirstal's models, you will need to get an API Key from Mistral. You can do this from their [console](https://console.mistral.ai/user/api-keys/). Next, you need to install the Mistral addon to embedJs - - -```bash -npm install @llm-tools/embedjs-mistral -``` - -Once these steps are complete, set Mistral as your LLM of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Mistral({ accessToken: "" })) -``` - -By default, the `mistral-medium` model from Mistral is used. If you want to use a different Mistral model, you can specify it via the optional parameter to the Mistral constructor, like so - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Mistral({ accessToken: "", modelName: "..." })) -``` - -**Note:** If you want to run Mistral open source for free, you can do so using the HuggingFace platform (read below). Just make sure to set the modelName to `mistralai/Mistral-7B-v0.1` or the version you want to run. - -## Hugging Face - -Hugging face needs no introduction. They host a variety of open source LLMs and you can use most of them for free. To run hugging face inference based LLMs with this library, you will need a free hugging face token. - -You can get an API Token by signing up for hugging face and generate a token from [this page](https://huggingface.co/settings/tokens). Once you get the token, set it to the environment like so - - -```bash -HUGGINGFACEHUB_API_KEY="" -``` - -Next you need to install the hugging face addon for embedJs - - -```bash -npm install @llm-tools/embedjs-huggingface -``` - -That's all, now you can use any of the hugging face models. To do this set `HuggingFace` as your model processor of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new HuggingFace({ modelName: "..." }))) -``` - -**Note:** Not all hugging face models are fully free to consume via their API. Since running these models takes a lot of resources, Hugging Face charges a fee for a few of the larger ones. This is the case with Meta's `meta-llama/Llama-2-7b-hf`, for example. - -To use these 'not-free' models via HuggingFace, you need to subscribe to their [Pro plan](https://huggingface.co/pricing) or create a custom [inference endpoint](https://ui.endpoints.huggingface.co/). It is possible to self host these models for free and run them locally via Ollama - support for which is coming soon. - -## Anthropic - -To use Anthropic's Claude models, you will need to get an API Key from Anthropic. You can do this from their [console](https://console.anthropic.com/settings/keys). Once you obtain a key, set it in the environment variable, like so - - -```bash -ANTHROPIC_API_KEY="" -``` - -Next, you need to install the Anthropic extension for embedJs like so - - -```bash -npm install @llm-tools/embedjs-anthropic -``` - -Once these are done, it is relatively easy to use Anthropic's Claude in your RAG application. Simply set Anthropic as your LLM of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Anthropic()) -``` - -By default, the `claude-3-sonnet-20240229` model from Anthropic is used. If you want to use a different Anthropic model, you can specify it via the optional parameter to the Anthropic constructor, like so - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Anthropic({ modelName: "..." })) -``` - -You can read more about the various models provided by Anthropic [here](https://docs.anthropic.com/claude/docs/models-overview). - -## Vertex AI - -You to use Gemini LLM and other models on Google Cloud Platform via [VertexAI](https://cloud.google.com/vertex-ai?hl=en). Read more about all the supported [LLMs](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models). - -To get started, you need to set the right access credentials to google cloud. You have two options here - - -- Authenticate by using `gcloud` CLI: - -``` -gcloud auth application-default login -``` - -- Authentication using Service Account with JSON key and environment variable: - -```bash -GOOGLE_APPLICATION_CREDENTIALS="" -``` - -- Install the vertex ai addon for embedJs - -```bash -npm install @llm-tools/embedjs-vertexai -``` - -Now all you need to do is set the model to `VertexAI`. Here's an example - - -```TS -const ragApplication = await new RAGApplicationBuilder() - .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409'})) - .setEmbeddingModel(new GeckoEmbedding()) -``` - -See also `/examples/vertexai` for [further documentation](/examples/vertexai/README.md) about authentication options and how to use it. - -**Note:** Default model is `gemini-1.0-pro`. - -## Ollama - -You can also use locally running Ollama models. Installation instructions for Ollama can be found [here](https://ollama.com/). - -Once Ollama is installed, you can start a local LLM by executing `ollama run `. Next to be able to use Ollama in embedJs, install the Ollama addon - - -```bash -npm install @llm-tools/embedjs-ollama -``` - -Once this is done, you can use that in the `Ollama` constructor by passing the `modelName` parameter. Here's an example - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Ollama({ - modelName: "llama3", - baseUrl: 'http://localhost:11434' -})) -``` - -**Note:** Default port in which Ollama runs, is `11434`, but if for some reason you use something else, you can pass `baseUrl` with the port number as the second argument: - -## Use custom LLM model - -You can use a custom LLM model by implementing the `BaseModel` interface. Here's how that would look like - - -```TS -class MyOwnLLMImplementation implements BaseModel { - override async init(): Promise {} //optional to override - - protected abstract runQuery( - system: string, //the system prompt - userQuery: string, //the current user query - supportingContext: Chunk[], //all supporting documents - pastConversations: ConversationHistory[], //the chat history so far - ): Promise { - throw new Error("Method not implemented."); - } -} -``` - -Once done, you can pass this class to the `setModel` method like shown in the examples above. That said, we really encourage you send in a PR to this library if you are implementing a famous or common LLM, so the community can benefit from it. - -## More LLMs coming soon - -If you want us to add support for a specific LLM, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will prioritize it. All PRs are welcome. - -Currently, we next plan to add support for Ollama. - -# Embedding models - -Embedding models are LLMs that convert a string into vector better suited for processing. In most cases, the default `text-embedding-3-small` model from OpenAI is going to be good enough. If you want to use this model, you do not have to do anything extra. - -However in some advanced cases, you may want to change this; after all, different embedding models perform differently under different curcumstances. The library allows you to do this using the method `setEmbeddingModel` while building the `RAGApplication`. - -The library supports the following embedding models - - -## OpenAI v3 Small - -The `text-embedding-3-small` is a new standard embedding model released by OpenAI in Jan, 2024. It is the default used by the libary. This model is cheaper and better than their older Ada model. This model returns vectors with dimension 1536. - -You do not have to do anything to enable it. - -## OpenAI v3 Large - -The `text-embedding-3-large` is also a new standard embedding model released by OpenAI in Jan, 2024. This model is the best embedding model provided by OpenAI as of now but is also the most expensive. This model returns vectors with dimension 3072. - -To set it as your model of choice - - -- Set `OpenAi3LargeEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { OpenAi3LargeEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new OpenAi3LargeEmbeddings()) -``` - -## Ada - -The `text-embedding-ada-002` is a well known model from OpenAI. You can read more about it [here](https://openai.com/blog/new-and-improved-embedding-model). This model returns vectors with dimension 1536. - -To set it as your model of choice - - -- Set `AdaEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { AdaEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new AdaEmbeddings()) -``` - -## Cohere - -The library supports usage of [Cohere-AI](https://cohere.com) `embed-english-v2.0` embedding model out of the box. This model returns vectors with dimension 4096. - -Here's what you have to do to enable it - - -- Sign up for an account with Cohere-AI if you have not done so already. Once done, go to the [API Keys](https://dashboard.cohere.ai/api-keys) section and copy an API_KEY. - -- Load the key you just obtained in the environment variable `COHERE_API_KEY` - -```bash -COHERE_API_KEY="" -``` - -- Set `CohereEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { CohereEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new CohereEmbeddings()) -``` - -## Gecko Embedding - -The libaray supports the embedding model `textembedding-gecko` with 768 dimensions on [VertexAI](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings). - -To use this, you can authenticate to Vertex AI on GCP. Refer [here](#vertex-ai) on how to do this. Once done, simply set `GeckoEmbedding` as your choice of embedding model, like so - - -```TS -import { GeckoEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new GeckoEmbeddings()) -``` - -For an example usage of GeckoEmbeddings with Gemini LLM on VertexAI check the folder `/examples/vertexai/`. - -## Ollama local embedding - -The libaray supports fully local embeddings via `Ollama`. Read more here [Ollama embeddings](https://ollama.com/blog/embedding-models). - -To use this, you need to setup and have Ollama working locally. Refer to their Github [here](https://github.com/ollama/ollama) to understand how to do this. Once done, simply set `OllamaEmbeddings` as your choice of embedding model, like so - - -```TS -import { OllamaEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new OllamaEmbeddings({ - model: '...', - baseUrl: '...' -})) -``` - -## Use custom embedding model - -You can use your own custom embedding model by implementing the `BaseEmbeddings` interface. Here's how that would look like - - -```TS -class MyOwnEmbeddingImplementation implements BaseEmbeddings { - embedDocuments(texts: string[]): Promise { - throw new Error("Method not implemented."); - } - - embedQuery(text: string): Promise { - throw new Error("Method not implemented."); - } - - getDimensions(): number { - throw new Error("Method not implemented."); - } -} -``` - -Once done, you can pass this class to the `setEmbeddingModel` method like shown in the Cohere example above. That said, we really encourage you send in a PR to this library if you are implementing a famous or common embedding provider, so the community can benefit from it. - -## More embedding models coming soon - -If you want us to add support for a specific embedding model, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will prioritize it. All PRs are welcome. - -# Vector databases supported - -The library allows you to save your processed and unique embeddings with the vector databases of your choice. Here are the supported databases right now - - -## Pinecone - -You can enable Pinecone storage by following these steps - - -- Create an account with [Pinecone](https://www.pinecone.io/) if you don't have one already. There is a _good free tier_. - -- Install pinecone package in your project - -```bash -npm install @pinecone-database/pinecone -``` - -- Set the pinecone environment variable `PINECONE_API_KEY`. This can be obtained from the **API Keys** section on the Pinecone dashboard. - -```bash -PINECONE_API_KEY= -``` - -- Set the Pinecone database as your choice of `vectorDb` - -```TS -import { PineconeDb } from '@llm-tools/embedjs/vectorDb/pinecone'; - -.setVectorDb(new PineconeDb({ - projectName: 'test', - namespace: 'dev', - indexSpec: { - pod: { - podType: 'p1.x1', - environment: 'us-east1-gcp', - }, - }, -})) -``` - -**Note:** Pinecone supports serverless and pod based index deployments. You can control how you want your index created using the indexSpec attribute. This is mandatory to be provided but comes with full type specification. Read more about configuring this [here](https://github.com/pinecone-io/pinecone-ts-client/blob/main/v2-migration.md). - -## LanceDB - -[LanceDB](https://lancedb.com/) is a local vector database with great performance. Follow these steps to use LanceDB as your vector database - - -- Install LanceDb package in your project - -```bash -npm install @lancedb/lancedb -``` - -- Set LanceDB database as your choice of `vectorDb` - -```TS -import { LanceDb } from '@llm-tools/embedjs/vectorDb/lance'; - -.setVectorDb(new LanceDb({ path: path.resolve('/db') })) -``` - -**Note:** The `path` property will be used by LanceDB to create a directory to host all the database files. There is also support for creating temporary directories for testing - - -```TS -import { LanceDb } from '@llm-tools/embedjs/vectorDb/lance'; - -.setVectorDb(new LanceDb({ path: 'lance-', isTemp: true })) -``` - -In this case, the `path` property is used as a prefix to create the temporary directory in the OS temp directory folder. - -## Chroma - -**Note:** The chroma addon is currently not published / supported. There are some type issues in the upstream dependencies that are being resolved. - -[Chroma](https://trychroma.com/) is an open source vector database. It's designed to be Python first and to connect to it from NodeJS, you will need to run Chroma in a container following the steps [listed here](https://docs.trychroma.com/deployment). Once done, follow these steps to use Chroma as your vector database - - -- Install Chroma package in your project - -```bash -npm install chromadb -``` - -- Set Chroma database as your choice of `vectorDb` - -```TS -import { ChromaDb } from '@llm-tools/embedjs/vectorDb/chroma'; - -.setVectorDb(new ChromaDb({ url: 'http://localhost:8000' })) -``` - -**Note:** This is the default url and port if you run the Chroma docker container using the command `docker-compose up -d --build`. - -A warning - - -> In our testing, chroma performed the poorest in being able to retrieve relevant documents among the supported vector databases. - -## HNSWLib - -[HNSWLib](https://github.com/nmslib/hnswlib) is an in-memory vectorstore. It is great for beginners to get started with since you do not need access to the file system or a cloud service. Follow these steps to use HNSWLib as your vector database - - -- Install HNSWLib package in your project - -```bash -npm install hnswlib-node -``` - -- Set HNSWLib database as your choice of `vectorDb` - -```TS -import { HNSWDb } from '@llm-tools/embedjs/vectorDb/hnswlib'; - -.setVectorDb(new HNSWDb()) -``` - -**Note:** This is a purely in-memory vector store. All values are lost when application is restarted. - -## Weaviate - -[Weaviate](https://weaviate.io/) is an open source vector store. You can deploy it locally on docker or use their managed cloud offering. Follow these steps to use Weaviate as your vector database - - -- Install Weaviate package in your project - -```bash -npm install weaviate-ts-client -``` - -- Set Weaviate database as your choice of `vectorDb` - -```TS -import { WeaviateDb } from '@llm-tools/embedjs/vectorDb/weaviate'; - -.setVectorDb(new WeaviateDb({ host: '...', apiKey: '...', className: '...', scheme: '...' })) -``` - -## Qdrant - -[Qdrant](https://qdrant.tech/) is an Open-Source Vector Database and Vector Search Engine written in Rust. To use it - - -- Install Qdrant package in your project - -```bash -npm install @qdrant/js-client-rest -``` - -- Set Qdrant database as your choice of `vectorDb` - -```TS -import { QdrantDb } from '@llm-tools/embedjs/vectorDb/qdrant'; - -.setVectorDb(new QdrantDb({ apiKey: '...'; url: '...'; clusterName: '...' })) -``` - -## MongoDB (vector database) - -[MongoDB](https://www.mongodb.com/products/platform/atlas-vector-search) is an open source document database. They offer a managed cloud offering **MongoDB Atlas**. As of right now, only the Atlas version supports vector search while the open source version does not. - -To use MongoDB as your vector database, follow these steps - - -- Sign up for a MongoDB Atlas account if you haven't already. Once you have signed up, you will need to spin up a new cluster (or use an existing one) - -**Note:** you will need to provision a M10 (or higher) instance type to use Atlas vector search. Cheaper instance types or the free version (M0) give an error when vector indexes are created programatically. - -- The cluster creation takes a few minutes. Once the cluster is ready, click on the connect button on the dashboard to get the connection string. - -**Note:** You will need to add a users separately and allow IP access from your relevant development and production environments. - -- Install mongodb package in your project - -```bash -npm install mongodb -``` - -- Set MongoDB database as your choice of `vectorDb` - -```TS -import { MongoDb } from '@llm-tools/embedjs/vectorDb/mongodb'; - -.setVectorDb( - new MongoDb({ - connectionString: 'mongodb+srv://:@', - }), -) -``` - -**Note:** you can also optionally configure the database and collection name the library will use with the constructor parameters `dbName` and `collectionName`. Default values are used if these are not provided. - -## Astra DB - -[Astra DB is a document database with a highly performant vector index](https://www.datastax.com/products/datastax-astra) powered by Apache Cassandra and available as a managed service. - -To use Astra DB as your vector database follow these steps: - -- [Sign up for an Astra DB account](https://astra.datastax.com/signup). It is free to sign up and doesn't require a credit card. -- Create a database (this takes a couple of minutes to provision) -- From the database overview page get the API Endpoint and generate an Application Token -- Install the Astra DB package in your project: - ```bash - npm install @datastax/astra-db-ts - ``` -- Set Astra DB as your choice of `vectorDb` - - ```TS - import { AstraDb } from '@llm-tools/embedjs/vectorDb/astra-db' - - .setVectorDb( - new AstraDb({ - endpoint: process.env.ASTRA_DB_API_ENDPOINT, - apiKey: process.env.ASTRA_DB_APP_TOKEN, - collectionName: "documents" - }), - ) - ``` - -## Bring your own database - -You can pass along your vector database to the `setVectorDb` method by implementing the interface `BaseDb`. Here's how that would look like - - -```TS -class MyOwnDb implements BaseDb { - async init({ dimensions }: { dimensions: number }): Promise { - throw new Error('Method not implemented.'); - } - - async insertChunks(chunks: EmbeddedChunk[]): Promise { - throw new Error('Method not implemented.'); - } - - async similaritySearch(query: number[], k: number): Promise { - throw new Error('Method not implemented.'); - } - - async getVectorCount(): Promise { - throw new Error('Method not implemented.'); - } - - async deleteKeys(keys: string[]): Promise { - throw new Error('Method not implemented.'); - } - - async reset(): Promise { - throw new Error('Method not implemented.'); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a famous or common database, so the community can benefit from it. - -## More databases coming soon - -If you want to add support for any other vector database, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported databases. All PRs are welcome. - -# Caches - -Caches serve to reduce re-processing embeddings, loaders and queries. There is no need to load, chunk and store a large PDF File or web page on every run. Caching smartly is built in and can be enabled out of the box simply by setting a cache processor using the method `setCache` while building the `RAGApplication`. - -The library supports the following caches - - -## LMDB - -You can use [LMDB](https://dbdb.io/db/lmdb) to cache values locally on disk. - -- Install LMDB package in your project - -```bash -npm install lmdb -``` - -- Set `LmdbCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { LmdbCache } from '@llm-tools/embedjs/cache/lmdb'; - -await new RAGApplicationBuilder() -.setCache(new LmdbCache({ path: path.resolve('./cache') })) -``` - -**Note:** The `path` property will be used by the LMDB driver to create a folder housing the LMDB database files. - -## InMemory (cache) - -You can use a simple in-memory cache to store values during testing. - -- Set `MemoryCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryCache } from '@llm-tools/embedjs/cache/memory'; - -await new RAGApplicationBuilder() -.setCache(new MemoryCache()) -``` - -**Note:** Although this cache can remove duplicate loaders and chunks, its store does not persist between process restarts. You should only be using it for testing. - -## Redis - -You can use redis as a cache to store values. - -- Set `RedisCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { RedisCache } from '@llm-tools/embedjs/cache/redis'; - -await new RAGApplicationBuilder() -.setCache(new RedisCache({ ... })) -``` - -**Note:** The library internally uses `IORedis` to work with redis. `RedisCache` constructor supports all `IORedis` constructor parameters. Check [`IORedis` documentation](https://github.com/redis/ioredis) for more detials. - -## MongoDB (cache) - -You can use a MongoDB as a cache to cache values. - -- Set `MongoCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryCache } from '@llm-tools/embedjs/cache/mongo'; - -await new RAGApplicationBuilder() -.setCache(new MongoCache({ ... })) -``` - -## Bring your own cache - -You can pass along your own cache provider to the `setCache` method by implementing the interface `BaseCache`. Here's how that would look like - - -```TS -class MyOwnCache implements BaseCache { - async init(): Promise { - throw new Error("Method not implemented."); - } - - async addLoader(loaderId: string, chunkCount: number ): Promise { - throw new Error("Method not implemented."); - } - - async getLoader(loaderId: string): Promise<{ chunkCount: number }> { - throw new Error("Method not implemented."); - } - - async hasLoader(loaderId: string): Promise { - throw new Error("Method not implemented."); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a famous or common cache provider, so the community can benefit from it. - -## More caches coming soon - -If you want to add support for any other cache providers, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported caches. All PRs are welcome. - -# Conversation history - -EmbedJS allows the addition of various storage layers for conversations. This allows the conversation history to be stored and made persistant between sessions. Like all other aspects of embedJS there is a base interface for conversations and you can create your own conversation history implementation. - -The library supports the following conversation history types out of the box - - -## InMemory (conversation) - -You can use a simple in-memory object to store conversation history during testing. This is the default activated conversation history manager if you don't specify anything else. This cache is used by **default** if no other cache is specified. - -- Set `InMemoryConversation` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryConversations } from '@llm-tools/embedjs/conversation/memory'; - -await new RAGApplicationBuilder() -.setConversationEngine(new InMemoryConversation()) -``` - -**Note:** Although this cache does remove duplicate loaders and chunks, its store does not persist between process restarts. - -## MongoDB (conversation) - -Can be used with any version of MongoDb. - -- Set `MongoConversation` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MongoConversation } from '@llm-tools/embedjs/conversation/mongo'; - -await new RAGApplicationBuilder() -.setConversationEngine(new MongoConversations({ - uri: MONGODB_URI, - dbName: DB_NAME, - collectionName: CONVERSATIONS_COLLECTION_NAME -});) -``` +

+ License + NPM Version +

-# Langsmith Integration +
-Langsmith allows you to keep track of how you use LLM and embedding models. It logs histories, token uses and other metadata. Follow these three simple steps to enable - +## What is EmbedJs? -- Sign up for an account with [Langsmith](https://smith.langchain.com/) -- Generate an API Key from your admin page -- Set the following environment keys in your project +EmbedJs is an Open Source Framework for personalizing LLM responses. An ultimate toolkit for building powerful Retrieval-Augmented Generation (RAG) and Large Language Model (LLM) applications with ease in Node.js. -```bash -export LANGCHAIN_TRACING_V2=true -export LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" -export LANGCHAIN_PROJECT="" -export LANGCHAIN_API_KEY="" -``` +It segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. It enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data. -# Sample projects +## πŸ“– Documentation -Here's a list of projects / examples built with RagKit +Comprehensive guides and API documentation are available to help you get the most out of EmbedJs: -| **Project** | **Description** | -| ------------------------------------------------------------ | ---------------------------------------------------------------------- | -| [slack-bot](https://github.com/llm-tools/slack-bot-template) | A NestJs based slack bot that can answer questions based on confluence | +- [Introduction](https://llm-tools.mintlify.app/get-started/introduction#what-is-embedjs) +- [Getting Started](https://llm-tools.mintlify.app/get-started/quickstart) +- [Supported data types](https://llm-tools.mintlify.app/components/data-sources/overview) +- [Examples](https://llm-tools.mintlify.app/examples) -# Contributing +## 🌐 Contributing Contributions are welcome! Please check out the issues on the repository, and feel free to open a pull request. For more information, please see the [contributing guidelines](CONTRIBUTING.md). diff --git a/core/embedjs-interfaces/src/index.ts b/core/embedjs-interfaces/src/index.ts index f23f0b1d..2a00fca9 100644 --- a/core/embedjs-interfaces/src/index.ts +++ b/core/embedjs-interfaces/src/index.ts @@ -3,8 +3,7 @@ import { BaseDb } from './interfaces/base-db.js'; import { BaseEmbeddings } from './interfaces/base-embeddings.js'; import { BaseCache } from './interfaces/base-cache.js'; import { BaseModel } from './interfaces/base-model.js'; -import { BaseConversation } from './interfaces/base-conversations.js'; export * from './types.js'; export * from './constants.js'; -export { BaseDb, BaseCache, BaseLoader, BaseEmbeddings, BaseModel, BaseConversation }; +export { BaseDb, BaseCache, BaseLoader, BaseEmbeddings, BaseModel }; diff --git a/core/embedjs-interfaces/src/interfaces/base-cache.ts b/core/embedjs-interfaces/src/interfaces/base-cache.ts index 662c7cb2..059d97ae 100644 --- a/core/embedjs-interfaces/src/interfaces/base-cache.ts +++ b/core/embedjs-interfaces/src/interfaces/base-cache.ts @@ -1,3 +1,5 @@ +import { Conversation, Message } from '../types.js'; + export interface BaseCache { init(): Promise; addLoader(loaderId: string, chunkCount: number): Promise; @@ -9,4 +11,11 @@ export interface BaseCache { loaderCustomGet>(loaderCombinedId: string): Promise; loaderCustomHas(loaderCombinedId: string): Promise; loaderCustomDelete(loaderCombinedId: string): Promise; + + addConversation(conversationId: string): Promise; + getConversation(conversationId: string): Promise; + hasConversation(conversationId: string): Promise; + deleteConversation(conversationId: string): Promise; + addEntryToConversation(conversationId: string, entry: Message): Promise; + clearConversations(): Promise; } diff --git a/core/embedjs-interfaces/src/interfaces/base-conversations.ts b/core/embedjs-interfaces/src/interfaces/base-conversations.ts index 2c300fdb..cec96e8f 100644 --- a/core/embedjs-interfaces/src/interfaces/base-conversations.ts +++ b/core/embedjs-interfaces/src/interfaces/base-conversations.ts @@ -1,11 +1,3 @@ -import { Conversation, Message } from '../types.js'; - export interface BaseConversation { init(): Promise; - addConversation(conversationId: string): Promise; - getConversation(conversationId: string): Promise; - hasConversation(conversationId: string): Promise; - deleteConversation(conversationId: string): Promise; - addEntryToConversation(conversationId: string, entry: Message): Promise; - clearConversations(): Promise; } diff --git a/core/embedjs-interfaces/src/interfaces/base-loader.ts b/core/embedjs-interfaces/src/interfaces/base-loader.ts index 04e4f1e3..b042cafe 100644 --- a/core/embedjs-interfaces/src/interfaces/base-loader.ts +++ b/core/embedjs-interfaces/src/interfaces/base-loader.ts @@ -9,13 +9,13 @@ export abstract class BaseLoader< T extends Record = Record, M extends Record = Record, > extends EventEmitter { - private static cache?: Pick< + private static cache: Pick< BaseCache, 'loaderCustomDelete' | 'loaderCustomGet' | 'loaderCustomHas' | 'loaderCustomSet' >; private static readonly LOADERS_LIST_CACHE_KEY = 'LOADERS_LIST_CACHE_KEY'; - public static setCache(cache?: BaseCache) { + public static setCache(cache: BaseCache) { BaseLoader.cache = cache; } @@ -72,9 +72,9 @@ export abstract class BaseLoader< constructor( uniqueId: string, loaderMetadata: Record, - chunkSize: number = 5, - chunkOverlap: number = 0, - canIncrementallyLoad: boolean = false, + chunkSize = 5, + chunkOverlap = 0, + canIncrementallyLoad = false, ) { super(); @@ -87,6 +87,7 @@ export abstract class BaseLoader< createDebugMessages('embedjs:loader:BaseLoader')(`New loader class initalized with key ${uniqueId}`); } + // eslint-disable-next-line @typescript-eslint/no-empty-function public async init(): Promise {} public get canIncrementallyLoad() { diff --git a/core/embedjs-interfaces/src/interfaces/base-model.ts b/core/embedjs-interfaces/src/interfaces/base-model.ts index 0d6ce799..c4b2f517 100644 --- a/core/embedjs-interfaces/src/interfaces/base-model.ts +++ b/core/embedjs-interfaces/src/interfaces/base-model.ts @@ -1,21 +1,21 @@ import createDebugMessages from 'debug'; import { v4 as uuidv4 } from 'uuid'; -import { BaseConversation } from './base-conversations.js'; import { Chunk, QueryResponse, Message, SourceDetail, ModelResponse } from '../types.js'; +import { BaseCache } from './base-cache.js'; export abstract class BaseModel { private readonly baseDebug = createDebugMessages('embedjs:model:BaseModel'); - private static conversations: BaseConversation; + private static cache: BaseCache; private static defaultTemperature: number; public static setDefaultTemperature(temperature?: number) { BaseModel.defaultTemperature = temperature; } - public static setConversations(conversations: BaseConversation) { - BaseModel.conversations = conversations; + public static setCache(cache: BaseCache) { + BaseModel.cache = cache; } private readonly _temperature?: number; @@ -37,11 +37,11 @@ export abstract class BaseModel { supportingContext: Chunk[], conversationId = 'default', ): Promise { - const conversation = await BaseModel.conversations.getConversation(conversationId); + const conversation = await BaseModel.cache.getConversation(conversationId); this.baseDebug(`${conversation.entries.length} history entries found for conversationId '${conversationId}'`); // Add user query to history - await BaseModel.conversations.addEntryToConversation(conversationId, { + await BaseModel.cache.addEntryToConversation(conversationId, { id: uuidv4(), timestamp: new Date(), actor: 'HUMAN', @@ -61,7 +61,7 @@ export abstract class BaseModel { }; // Add AI response to history - await BaseModel.conversations.addEntryToConversation(conversationId, newEntry); + await BaseModel.cache.addEntryToConversation(conversationId, newEntry); return { ...newEntry, tokenUse: { diff --git a/core/embedjs/README.md b/core/embedjs/README.md index 0bf37efe..89d58035 100644 --- a/core/embedjs/README.md +++ b/core/embedjs/README.md @@ -1,1308 +1,30 @@ -# EmbedJs - -

-License -NPM Version +

+ EmbedJs Logo

-EmbedJs is an Open Source Framework for personalizing LLM responses. An ultimate toolkit for building powerful Retrieval-Augmented Generation (RAG) and Large Language Model (LLM) applications with ease in Node.js. - -It segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. It enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data. - -Here's an example of how easy it is to get started - - -```TS -const ragApplication = await new RAGApplicationBuilder() - .addLoader({ type: 'YoutubeSearch', youtubeSearchString: 'Tesla cars' }) - .addLoader('https://en.wikipedia.org/wiki/Tesla,_Inc.') - .addLoader('https://tesla-info.com/sitemap.xml') - .setVectorDb(new LanceDb({ path: '.db' })) - .build(); -``` - -That's it. Now you can ask questions - - -```TS -console.log(await ragApplication.query('Give me the history of Tesla?')); -``` - -## Features - -- Supports all popular large language models - paid and open source - -- Supports many vector databases including self-hosted and cloud variants. - -- Load different kinds of unstructured data. Comes built in with several loaders that makes this easy. - -- Supports several cache options that can greatly improve the performance of your RAG applications in production. - -- Exposes a simple and highly configureable API allows both quick launch and deep customizabilty. - -- Use just as an embedding engine or a full blown chat API with history - -## Quick note - -The author(s) are looking to add core maintainers for this opensource project. Reach out on [Linkedin](https://www.linkedin.com/in/adhityan/) if you are interested. If you want to contribute in general - create issues on GitHub or send in PRs. - -# Contents - -- [EmbedJs](#embedjs) - - [Features](#features) - - [Quick note](#quick-note) -- [Contents](#contents) -- [Getting started](#getting-started) - - [Installation](#installation) - - [Usage](#usage) - - [Temperature](#temperature) - - [Search results count](#search-results-count) - - [Customize the prompt](#customize-the-prompt) - - [Get context (dry run)](#get-context-dry-run) - - [Delete loader](#delete-loader) - - [Get count of embedded chunks](#get-count-of-embedded-chunks) - - [Remove all embeddings / reset](#remove-all-embeddings--reset) - - [Set cut-off for relevance](#set-cut-off-for-relevance) - - [Add new loaders later](#add-new-loaders-later) - - [Loader inference](#loader-inference) -- [Loaders supported](#loaders-supported) - - [Youtube video](#youtube-video) - - [Youtube channel](#youtube-channel) - - [Youtube search](#youtube-search) - - [PDF file](#pdf-file) - - [Docx file](#docx-file) - - [Excel file](#excel-file) - - [Powerpoint file](#powerpoint-file) - - [Web page](#web-page) - - [Confluence](#confluence) - - [Sitemap](#sitemap) - - [Text](#text) - - [Json](#json) - - [Csv](#csv) - - [Add a custom loader](#add-a-custom-loader) - - [More loaders coming soon](#more-loaders-coming-soon) -- [LLMs](#llms) - - [OpenAI](#openai) - - [Azure OpenAI](#azure-openai) - - [Mistral](#mistral) - - [Hugging Face](#hugging-face) - - [Anthropic](#anthropic) - - [Vertex AI](#vertex-ai) - - [Ollama](#ollama) - - [Use custom LLM model](#use-custom-llm-model) - - [More LLMs coming soon](#more-llms-coming-soon) -- [Embedding models](#embedding-models) - - [OpenAI v3 Small](#openai-v3-small) - - [OpenAI v3 Large](#openai-v3-large) - - [Ada](#ada) - - [Cohere](#cohere) - - [Gecko Embedding](#gecko-embedding) - - [Ollama local embedding](#ollama-local-embedding) - - [Use custom embedding model](#use-custom-embedding-model) - - [More embedding models coming soon](#more-embedding-models-coming-soon) -- [Vector databases supported](#vector-databases-supported) - - [Pinecone](#pinecone) - - [LanceDB](#lancedb) - - [Chroma](#chroma) - - [HNSWLib](#hnswlib) - - [Weaviate](#weaviate) - - [Qdrant](#qdrant) - - [MongoDB (vector database)](#mongodb-vector-database) - - [Astra DB](#astra-db) - - [Bring your own database](#bring-your-own-database) - - [More databases coming soon](#more-databases-coming-soon) -- [Caches](#caches) - - [LMDB](#lmdb) - - [InMemory (cache)](#inmemory-cache) - - [Redis](#redis) - - [MongoDB (cache)](#mongodb-cache) - - [Bring your own cache](#bring-your-own-cache) - - [More caches coming soon](#more-caches-coming-soon) -- [Conversation history](#conversation-history) - - [InMemory (conversation)](#inmemory-conversation) - - [MongoDB (conversation)](#mongodb-conversation) -- [Langsmith Integration](#langsmith-integration) -- [Sample projects](#sample-projects) -- [Contributing](#contributing) - -# Getting started - -## Installation - -You can install the library via NPM or Yarn - -```bash -npm i @llm-tools/embedjs -``` - -## Usage - -To configure a new EmbedJs application, you need to do three steps - - -> **1. Pick an LLM**
-> The library supports several LLMs. Activate one by allowing the instructions in the [LLM](#llms) section. - -```TS -const ragApplication = await new RAGApplicationBuilder() - .setModel(new HuggingFace({ modelName: 'mistralai/Mixtral-8x7B-v0.1' })) - ... -``` - -**Note:** To use the library only for embeddings and not instantiate a LLM, you can pass the string `NO_MODEL` to the setModel function here. This will disable the option to call the `query` function but you can still get the embeddings with the [`getContext`](#get-context-dry-run) method. - -> **2. Pick a Vector database**
-> The library supports several vector databases. Enable one by allowing the instructions in the [Vector Databases](#vector-databases-supported) section. - -```TS - .setVectorDb(new PineconeDb({ projectName: 'test', namespace: 'dev' })) -``` - -> **3. Load some data**
-> The library supports several kinds of loaders. You can use zero, one or many kinds of loaders together to import custom knowledge. Read the [loaders](#loaders-supported) section to learn more about the different supported loaders. - -```TS - .addLoader(new YoutubeSearchLoader({ searchString: 'Tesla cars' })) - .addLoader(new SitemapLoader({ url: 'https://tesla-info.com/sitemap.xml' })) - .build(); -``` - -That's it! Now that you have your instance of `RAGApplication`, you can use it to query against the loaded data sets, like so - - -```TS -await ragApplication.query('What is Tesla?'); -``` - -## Temperature - -The temperature is a number between 0 and 1. It governs the randomness and creativity of the LLM responses. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. You can alter it by - - -```TS -await new RAGApplicationBuilder() -.setTemperature(0.1) -``` - -**NOTE:** The default value is 0.1, which makes the GPT responses very precise. - -## Search results count - -This is the number of documents to aim for when retrieving results from the vector database. A high number of results might mean there is more non-relevant data in the context. A low number might mean none of the relevant documents are retrieved. You need to set the number that works best for you. The parameter can be altered by - - -```TS -await new RAGApplicationBuilder() -.setSearchResultCount(10) -``` - -**NOTE:** The default value is 7. - -It is important to note that the library does not simply dump all contextual document chunks into the prompt. It sends them to the model marking them as context documents. The number of documents still counts toward the token limit. - -When the number of documents fetched leads to a request above the token limit, the library uses the following strategy - - -> It runs a preprocessing step to select relevant sections from each document until the total number of tokens is less than the maximum number of tokens allowed by the model. It then uses the transformed documents as context to answer the question. - -## Customize the prompt - -LLM models need some care. The models are notorious for inventing responses when they don't know the answer. Keeping this in mind, the library auto adds a wrapper to all user queries. The default prompt is - - -> Use all the provided context to answer the query at the end. Answer in full. If you don't know the answer, just say that you don't know, don't try to make up an answer. Query: {0} - -The placeholder `{0}` is replaced with the input query. In some cases, you may want to customize this prompt. This can be done with ease by - - -```TS -await new RAGApplicationBuilder() -.setQueryTemplate('My own query template') -``` - -## Get context (dry run) - -During development, you may want to test the performance and quality of the `Loaders` you have enabled without making any LLM calls. You can do this by using the `getContext` method - - -```TS -await ragApplication.getContext('What is Steve Jobs?') -``` - -## Delete loader - -You can remove the embeddings added from a specific loader by calling the `deleteLoader` method with the uniqueId of the loader. - -```TS -await ragApplication.deleteLoader('uniqueId...', true) -``` - -## Get count of embedded chunks - -You can fetch the count of embeddedings stored in your vector database at any time by calling the `getEmbeddingsCount` method - - -```TS -await ragApplication.getEmbeddingsCount() -``` - -## Remove all embeddings / reset - -You can remove all stored embeddings in the vectorDb using the `deleteAllEmbeddings` method - - -```TS -await ragApplication.deleteAllEmbeddings(true) -``` - -## Set cut-off for relevance - -The library can filter the embeddings returned from a vector store that have a low relevance score to the query being asked. To do this, set the cut-off value using the `setEmbeddingRelevanceCutOff` method - - -```TS -await ragApplication.setEmbeddingRelevanceCutOff(0.23) -``` - -## Add new loaders later - -You can add new loaders at any point dynamically (even after calling the `build` function on `RAGApplicationBuilder`). To do this, simply call the `addLoader` method - - -```TS -await ragApplication.addLoader(new YoutubeLoader({ videoIdOrUrl: 'pQiT2U5E9tI' })); -``` - -**Note:** Do not forget to await the dynamically added loaders to ensure you wait for the load to complete before making queries on it. - -## Loader inference - -You can add most loaders by passing a string to the `addLoader` or the `addLoaders` methods. The value can be a URL, path, JSON or youtube video id. The library will infer the type of content and invoke the appropirate loader automatically. - -```TS -await ragApplication.addLoader('pQiT2U5E9tI'); //invokes youtube URL -await ragApplication.addLoader('https://lamport.azurewebsites.net/pubs/paxos-simple.pdf'); //invokes PDF loader -``` - -**Note:** If you pass the path to a local directory, every file in that directory is recursively added (including subfolders)! - -# Loaders supported - -Loaders take a specific format, process the input and create chunks of the data. You can import all the loaders from the path `@llm-tools/embedjs`. Currently, the library supports the following formats - - -## Youtube video - -To add any youtube video to your app, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeLoader` - -```TS -.addLoader(new YoutubeLoader({ videoIdOrUrl: 'w2KbwC-s7pY' })) -``` - -## Youtube channel - -To add all videos in a youtube channel, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeChannelLoader` - -```TS -.addLoader(new YoutubeChannelLoader({ youtubeChannelId: '...' })) -``` - -## Youtube search - -To do a general youtube search and add the popular search results, follow these steps - - -- Install Youtube loader package in your project - -```bash -npm install @llm-tools/embedjs-loader-youtube -``` - -- Now you can use the `YoutubeSearchLoader` - -```TS -.addLoader(new YoutubeSearchLoader({ youtubeSearchString: '...' })) -``` - -## PDF file - -To add a pdf file, use `PdfLoader`. To get started, install the PdfLoader addon - - -```bash -npm install @llm-tools/embedjs-loader-pdf -``` - -Now, you can add a local file - - -```TS -.addLoader(new PdfLoader({ filePathOrUrl: path.resolve('paxos-simple.pdf') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new PdfLoader({ url: 'https://lamport.azurewebsites.net/pubs/paxos-simple.pdf' })) -``` - -**Note:** Currently there is no support for PDF forms and password protected documents - -## Docx file - -To add a docx file, use `DocxLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new DocxLoader({ filePathOrUrl: path.resolve('paxos.docx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new DocxLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Excel file - -To add an excel xlsx file, use `ExcelLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new ExcelLoader({ filePathOrUrl: path.resolve('numbers.xlsx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new ExcelLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Powerpoint file - -To add an powerpoint / pptx file, use `PptLoader`. To get started, install the MSOffice loader addon - - -```bash -npm install @llm-tools/embedjs-loader-msoffice -``` - -Now, you can add a local file - - -```TS -.addLoader(new PptLoader({ filePathOrUrl: path.resolve('wow.pptx') })) -``` - -Or, you can add a remote file - - -```TS -.addLoader(new PptLoader({ filePathOrUrl: 'https://xxx' })) -``` - -## Web page - -To add a web page, follow these steps - - -- Install Web loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-web -``` - -- Now you can use the `WebLoader` - -```TS -.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Formula_One' })) -``` - -## Confluence - -To add a confluence space, follow these steps - - -- Install the Confluence loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-confluence -``` - -- set the following environment variables - - -```bash -CONFLUENCE_BASE_URL= -CONFLUENCE_USER_NAME= -CONFLUENCE_API_TOKEN= -``` - -**Note:** The confluence space name is the value you see in the url in the space overview page `/wiki/spaces/{{ space name }}/overview`. - -- Now you can use the `ConfluenceLoader` - -```TS -.addLoader(new ConfluenceLoader({ spaceNames: ['...'] })) -``` - -## Sitemap - -To add a XML sitemap, follow these steps - - -- Install the Sitemap loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-sitemap -``` - -- Now you can use the `SitemapLoader` - -```TS -.addLoader(new SitemapLoader({ url: '...' })) -``` - -This will load all URLs in a sitemap via the WebLoader. - -## Text - -To supply your own text, use `TextLoader`. - -```TS -.addLoader(new TextLoader({ text: 'The best company name for a company making colorful socks is MrSocks' })) -``` - -**Note:** Feel free to add your custom text without worrying about duplication. The library will chuck, cache and update the vector databases without duplication. - -## Json - -To add a parsed Javascript object to your embeddings, use `JsonLoader`. The library will not parse a string to JSON on its own but once this is done, it can be injested easily. - -```TS -.addLoader(new JsonLoader({ object: { key: value, ... } })) -``` - -**Note:** if you want to restrict the keys that get added to the vectorDb in a dynamically obtained object, you can use the `pickKeysForEmbedding` optional parameter in the `JsonLoader` constructor. - -## Csv - -To add a Csv file (or URL) to your embeddings, use `CsvLoader`. The library will parse the Csv and add each row to its vector database. To get started, follow these steps - - -- Install the Csv loader addon in your project - -```bash -npm install @llm-tools/embedjs-loader-csv -``` - -- Now you can use the `CsvLoader` like so - - -```TS -.addLoader(new CsvLoader({ filePathOrUrl: '...' })) -``` - -**Note:** You can control how the `CsvLoader` parses the file in great detail by passing in the optional `csvParseOptions` constructor parameter. - -## Add a custom loader - -You can pass along a custom loader to the `addLoader` method by extending and implementing the abstract class `BaseLoader`. Here's how that would look like - - -```TS -class CustomLoader extends BaseLoader<{ customChunkMetadata: string }> { - constructor() { - super('uniqueId'); - } - - async *getChunks() { - throw new Error('Method not implemented.'); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a common loader pattern, so the community can benefit from it. - -## More loaders coming soon - -If you want to add any other format, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported formats. All PRs are welcome. - -# LLMs - -It's relatively easy to switch between different LLMs using the library. You can import any of the LLMs from the path `@llm-tools/embedjs`. We support the following LLMs today - - -## OpenAI - -To use the OpenAI LLM models, you need a API key from OpenAI. You can alternatively use Azure OpenAI to run these models. Read the [Azure OpenAI](#azure-openai) section below to learn more about this. In this section, we will cover how to use OpenAI provided LLMs. - -The first step is to obtain an API Key from OpenAI. You can do this by visiting their [API Portal](https://platform.openai.com/api-keys). Once you obtain a key, set it as an environment variable, like so - - -```bash -OPENAI_API_KEY="" -``` - -Next, you need to install the OpenAI addon - - -```bash -npm install @llm-tools/embedjs-openai -``` - -Once these two steps are done, it is relatively easy to run OpenAI LLMs. All you need is to indicate the model type you want to run. - -- For GPT 3.5 Turbo - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(SIMPLE_MODELS.OPENAI_GPT3_TURBO) -``` - -- For GPT 4 - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(SIMPLE_MODELS.OPENAI_GPT4) -``` - -- To use a custom model name - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new OpenAi({ modelName: 'gpt-4' })) -``` - -**Note:** GPT 3.5 Turbo is used as the default model if you do not specifiy one. - -## Azure OpenAI - -In order to be able to use an OpenAI model on Azure, it first needs to be deployed. Please refer to [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/) on how to deploy a model on Azure. To run this library, you will need to deploy two models - - -- text-embedding-ada (or any of the newer models) -- GPT-3.5-turbo (or the 4 series) - -Once these models are deployed, using Azure OpenAI instead of the regular OpenAI is easy to do. Just follow these steps - - -- Remove the `OPENAI_API_KEY` environment variable if you have set it already. - -- Set the following environment variables - - -```bash -# Set this to `azure` -OPENAI_API_TYPE=azure -# The API version you want to use -AZURE_OPENAI_API_VERSION=2023-05-15 -# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. -export AZURE_OPENAI_BASE_PATH=https://your-resource-name.openai.azure.com/openai/deployments -# The API key1 or key2 for your Azure OpenAI resource -export AZURE_OPENAI_API_KEY= -# The deployment name you used for your embedding model -AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=text-embedding-ada-002 -# The deployment name you used for your llm -AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-35-turbo -``` - -You are now all set and can now run the Azure OpenAI LLMs using the [`OpenAi` model](#openai) steps detailed above. - -## Mistral - -To use Mirstal's models, you will need to get an API Key from Mistral. You can do this from their [console](https://console.mistral.ai/user/api-keys/). Next, you need to install the Mistral addon to embedJs - - -```bash -npm install @llm-tools/embedjs-mistral -``` - -Once these steps are complete, set Mistral as your LLM of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Mistral({ accessToken: "" })) -``` - -By default, the `mistral-medium` model from Mistral is used. If you want to use a different Mistral model, you can specify it via the optional parameter to the Mistral constructor, like so - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Mistral({ accessToken: "", modelName: "..." })) -``` - -**Note:** If you want to run Mistral open source for free, you can do so using the HuggingFace platform (read below). Just make sure to set the modelName to `mistralai/Mistral-7B-v0.1` or the version you want to run. - -## Hugging Face - -Hugging face needs no introduction. They host a variety of open source LLMs and you can use most of them for free. To run hugging face inference based LLMs with this library, you will need a free hugging face token. - -You can get an API Token by signing up for hugging face and generate a token from [this page](https://huggingface.co/settings/tokens). Once you get the token, set it to the environment like so - - -```bash -HUGGINGFACEHUB_API_KEY="" -``` - -Next you need to install the hugging face addon for embedJs - - -```bash -npm install @llm-tools/embedjs-huggingface -``` - -That's all, now you can use any of the hugging face models. To do this set `HuggingFace` as your model processor of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new HuggingFace({ modelName: "..." }))) -``` - -**Note:** Not all hugging face models are fully free to consume via their API. Since running these models takes a lot of resources, Hugging Face charges a fee for a few of the larger ones. This is the case with Meta's `meta-llama/Llama-2-7b-hf`, for example. - -To use these 'not-free' models via HuggingFace, you need to subscribe to their [Pro plan](https://huggingface.co/pricing) or create a custom [inference endpoint](https://ui.endpoints.huggingface.co/). It is possible to self host these models for free and run them locally via Ollama - support for which is coming soon. - -## Anthropic - -To use Anthropic's Claude models, you will need to get an API Key from Anthropic. You can do this from their [console](https://console.anthropic.com/settings/keys). Once you obtain a key, set it in the environment variable, like so - - -```bash -ANTHROPIC_API_KEY="" -``` - -Next, you need to install the Anthropic extension for embedJs like so - - -```bash -npm install @llm-tools/embedjs-anthropic -``` - -Once these are done, it is relatively easy to use Anthropic's Claude in your RAG application. Simply set Anthropic as your LLM of choice - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Anthropic()) -``` - -By default, the `claude-3-sonnet-20240229` model from Anthropic is used. If you want to use a different Anthropic model, you can specify it via the optional parameter to the Anthropic constructor, like so - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Anthropic({ modelName: "..." })) -``` - -You can read more about the various models provided by Anthropic [here](https://docs.anthropic.com/claude/docs/models-overview). - -## Vertex AI - -You to use Gemini LLM and other models on Google Cloud Platform via [VertexAI](https://cloud.google.com/vertex-ai?hl=en). Read more about all the supported [LLMs](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models). - -To get started, you need to set the right access credentials to google cloud. You have two options here - - -- Authenticate by using `gcloud` CLI: - -``` -gcloud auth application-default login -``` - -- Authentication using Service Account with JSON key and environment variable: - -```bash -GOOGLE_APPLICATION_CREDENTIALS="" -``` - -- Install the vertex ai addon for embedJs - -```bash -npm install @llm-tools/embedjs-vertexai -``` - -Now all you need to do is set the model to `VertexAI`. Here's an example - - -```TS -const ragApplication = await new RAGApplicationBuilder() - .setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409'})) - .setEmbeddingModel(new GeckoEmbedding()) -``` - -See also `/examples/vertexai` for [further documentation](/examples/vertexai/README.md) about authentication options and how to use it. - -**Note:** Default model is `gemini-1.0-pro`. - -## Ollama - -You can also use locally running Ollama models. Installation instructions for Ollama can be found [here](https://ollama.com/). - -Once Ollama is installed, you can start a local LLM by executing `ollama run `. Next to be able to use Ollama in embedJs, install the Ollama addon - - -```bash -npm install @llm-tools/embedjs-ollama -``` - -Once this is done, you can use that in the `Ollama` constructor by passing the `modelName` parameter. Here's an example - - -```TS -const ragApplication = await new RAGApplicationBuilder() -.setModel(new Ollama({ - modelName: "llama3", - baseUrl: 'http://localhost:11434' -})) -``` - -**Note:** Default port in which Ollama runs, is `11434`, but if for some reason you use something else, you can pass `baseUrl` with the port number as the second argument: - -## Use custom LLM model - -You can use a custom LLM model by implementing the `BaseModel` interface. Here's how that would look like - - -```TS -class MyOwnLLMImplementation implements BaseModel { - override async init(): Promise {} //optional to override - - protected abstract runQuery( - system: string, //the system prompt - userQuery: string, //the current user query - supportingContext: Chunk[], //all supporting documents - pastConversations: ConversationHistory[], //the chat history so far - ): Promise { - throw new Error("Method not implemented."); - } -} -``` - -Once done, you can pass this class to the `setModel` method like shown in the examples above. That said, we really encourage you send in a PR to this library if you are implementing a famous or common LLM, so the community can benefit from it. - -## More LLMs coming soon - -If you want us to add support for a specific LLM, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will prioritize it. All PRs are welcome. - -Currently, we next plan to add support for Ollama. - -# Embedding models - -Embedding models are LLMs that convert a string into vector better suited for processing. In most cases, the default `text-embedding-3-small` model from OpenAI is going to be good enough. If you want to use this model, you do not have to do anything extra. - -However in some advanced cases, you may want to change this; after all, different embedding models perform differently under different curcumstances. The library allows you to do this using the method `setEmbeddingModel` while building the `RAGApplication`. - -The library supports the following embedding models - - -## OpenAI v3 Small - -The `text-embedding-3-small` is a new standard embedding model released by OpenAI in Jan, 2024. It is the default used by the libary. This model is cheaper and better than their older Ada model. This model returns vectors with dimension 1536. - -You do not have to do anything to enable it. - -## OpenAI v3 Large - -The `text-embedding-3-large` is also a new standard embedding model released by OpenAI in Jan, 2024. This model is the best embedding model provided by OpenAI as of now but is also the most expensive. This model returns vectors with dimension 3072. - -To set it as your model of choice - - -- Set `OpenAi3LargeEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { OpenAi3LargeEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new OpenAi3LargeEmbeddings()) -``` - -## Ada - -The `text-embedding-ada-002` is a well known model from OpenAI. You can read more about it [here](https://openai.com/blog/new-and-improved-embedding-model). This model returns vectors with dimension 1536. - -To set it as your model of choice - - -- Set `AdaEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { AdaEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new AdaEmbeddings()) -``` - -## Cohere - -The library supports usage of [Cohere-AI](https://cohere.com) `embed-english-v2.0` embedding model out of the box. This model returns vectors with dimension 4096. - -Here's what you have to do to enable it - - -- Sign up for an account with Cohere-AI if you have not done so already. Once done, go to the [API Keys](https://dashboard.cohere.ai/api-keys) section and copy an API_KEY. - -- Load the key you just obtained in the environment variable `COHERE_API_KEY` - -```bash -COHERE_API_KEY="" -``` - -- Set `CohereEmbeddings` as your embedding model on `RAGApplicationBuilder` - -```TS -import { CohereEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new CohereEmbeddings()) -``` - -## Gecko Embedding - -The libaray supports the embedding model `textembedding-gecko` with 768 dimensions on [VertexAI](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings). - -To use this, you can authenticate to Vertex AI on GCP. Refer [here](#vertex-ai) on how to do this. Once done, simply set `GeckoEmbedding` as your choice of embedding model, like so - - -```TS -import { GeckoEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new GeckoEmbeddings()) -``` - -For an example usage of GeckoEmbeddings with Gemini LLM on VertexAI check the folder `/examples/vertexai/`. - -## Ollama local embedding - -The libaray supports fully local embeddings via `Ollama`. Read more here [Ollama embeddings](https://ollama.com/blog/embedding-models). - -To use this, you need to setup and have Ollama working locally. Refer to their Github [here](https://github.com/ollama/ollama) to understand how to do this. Once done, simply set `OllamaEmbeddings` as your choice of embedding model, like so - - -```TS -import { OllamaEmbeddings } from '@llm-tools/embedjs'; - -await new RAGApplicationBuilder() -.setEmbeddingModel(new OllamaEmbeddings({ - model: '...', - baseUrl: '...' -})) -``` - -## Use custom embedding model - -You can use your own custom embedding model by implementing the `BaseEmbeddings` interface. Here's how that would look like - - -```TS -class MyOwnEmbeddingImplementation implements BaseEmbeddings { - embedDocuments(texts: string[]): Promise { - throw new Error("Method not implemented."); - } - - embedQuery(text: string): Promise { - throw new Error("Method not implemented."); - } - - getDimensions(): number { - throw new Error("Method not implemented."); - } -} -``` - -Once done, you can pass this class to the `setEmbeddingModel` method like shown in the Cohere example above. That said, we really encourage you send in a PR to this library if you are implementing a famous or common embedding provider, so the community can benefit from it. - -## More embedding models coming soon - -If you want us to add support for a specific embedding model, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will prioritize it. All PRs are welcome. - -# Vector databases supported - -The library allows you to save your processed and unique embeddings with the vector databases of your choice. Here are the supported databases right now - - -## Pinecone - -You can enable Pinecone storage by following these steps - - -- Create an account with [Pinecone](https://www.pinecone.io/) if you don't have one already. There is a _good free tier_. - -- Install pinecone package in your project - -```bash -npm install @pinecone-database/pinecone -``` - -- Set the pinecone environment variable `PINECONE_API_KEY`. This can be obtained from the **API Keys** section on the Pinecone dashboard. - -```bash -PINECONE_API_KEY= -``` - -- Set the Pinecone database as your choice of `vectorDb` - -```TS -import { PineconeDb } from '@llm-tools/embedjs/vectorDb/pinecone'; - -.setVectorDb(new PineconeDb({ - projectName: 'test', - namespace: 'dev', - indexSpec: { - pod: { - podType: 'p1.x1', - environment: 'us-east1-gcp', - }, - }, -})) -``` - -**Note:** Pinecone supports serverless and pod based index deployments. You can control how you want your index created using the indexSpec attribute. This is mandatory to be provided but comes with full type specification. Read more about configuring this [here](https://github.com/pinecone-io/pinecone-ts-client/blob/main/v2-migration.md). - -## LanceDB - -[LanceDB](https://lancedb.com/) is a local vector database with great performance. Follow these steps to use LanceDB as your vector database - - -- Install LanceDb package in your project - -```bash -npm install @lancedb/lancedb -``` - -- Set LanceDB database as your choice of `vectorDb` - -```TS -import { LanceDb } from '@llm-tools/embedjs/vectorDb/lance'; - -.setVectorDb(new LanceDb({ path: path.resolve('/db') })) -``` - -**Note:** The `path` property will be used by LanceDB to create a directory to host all the database files. There is also support for creating temporary directories for testing - - -```TS -import { LanceDb } from '@llm-tools/embedjs/vectorDb/lance'; - -.setVectorDb(new LanceDb({ path: 'lance-', isTemp: true })) -``` - -In this case, the `path` property is used as a prefix to create the temporary directory in the OS temp directory folder. - -## Chroma - -**Note:** The chroma addon is currently not published / supported. There are some type issues in the upstream dependencies that are being resolved. - -[Chroma](https://trychroma.com/) is an open source vector database. It's designed to be Python first and to connect to it from NodeJS, you will need to run Chroma in a container following the steps [listed here](https://docs.trychroma.com/deployment). Once done, follow these steps to use Chroma as your vector database - - -- Install Chroma package in your project - -```bash -npm install chromadb -``` - -- Set Chroma database as your choice of `vectorDb` - -```TS -import { ChromaDb } from '@llm-tools/embedjs/vectorDb/chroma'; - -.setVectorDb(new ChromaDb({ url: 'http://localhost:8000' })) -``` - -**Note:** This is the default url and port if you run the Chroma docker container using the command `docker-compose up -d --build`. - -A warning - - -> In our testing, chroma performed the poorest in being able to retrieve relevant documents among the supported vector databases. - -## HNSWLib - -[HNSWLib](https://github.com/nmslib/hnswlib) is an in-memory vectorstore. It is great for beginners to get started with since you do not need access to the file system or a cloud service. Follow these steps to use HNSWLib as your vector database - - -- Install HNSWLib package in your project - -```bash -npm install hnswlib-node -``` - -- Set HNSWLib database as your choice of `vectorDb` - -```TS -import { HNSWDb } from '@llm-tools/embedjs/vectorDb/hnswlib'; - -.setVectorDb(new HNSWDb()) -``` - -**Note:** This is a purely in-memory vector store. All values are lost when application is restarted. - -## Weaviate - -[Weaviate](https://weaviate.io/) is an open source vector store. You can deploy it locally on docker or use their managed cloud offering. Follow these steps to use Weaviate as your vector database - - -- Install Weaviate package in your project - -```bash -npm install weaviate-ts-client -``` - -- Set Weaviate database as your choice of `vectorDb` - -```TS -import { WeaviateDb } from '@llm-tools/embedjs/vectorDb/weaviate'; - -.setVectorDb(new WeaviateDb({ host: '...', apiKey: '...', className: '...', scheme: '...' })) -``` - -## Qdrant - -[Qdrant](https://qdrant.tech/) is an Open-Source Vector Database and Vector Search Engine written in Rust. To use it - - -- Install Qdrant package in your project - -```bash -npm install @qdrant/js-client-rest -``` - -- Set Qdrant database as your choice of `vectorDb` - -```TS -import { QdrantDb } from '@llm-tools/embedjs/vectorDb/qdrant'; - -.setVectorDb(new QdrantDb({ apiKey: '...'; url: '...'; clusterName: '...' })) -``` - -## MongoDB (vector database) - -[MongoDB](https://www.mongodb.com/products/platform/atlas-vector-search) is an open source document database. They offer a managed cloud offering **MongoDB Atlas**. As of right now, only the Atlas version supports vector search while the open source version does not. - -To use MongoDB as your vector database, follow these steps - - -- Sign up for a MongoDB Atlas account if you haven't already. Once you have signed up, you will need to spin up a new cluster (or use an existing one) - -**Note:** you will need to provision a M10 (or higher) instance type to use Atlas vector search. Cheaper instance types or the free version (M0) give an error when vector indexes are created programatically. - -- The cluster creation takes a few minutes. Once the cluster is ready, click on the connect button on the dashboard to get the connection string. - -**Note:** You will need to add a users separately and allow IP access from your relevant development and production environments. - -- Install mongodb package in your project - -```bash -npm install mongodb -``` - -- Set MongoDB database as your choice of `vectorDb` - -```TS -import { MongoDb } from '@llm-tools/embedjs/vectorDb/mongodb'; - -.setVectorDb( - new MongoDb({ - connectionString: 'mongodb+srv://:@', - }), -) -``` - -**Note:** you can also optionally configure the database and collection name the library will use with the constructor parameters `dbName` and `collectionName`. Default values are used if these are not provided. - -## Astra DB - -[Astra DB is a document database with a highly performant vector index](https://www.datastax.com/products/datastax-astra) powered by Apache Cassandra and available as a managed service. - -To use Astra DB as your vector database follow these steps: - -- [Sign up for an Astra DB account](https://astra.datastax.com/signup). It is free to sign up and doesn't require a credit card. -- Create a database (this takes a couple of minutes to provision) -- From the database overview page get the API Endpoint and generate an Application Token -- Install the Astra DB package in your project: - ```bash - npm install @datastax/astra-db-ts - ``` -- Set Astra DB as your choice of `vectorDb` - - ```TS - import { AstraDb } from '@llm-tools/embedjs/vectorDb/astra-db' - - .setVectorDb( - new AstraDb({ - endpoint: process.env.ASTRA_DB_API_ENDPOINT, - apiKey: process.env.ASTRA_DB_APP_TOKEN, - collectionName: "documents" - }), - ) - ``` - -## Bring your own database - -You can pass along your vector database to the `setVectorDb` method by implementing the interface `BaseDb`. Here's how that would look like - - -```TS -class MyOwnDb implements BaseDb { - async init({ dimensions }: { dimensions: number }): Promise { - throw new Error('Method not implemented.'); - } - - async insertChunks(chunks: EmbeddedChunk[]): Promise { - throw new Error('Method not implemented.'); - } - - async similaritySearch(query: number[], k: number): Promise { - throw new Error('Method not implemented.'); - } - - async getVectorCount(): Promise { - throw new Error('Method not implemented.'); - } - - async deleteKeys(keys: string[]): Promise { - throw new Error('Method not implemented.'); - } - - async reset(): Promise { - throw new Error('Method not implemented.'); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a famous or common database, so the community can benefit from it. - -## More databases coming soon - -If you want to add support for any other vector database, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported databases. All PRs are welcome. - -# Caches - -Caches serve to reduce re-processing embeddings, loaders and queries. There is no need to load, chunk and store a large PDF File or web page on every run. Caching smartly is built in and can be enabled out of the box simply by setting a cache processor using the method `setCache` while building the `RAGApplication`. - -The library supports the following caches - - -## LMDB - -You can use [LMDB](https://dbdb.io/db/lmdb) to cache values locally on disk. - -- Install LMDB package in your project - -```bash -npm install lmdb -``` - -- Set `LmdbCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { LmdbCache } from '@llm-tools/embedjs/cache/lmdb'; - -await new RAGApplicationBuilder() -.setCache(new LmdbCache({ path: path.resolve('./cache') })) -``` - -**Note:** The `path` property will be used by the LMDB driver to create a folder housing the LMDB database files. - -## InMemory (cache) - -You can use a simple in-memory cache to store values during testing. - -- Set `MemoryCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryCache } from '@llm-tools/embedjs/cache/memory'; - -await new RAGApplicationBuilder() -.setCache(new MemoryCache()) -``` - -**Note:** Although this cache can remove duplicate loaders and chunks, its store does not persist between process restarts. You should only be using it for testing. - -## Redis - -You can use redis as a cache to store values. - -- Set `RedisCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { RedisCache } from '@llm-tools/embedjs/cache/redis'; - -await new RAGApplicationBuilder() -.setCache(new RedisCache({ ... })) -``` - -**Note:** The library internally uses `IORedis` to work with redis. `RedisCache` constructor supports all `IORedis` constructor parameters. Check [`IORedis` documentation](https://github.com/redis/ioredis) for more detials. - -## MongoDB (cache) - -You can use a MongoDB as a cache to cache values. - -- Set `MongoCache` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryCache } from '@llm-tools/embedjs/cache/mongo'; - -await new RAGApplicationBuilder() -.setCache(new MongoCache({ ... })) -``` - -## Bring your own cache - -You can pass along your own cache provider to the `setCache` method by implementing the interface `BaseCache`. Here's how that would look like - - -```TS -class MyOwnCache implements BaseCache { - async init(): Promise { - throw new Error("Method not implemented."); - } - - async addLoader(loaderId: string, chunkCount: number ): Promise { - throw new Error("Method not implemented."); - } - - async getLoader(loaderId: string): Promise<{ chunkCount: number }> { - throw new Error("Method not implemented."); - } - - async hasLoader(loaderId: string): Promise { - throw new Error("Method not implemented."); - } -} -``` - -We really encourage you send in a PR to this library if you are implementing a famous or common cache provider, so the community can benefit from it. - -## More caches coming soon - -If you want to add support for any other cache providers, please create an [issue](https://github.com/llm-tools/embedjs/issues) and we will add it to the list of supported caches. All PRs are welcome. - -# Conversation history - -EmbedJS allows the addition of various storage layers for conversations. This allows the conversation history to be stored and made persistant between sessions. Like all other aspects of embedJS there is a base interface for conversations and you can create your own conversation history implementation. - -The library supports the following conversation history types out of the box - - -## InMemory (conversation) - -You can use a simple in-memory object to store conversation history during testing. This is the default activated conversation history manager if you don't specify anything else. This cache is used by **default** if no other cache is specified. - -- Set `InMemoryConversation` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MemoryConversations } from '@llm-tools/embedjs/conversation/memory'; - -await new RAGApplicationBuilder() -.setConversationEngine(new InMemoryConversation()) -``` - -**Note:** Although this cache does remove duplicate loaders and chunks, its store does not persist between process restarts. - -## MongoDB (conversation) - -Can be used with any version of MongoDb. - -- Set `MongoConversation` as your cache provider on `RAGApplicationBuilder` - -```TS -import { MongoConversation } from '@llm-tools/embedjs/conversation/mongo'; - -await new RAGApplicationBuilder() -.setConversationEngine(new MongoConversations({ - uri: MONGODB_URI, - dbName: DB_NAME, - collectionName: CONVERSATIONS_COLLECTION_NAME -});) -``` +

+ License + NPM Version +

-# Langsmith Integration +
-Langsmith allows you to keep track of how you use LLM and embedding models. It logs histories, token uses and other metadata. Follow these three simple steps to enable - +## What is EmbedJs? -- Sign up for an account with [Langsmith](https://smith.langchain.com/) -- Generate an API Key from your admin page -- Set the following environment keys in your project +EmbedJs is an Open Source Framework for personalizing LLM responses. An ultimate toolkit for building powerful Retrieval-Augmented Generation (RAG) and Large Language Model (LLM) applications with ease in Node.js. -```bash -export LANGCHAIN_TRACING_V2=true -export LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" -export LANGCHAIN_PROJECT="" -export LANGCHAIN_API_KEY="" -``` +It segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. It enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data. -# Sample projects +## πŸ“– Documentation -Here's a list of projects / examples built with RagKit +Comprehensive guides and API documentation are available to help you get the most out of EmbedJs: -| **Project** | **Description** | -| ------------------------------------------------------------ | ---------------------------------------------------------------------- | -| [slack-bot](https://github.com/llm-tools/slack-bot-template) | A NestJs based slack bot that can answer questions based on confluence | +- [Introduction](https://llm-tools.mintlify.app/get-started/introduction#what-is-embedjs) +- [Getting Started](https://llm-tools.mintlify.app/get-started/quickstart) +- [Supported data types](https://llm-tools.mintlify.app/components/data-sources/overview) +- [Examples](https://llm-tools.mintlify.app/examples) -# Contributing +## 🌐 Contributing Contributions are welcome! Please check out the issues on the repository, and feel free to open a pull request. For more information, please see the [contributing guidelines](CONTRIBUTING.md). diff --git a/core/embedjs/src/cache/memory-cache.ts b/core/embedjs/src/cache/memory-cache.ts index 49cc52a7..9bd31af8 100644 --- a/core/embedjs/src/cache/memory-cache.ts +++ b/core/embedjs/src/cache/memory-cache.ts @@ -1,12 +1,14 @@ -import { BaseCache } from '@llm-tools/embedjs-interfaces'; +import { BaseCache, Conversation, Message } from '@llm-tools/embedjs-interfaces'; export class MemoryCache implements BaseCache { private loaderList: Record; private loaderCustomValues: Record>; + private conversations: Map; async init(): Promise { this.loaderList = {}; this.loaderCustomValues = {}; + this.conversations = new Map(); } async addLoader(loaderId: string, chunkCount: number): Promise { @@ -40,4 +42,36 @@ export class MemoryCache implements BaseCache { async loaderCustomDelete(loaderCombinedId: string): Promise { delete this.loaderList[loaderCombinedId]; } + + async addConversation(conversationId: string): Promise { + if (!this.conversations.has(conversationId)) { + this.conversations.set(conversationId, { conversationId, entries: [] }); + } + } + + async getConversation(conversationId: string): Promise { + if (!this.conversations.has(conversationId)) { + // Automatically create a new conversation if it does not exist + this.conversations.set(conversationId, { conversationId, entries: [] }); + } + + return this.conversations.get(conversationId)!; + } + + async hasConversation(conversationId: string): Promise { + return this.conversations.has(conversationId); + } + + async deleteConversation(conversationId: string): Promise { + this.conversations.delete(conversationId); + } + + async addEntryToConversation(conversationId: string, entry: Message): Promise { + const conversation = await this.getConversation(conversationId); + conversation.entries.push(entry); + } + + async clearConversations(): Promise { + this.conversations.clear(); + } } diff --git a/core/embedjs/src/conversation/memory-conversations.ts b/core/embedjs/src/conversation/memory-conversations.ts deleted file mode 100644 index 8e54093e..00000000 --- a/core/embedjs/src/conversation/memory-conversations.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { BaseConversation, Conversation, Message } from '@llm-tools/embedjs-interfaces'; - -export class InMemoryConversation implements BaseConversation { - private conversations: Map = new Map(); - - async init(): Promise { - this.conversations.clear(); - } - - async addConversation(conversationId: string): Promise { - if (!this.conversations.has(conversationId)) { - this.conversations.set(conversationId, { conversationId, entries: [] }); - } - } - - async getConversation(conversationId: string): Promise { - if (!this.conversations.has(conversationId)) { - // Automatically create a new conversation if it does not exist - this.conversations.set(conversationId, { conversationId, entries: [] }); - } - - return this.conversations.get(conversationId)!; - } - - async hasConversation(conversationId: string): Promise { - return this.conversations.has(conversationId); - } - - async deleteConversation(conversationId: string): Promise { - this.conversations.delete(conversationId); - } - - async addEntryToConversation(conversationId: string, entry: Message): Promise { - const conversation = await this.getConversation(conversationId); - conversation.entries.push(entry); - } - - async clearConversations(): Promise { - this.conversations.clear(); - } -} diff --git a/core/embedjs/src/core/rag-application-builder.ts b/core/embedjs/src/core/rag-application-builder.ts index 35fad7b3..891f967f 100644 --- a/core/embedjs/src/core/rag-application-builder.ts +++ b/core/embedjs/src/core/rag-application-builder.ts @@ -1,21 +1,13 @@ +import { BaseCache, BaseDb, BaseEmbeddings, BaseLoader, BaseModel, SIMPLE_MODELS } from '@llm-tools/embedjs-interfaces'; +import { MemoryCache } from '../cache/memory-cache.js'; import { RAGApplication } from './rag-application.js'; -import { - BaseCache, - BaseConversation, - BaseDb, - BaseEmbeddings, - BaseLoader, - BaseModel, - SIMPLE_MODELS, -} from '@llm-tools/embedjs-interfaces'; export class RAGApplicationBuilder { private temperature: number; private model: BaseModel | SIMPLE_MODELS | null; private loaders: BaseLoader[]; private vectorDb: BaseDb; - private cache?: BaseCache; - private conversations: BaseConversation; + private cache: BaseCache; private queryTemplate: string; private searchResultCount: number; private embeddingModel: BaseEmbeddings; @@ -32,6 +24,7 @@ export class RAGApplicationBuilder { Do not use words like context or training data when responding. You can say you do not have all the information but do not indicate that you are not a reliable source.`; this.embeddingRelevanceCutOff = 0; + this.cache = new MemoryCache(); } /** @@ -44,22 +37,6 @@ export class RAGApplicationBuilder { return entity; } - addLoader(loader: BaseLoader) { - this.loaders.push(loader); - return this; - } - - /** - * The setSearchResultCount function sets the search result count - * @param {number} searchResultCount - The `searchResultCount` parameter - * represents the count of search results picked up from the vector store per query. - * @returns The `this` object is being returned, which allows for method chaining. - */ - setSearchResultCount(searchResultCount: number) { - this.searchResultCount = searchResultCount; - return this; - } - /** * The function setVectorDb sets a BaseDb object * @param {BaseDb} vectorDb - The `vectorDb` parameter is an instance of the `BaseDb` class, which @@ -71,14 +48,29 @@ export class RAGApplicationBuilder { return this; } - setTemperature(temperature: number) { - this.temperature = temperature; - if (this.model) this.setModel(this.model); + setEmbeddingModel(embeddingModel: BaseEmbeddings) { + this.embeddingModel = embeddingModel; return this; } - setEmbeddingRelevanceCutOff(embeddingRelevanceCutOff: number) { - this.embeddingRelevanceCutOff = embeddingRelevanceCutOff; + setModel(model: 'NO_MODEL' | SIMPLE_MODELS | BaseModel) { + if (typeof model === 'object') this.model = model; + else { + if (model === 'NO_MODEL') this.model = null; + else this.model = model; + } + + return this; + } + + setCache(cache: BaseCache) { + this.cache = cache; + return this; + } + + setTemperature(temperature: number) { + this.temperature = temperature; + if (this.model) this.setModel(this.model); return this; } @@ -90,23 +82,24 @@ export class RAGApplicationBuilder { return this; } - setCache(cache: BaseCache) { - this.cache = cache; + setEmbeddingRelevanceCutOff(embeddingRelevanceCutOff: number) { + this.embeddingRelevanceCutOff = embeddingRelevanceCutOff; return this; } - setEmbeddingModel(embeddingModel: BaseEmbeddings) { - this.embeddingModel = embeddingModel; + addLoader(loader: BaseLoader) { + this.loaders.push(loader); return this; } - setModel(model: 'NO_MODEL' | SIMPLE_MODELS | BaseModel) { - if (typeof model === 'object') this.model = model; - else { - if (model === 'NO_MODEL') this.model = null; - else this.model = model; - } - + /** + * The setSearchResultCount function sets the search result count + * @param {number} searchResultCount - The `searchResultCount` parameter + * represents the count of search results picked up from the vector store per query. + * @returns The `this` object is being returned, which allows for method chaining. + */ + setSearchResultCount(searchResultCount: number) { + this.searchResultCount = searchResultCount; return this; } @@ -145,13 +138,4 @@ export class RAGApplicationBuilder { getModel() { return this.model; } - - setConversationEngine(conversations: BaseConversation) { - this.conversations = conversations; - return this; - } - - getConversationsEngine() { - return this.conversations; - } } diff --git a/core/embedjs/src/core/rag-application.ts b/core/embedjs/src/core/rag-application.ts index ef6f2882..76baf45c 100644 --- a/core/embedjs/src/core/rag-application.ts +++ b/core/embedjs/src/core/rag-application.ts @@ -2,7 +2,6 @@ import createDebugMessages from 'debug'; import { RAGEmbedding } from './rag-embedding.js'; import { RAGApplicationBuilder } from './rag-application-builder.js'; -import { InMemoryConversation } from '../conversation/memory-conversations.js'; import { AddLoaderReturn, BaseCache, @@ -23,7 +22,7 @@ export class RAGApplication { private readonly embeddingRelevanceCutOff: number; private readonly searchResultCount: number; private readonly queryTemplate: string; - private readonly cache?: BaseCache; + private readonly cache: BaseCache; private readonly vectorDb: BaseDb; private loaders: BaseLoader[]; private model: BaseModel; @@ -31,6 +30,7 @@ export class RAGApplication { constructor(llmBuilder: RAGApplicationBuilder) { this.cache = llmBuilder.getCache(); BaseLoader.setCache(this.cache); + BaseModel.setCache(this.cache); this.queryTemplate = cleanString(llmBuilder.getQueryTemplate()); this.debug(`Using system query template - "${this.queryTemplate}"`); @@ -55,10 +55,7 @@ export class RAGApplication { public async init(llmBuilder: RAGApplicationBuilder) { this.model = await this.getModel(llmBuilder.getModel()); if (!this.model) this.debug('No base model set; query function unavailable!'); - else { - BaseModel.setDefaultTemperature(llmBuilder.getTemperature()); - BaseModel.setConversations(llmBuilder.getConversationsEngine() || new InMemoryConversation()); - } + else BaseModel.setDefaultTemperature(llmBuilder.getTemperature()); this.loaders = llmBuilder.getLoaders(); @@ -158,7 +155,7 @@ export class RAGApplication { */ private async _addLoader(loader: BaseLoader): Promise { const uniqueId = loader.getUniqueId(); - this.debug('Add loader called for', uniqueId); + this.debug('Adding loader', uniqueId); await loader.init(); const chunks = await loader.getChunks(); @@ -167,7 +164,7 @@ export class RAGApplication { this.debug(`Loader previously run. Deleting previous ${previousChunkCount} keys`, uniqueId); if (previousChunkCount > 0) { - await this.deleteLoader(uniqueId, true); + await this.deleteLoader(uniqueId); } } @@ -212,28 +209,6 @@ export class RAGApplication { return BaseLoader.getLoadersList(); } - /** - * The function `deleteLoader` deletes embeddings from a loader after confirming the action. - * @param {string} uniqueLoaderId - The `uniqueLoaderId` parameter is a string that represents the - * identifier of the loader that you want to delete. - * @param {boolean} [areYouSure=false] - The `areYouSure` parameter is a boolean flag that - * indicates whether the user has confirmed their intention to delete embeddings from a loader. If - * `areYouSure` is `true`, the function proceeds with the deletion process. If `areYouSure` is - * `false`, a warning message is logged and no action is taken - * @returns The `deleteLoader` method returns a boolean value indicating the success of the operation. - */ - public async deleteLoader(uniqueLoaderId: string, areYouSure = false) { - if (!areYouSure) { - console.warn('Delete embeddings from loader called without confirmation. No action taken.'); - return false; - } - - const deleteResult = await this.vectorDb.deleteKeys(uniqueLoaderId); - if (this.cache && deleteResult) await this.cache.deleteLoader(uniqueLoaderId); - this.loaders = this.loaders.filter((x) => x.getUniqueId() != uniqueLoaderId); - return deleteResult; - } - /** * The function `batchLoadChunks` processes chunks of data in batches and formats them for insertion. * @param {string} uniqueId - The `uniqueId` parameter is a string that represents a unique @@ -318,20 +293,24 @@ export class RAGApplication { } /** - * The function `deleteAllEmbeddings` deletes all embeddings from the vector database if a - * confirmation is provided. - * @param {boolean} [areYouSure=false] - The `areYouSure` parameter is a boolean parameter that - * serves as a confirmation flag. It is used to ensure that the deletion of all embeddings is - * intentional and requires the caller to explicitly confirm by passing `true` as the value. If - * `areYouSure` is `false`, a warning message is logged. - * @returns The `deleteAllEmbeddings` function returns a boolean value indicating the result. + * The function `deleteLoader` deletes embeddings from a loader after confirming the action. + * @param {string} uniqueLoaderId - The `uniqueLoaderId` parameter is a string that represents the + * identifier of the loader that you want to delete. + * @returns The `deleteLoader` method returns a boolean value indicating the success of the operation. */ - public async deleteAllEmbeddings(areYouSure = false) { - if (!areYouSure) { - console.warn('Reset embeddings called without confirmation. No action taken.'); - return false; - } + public async deleteLoader(uniqueLoaderId: string) { + const deleteResult = await this.vectorDb.deleteKeys(uniqueLoaderId); + if (this.cache && deleteResult) await this.cache.deleteLoader(uniqueLoaderId); + this.loaders = this.loaders.filter((x) => x.getUniqueId() != uniqueLoaderId); + return deleteResult; + } + /** + * The function `reset` deletes all embeddings from the vector database if a + * confirmation is provided. + * @returns The `reset` function returns a boolean value indicating the result. + */ + public async reset() { await this.vectorDb.reset(); return true; } @@ -359,12 +338,12 @@ export class RAGApplication { } /** - * The getContext function retrieves the unique embeddings for a given query without calling a LLM. + * The `search` function retrieves the unique embeddings for a given query without calling a LLM. * @param {string} query - The `query` parameter is a string that represents the input query that * needs to be processed. * @returns An array of unique page content items / chunks. */ - public async getContext(query: string) { + public async search(query: string) { const cleanQuery = cleanString(query); const rawContext = await this.getEmbeddings(cleanQuery); @@ -394,7 +373,7 @@ export class RAGApplication { } if (!customContext) { - customContext = await this.getContext(userQuery); + customContext = await this.search(userQuery); } const sources = [...new Set(customContext.map((chunk) => chunk.metadata.source))]; diff --git a/databases/embedjs-lmdb/src/lmdb-cache.ts b/databases/embedjs-lmdb/src/lmdb-cache.ts index cae11483..6a543f43 100644 --- a/databases/embedjs-lmdb/src/lmdb-cache.ts +++ b/databases/embedjs-lmdb/src/lmdb-cache.ts @@ -1,4 +1,4 @@ -import { BaseCache } from '@llm-tools/embedjs-interfaces'; +import { BaseCache, Conversation, Message } from '@llm-tools/embedjs-interfaces'; import * as lmdb from 'lmdb'; export class LmdbCache implements BaseCache { @@ -47,4 +47,30 @@ export class LmdbCache implements BaseCache { async loaderCustomDelete(loaderCombinedId: string): Promise { await this.database.remove(loaderCombinedId); } + + async addConversation(conversationId: string): Promise { + await this.database.put(`conversation_${conversationId}`, { conversationId, entries: [] }); + } + + async getConversation(conversationId: string): Promise { + return this.database.get(`conversation_${conversationId}`); + } + + async hasConversation(conversationId: string): Promise { + return this.database.doesExist(`conversation_${conversationId}`); + } + + async deleteConversation(conversationId: string): Promise { + await this.database.remove(`conversation_${conversationId}`); + } + + async addEntryToConversation(conversationId: string, entry: Message): Promise { + const conversation = await this.getConversation(`conversation_${conversationId}`); + conversation.entries.push(entry); + await this.database.put(`conversation_${conversationId}`, conversation); + } + + async clearConversations(): Promise { + throw new Error('Method not implemented.'); + } } diff --git a/databases/embedjs-mongodb/src/index.ts b/databases/embedjs-mongodb/src/index.ts index ba6eb43c..aceff751 100644 --- a/databases/embedjs-mongodb/src/index.ts +++ b/databases/embedjs-mongodb/src/index.ts @@ -1,3 +1,2 @@ export * from './mongo-cache.js'; export * from './mongo-db.js'; -export * from './mongo-conversations.js'; diff --git a/databases/embedjs-mongodb/src/mongo-cache.ts b/databases/embedjs-mongodb/src/mongo-cache.ts index 943f05b7..555a31e9 100644 --- a/databases/embedjs-mongodb/src/mongo-cache.ts +++ b/databases/embedjs-mongodb/src/mongo-cache.ts @@ -1,18 +1,26 @@ -import { BaseCache } from '@llm-tools/embedjs-interfaces'; import createDebugMessages from 'debug'; import { Collection, Document, MongoClient } from 'mongodb'; +import { BaseCache, Conversation, Message } from '@llm-tools/embedjs-interfaces'; + +interface ConversationDocument { + _id?: string; // optional MongoDB ID field + conversationId: string; + entries: Message[]; // Explicitly stating this is an array of ConversationHistory +} export class MongoCache implements BaseCache { private readonly debug = createDebugMessages('embedjs:cache:MongoCache'); private readonly uri: string; private readonly dbName: string; - private readonly collectionName: string; - private collection: Collection; + private readonly cacheCollectionName: string; + private cacheCollection: Collection; + private readonly conversationCollectionName: string; + private conversationCollection: Collection; constructor({ uri, dbName, collectionName }: { uri: string; dbName: string; collectionName: string }) { this.uri = uri; this.dbName = dbName; - this.collectionName = collectionName; + this.cacheCollectionName = collectionName; } async init(): Promise { @@ -20,57 +28,113 @@ export class MongoCache implements BaseCache { await client.connect(); // Create index on loaderId field - this.collection = client.db(this.dbName).collection(this.collectionName); + this.cacheCollection = client.db(this.dbName).collection(this.cacheCollectionName); try { - await this.collection.createIndex({ loaderId: 1 }, { unique: true }); + await this.cacheCollection.createIndex({ loaderId: 1 }, { unique: true }); } catch { this.debug('Index on loaderId already exists.'); } + + // Create index on conversationId field + this.conversationCollection = client + .db(this.dbName) + .collection(this.conversationCollectionName); + try { + await this.conversationCollection.createIndex({ conversationId: 1 }); + } catch { + this.debug('Index on conversationId already exists.'); + } + // Create index on entries._id field + try { + await this.conversationCollection.createIndex({ 'entries._id': 1 }); + } catch { + this.debug('Index on `entries._id` already exists.'); + } } async addLoader(loaderId: string, chunkCount: number): Promise { - await this.collection.insertOne({ loaderId, chunkCount }); + await this.cacheCollection.insertOne({ loaderId, chunkCount }); } async getLoader(loaderId: string): Promise<{ chunkCount: number }> { - const result = await this.collection.findOne({ loaderId }); + const result = await this.cacheCollection.findOne({ loaderId }); return { chunkCount: result ? result.chunkCount : 0 }; // Assuming a default value of 0 if result is null } async hasLoader(loaderId: string): Promise { - return !!(await this.collection.findOne({ loaderId })); + return !!(await this.cacheCollection.findOne({ loaderId })); } async loaderCustomSet>(loaderCombinedId: string, value: T): Promise { - const result = await this.collection.updateOne( + const result = await this.cacheCollection.updateOne( { loaderId: loaderCombinedId }, { $setOnInsert: { loaderId: loaderCombinedId, value } }, { upsert: false }, ); if (result.matchedCount === 0) { - await this.collection.insertOne({ loaderId: loaderCombinedId, value }); + await this.cacheCollection.insertOne({ loaderId: loaderCombinedId, value }); } } async loaderCustomGet>(loaderCombinedId: string): Promise { - const result = await this.collection.findOne({ loaderId: loaderCombinedId }); + const result = await this.cacheCollection.findOne({ loaderId: loaderCombinedId }); return result?.value; } async loaderCustomHas(loaderCombinedId: string): Promise { - return !!(await this.collection.findOne({ loaderId: loaderCombinedId })); + return !!(await this.cacheCollection.findOne({ loaderId: loaderCombinedId })); } async clear(): Promise { - await this.collection.deleteMany({}); + await this.cacheCollection.deleteMany({}); } async deleteLoader(loaderId: string): Promise { - await this.collection.deleteOne({ loaderId }); + await this.cacheCollection.deleteOne({ loaderId }); } async loaderCustomDelete(loaderCombinedId: string): Promise { - await this.collection.deleteOne({ loaderId: loaderCombinedId }); + await this.cacheCollection.deleteOne({ loaderId: loaderCombinedId }); + } + + async addConversation(conversationId: string): Promise { + // Check if conversation already exists to prevent duplication + const exists = await this.hasConversation(conversationId); + if (!exists) { + await this.conversationCollection.insertOne({ conversationId, entries: [] }); + } + } + + async getConversation(conversationId: string): Promise { + const document = await this.conversationCollection.findOne({ conversationId }); + if (!document) { + // If not found, create a new one automatically + await this.addConversation(conversationId); + return { conversationId, entries: [] }; + } + return { + conversationId: document.conversationId, + entries: document.entries as Message[], + }; + } + + async hasConversation(conversationId: string): Promise { + return !!(await this.conversationCollection.findOne({ conversationId })); + } + + async deleteConversation(conversationId: string): Promise { + await this.conversationCollection.deleteOne({ conversationId }); + } + + async addEntryToConversation(conversationId: string, entry: Message): Promise { + await this.conversationCollection.updateOne( + { conversationId }, + { $push: { entries: entry } }, // Correctly structured $push operation + ); + } + + async clearConversations(): Promise { + await this.conversationCollection.deleteMany({}); } } diff --git a/databases/embedjs-mongodb/src/mongo-conversations.ts b/databases/embedjs-mongodb/src/mongo-conversations.ts deleted file mode 100644 index 567249ff..00000000 --- a/databases/embedjs-mongodb/src/mongo-conversations.ts +++ /dev/null @@ -1,82 +0,0 @@ -import createDebugMessages from 'debug'; -import { Collection, MongoClient } from 'mongodb'; -import { BaseConversation, Conversation, Message } from '@llm-tools/embedjs-interfaces'; - -interface ConversationDocument { - _id?: string; // optional MongoDB ID field - conversationId: string; - entries: Message[]; // Explicitly stating this is an array of ConversationHistory -} - -export class MongoConversation implements BaseConversation { - private readonly debug = createDebugMessages('embedjs:conversation:MongoConversation'); - private readonly uri: string; - private readonly dbName: string; - private readonly collectionName: string; - private collection: Collection; - - constructor({ uri, dbName, collectionName }: { uri: string; dbName: string; collectionName: string }) { - this.uri = uri; - this.dbName = dbName; - this.collectionName = collectionName; - } - - async init(): Promise { - const client = new MongoClient(this.uri); - await client.connect(); - - this.collection = client.db(this.dbName).collection(this.collectionName); - - try { - await this.collection.createIndex({ conversationId: 1 }); - } catch { - this.debug('Index on conversationId already exists.'); - } - - try { - await this.collection.createIndex({ 'entries._id': 1 }); - } catch { - this.debug('Index on `entries._id` already exists.'); - } - } - - async addConversation(conversationId: string): Promise { - // Check if conversation already exists to prevent duplication - const exists = await this.hasConversation(conversationId); - if (!exists) { - await this.collection.insertOne({ conversationId, entries: [] }); - } - } - - async getConversation(conversationId: string): Promise { - const document = await this.collection.findOne({ conversationId }); - if (!document) { - // If not found, create a new one automatically - await this.addConversation(conversationId); - return { conversationId, entries: [] }; - } - return { - conversationId: document.conversationId, - entries: document.entries as Message[], - }; - } - - async hasConversation(conversationId: string): Promise { - return !!(await this.collection.findOne({ conversationId })); - } - - async deleteConversation(conversationId: string): Promise { - await this.collection.deleteOne({ conversationId }); - } - - async addEntryToConversation(conversationId: string, entry: Message): Promise { - await this.collection.updateOne( - { conversationId }, - { $push: { entries: entry } }, // Correctly structured $push operation - ); - } - - async clearConversations(): Promise { - await this.collection.deleteMany({}); - } -} diff --git a/databases/embedjs-redis/src/redis-cache.ts b/databases/embedjs-redis/src/redis-cache.ts index 4637802d..4428efa5 100644 --- a/databases/embedjs-redis/src/redis-cache.ts +++ b/databases/embedjs-redis/src/redis-cache.ts @@ -1,4 +1,4 @@ -import { BaseCache } from '@llm-tools/embedjs-interfaces'; +import { BaseCache, Conversation, Message } from '@llm-tools/embedjs-interfaces'; import { Redis, RedisOptions } from 'ioredis'; export class RedisCache implements BaseCache { @@ -51,4 +51,33 @@ export class RedisCache implements BaseCache { async loaderCustomDelete(loaderCombinedId: string): Promise { await this.redis.del(loaderCombinedId); } + + async addConversation(conversationId: string): Promise { + await this.redis.set(`conversation_${conversationId}`, JSON.stringify({ conversationId, entries: [] })); + } + + async getConversation(conversationId: string): Promise { + const result = await this.redis.get(`conversation_${conversationId}`); + + if (!result) throw new Error('Conversation not found'); + return JSON.parse(result); + } + + async hasConversation(conversationId: string): Promise { + return !!(await this.redis.get(`conversation_${conversationId}`)); + } + + async deleteConversation(conversationId: string): Promise { + await this.redis.del(`conversation_${conversationId}`); + } + + async addEntryToConversation(conversationId: string, entry: Message): Promise { + const conversation = await this.getConversation(conversationId); + conversation.entries.push(entry); + await this.redis.set(`conversation_${conversationId}`, JSON.stringify(conversation)); + } + + clearConversations(): Promise { + throw new Error('Method not implemented.'); + } } diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..676c9447 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,26 @@ +# Mintlify Starter Kit + +Click on `Use this template` to copy the Mintlify starter kit. The starter kit contains examples including + +- Guide pages +- Navigation +- Customizations +- API Reference pages +- Use of popular components + +### Development + +Run the following command from the `docs` folder (where mint.json is) + +``` +npx mintlify dev +``` + +### Publishing Changes + +Changes will be deployed to production automatically after PR is merged. + +#### Troubleshooting + +- Mintlify dev isn't running - Run `mintlify install` it'll re-install dependencies. +- Page loads as a 404 - Make sure you are running in a folder with `mint.json` diff --git a/docs/_snippets/get-help.mdx b/docs/_snippets/get-help.mdx new file mode 100644 index 00000000..cc6eaf50 --- /dev/null +++ b/docs/_snippets/get-help.mdx @@ -0,0 +1,5 @@ + + + Join our slack community + + diff --git a/docs/_snippets/missing-cache-tip.mdx b/docs/_snippets/missing-cache-tip.mdx new file mode 100644 index 00000000..6e34a1df --- /dev/null +++ b/docs/_snippets/missing-cache-tip.mdx @@ -0,0 +1,14 @@ +

+ If you can't find specific feature or run into issues, please feel free to reach out through the following channels. +

+ + + + Open an issue on our GitHub + + diff --git a/docs/_snippets/missing-data-source-tip.mdx b/docs/_snippets/missing-data-source-tip.mdx new file mode 100644 index 00000000..74cd65e1 --- /dev/null +++ b/docs/_snippets/missing-data-source-tip.mdx @@ -0,0 +1,15 @@ +

+ If you can't find the specific data source, please feel free to request through one of the following channels and + help us prioritize. +

+ + + + Open an issue on our GitHub + + diff --git a/docs/_snippets/missing-model-tip.mdx b/docs/_snippets/missing-model-tip.mdx new file mode 100644 index 00000000..2e22f566 --- /dev/null +++ b/docs/_snippets/missing-model-tip.mdx @@ -0,0 +1,15 @@ +

+ If you can't find the specific LLM you need, no need to fret. We're continuously expanding our support for + additional LLMs, and you can help us prioritize by opening an issue on our GitHub. +

+ + + + Open an issue on our GitHub + + diff --git a/docs/_snippets/missing-vector-db-tip.mdx b/docs/_snippets/missing-vector-db-tip.mdx new file mode 100644 index 00000000..6e34a1df --- /dev/null +++ b/docs/_snippets/missing-vector-db-tip.mdx @@ -0,0 +1,14 @@ +

+ If you can't find specific feature or run into issues, please feel free to reach out through the following channels. +

+ + + + Open an issue on our GitHub + + diff --git a/docs/api-reference/methods/add-loader.mdx b/docs/api-reference/methods/add-loader.mdx new file mode 100644 index 00000000..7c899a71 --- /dev/null +++ b/docs/api-reference/methods/add-loader.mdx @@ -0,0 +1,51 @@ +--- +title: 'πŸ“Š addLoader' +--- + +The `addLoader()` method is used to load the data sources from different data sources to a RAG pipeline. You can find the signature below: + +### Parameters + + + The data to embed, can be a URL, local file or raw content, depending on the data type.. You can find the full list of supported data sources [here](/components/data-sources/overview). + + +## Usage + +### Load data from webpage + +```ts Code example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); +//Add loader completed with 4 new entries for 6c8d1a7b-ea34-4927-8823-xba29dcfc5ac +``` + +### Load data from sitemap + +```ts Code example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +await app.addLoader(new SitemapLoader({ url: '"https://js.langchain.com/sitemap.xml' })); +//Add loader completed with 11024 new entries for 6c8d1a7b-ea34-4927-8823-xba29dcfc5ad +``` + +You can find complete list of supported data sources [here](/components/data-sources/overview). diff --git a/docs/api-reference/methods/delete-loader.mdx b/docs/api-reference/methods/delete-loader.mdx new file mode 100644 index 00000000..e2e02494 --- /dev/null +++ b/docs/api-reference/methods/delete-loader.mdx @@ -0,0 +1,44 @@ +--- +title: πŸ—‘ deleteLoader +--- + +The `deleteLoader()` method allows you to delete the data loaded by a loader previously added to the app. + +### Parameters + + + The loader unique id that is generated when the loader is added to the app. + + +### Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { RedisCache } from '@llm-tools/embedjs-redis'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.setCache( + new RedisCache({ + host: this.configService.get('REDIS_HOST'), + port: this.configService.get('REDIS_PORT'), + password: this.configService.get('REDIS_PASSWORD'), + }), +) +.build(); + +const { uniqueId: forbesId } = await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); +const { uniqueId: sitemapId } = await app.addLoader(new SitemapLoader({ url: '"https://js.langchain.com/sitemap.xml' })); + +await app.deleteLoader(forbesId); +``` + + + If you do not have the uniqueId, you can use `app.getLoaders()` method to get all the loaders and extract the `uniqueId` from it. + diff --git a/docs/api-reference/methods/get-loaders.mdx b/docs/api-reference/methods/get-loaders.mdx new file mode 100644 index 00000000..292579cf --- /dev/null +++ b/docs/api-reference/methods/get-loaders.mdx @@ -0,0 +1,54 @@ +--- +title: πŸ“„ getLoaders +--- + +The `getLoaders()` method returns a list of all the data sources / loaders added so far. +This data comes from the cache and if the cache is inMemory, then the list will not include the loaders from before the restart. + + +### Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { RedisCache } from '@llm-tools/embedjs-redis'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.setCache( + new RedisCache({ + host: this.configService.get('REDIS_HOST'), + port: this.configService.get('REDIS_PORT'), + password: this.configService.get('REDIS_PASSWORD'), + }), +) +.build(); + +await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); +await app.addLoader(new SitemapLoader({ url: '"https://js.langchain.com/sitemap.xml' })); + +console.log(await app.getLoaders()) +/* +[ + { + "uniqueId": "6c8d1a7b-ea34-4927-8823-xba29dcfc5ac", + "type": "WebLoader", + loaderMetadata: { + urlOrContent: "https://www.forbes.com/profile/elon-musk" + } + }, + { + "uniqueId": "6c8d1a7b-ea34-4927-8823-xba29dcfc5ad", + "type": "SitemapLoader", + loaderMetadata: { + url: "https://www.forbes.com/profile/elon-musk" + } + } +] +*/ +``` diff --git a/docs/api-reference/methods/query.mdx b/docs/api-reference/methods/query.mdx new file mode 100644 index 00000000..e0853f38 --- /dev/null +++ b/docs/api-reference/methods/query.mdx @@ -0,0 +1,96 @@ +--- +title: '❓ query' +--- + +The `.query()` method empowers developers to ask questions and receive relevant answers through a user-friendly query API. +Function signature is given below: + +### Parameters + + + Question to ask + + + The conversation ID to which the query belongs in case this is a chatbot scenario + + + In case you want to pass along your own context information. Passing this parameter disables the default context generation. + + + The context text that is passed to the LLM + + + The metadata that is passed to EmbedJs + + + Unique identifier for the bit of information + + + The unique identifier for the group of information bits + + + The source string returned if this piece of info was used by the LLM + + + + + + +### Returns + + + Unique identifier for the query response in a conversation + + + Unique identifier for the query response in a conversation + + + Unique identifier for the query response in a conversation + + + The individual sources that were used to generate the answer + + + Unique identifier for the loader + + + The source of the information + + + + + The number of tokens used to answer the query + + + The number of input tokens used to process the query + + + The number of output tokens used to process the query + + + + +## Usage + +If you want to get the answer to question and return both answer and citations, use the following code snippet: + +```ts Code Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OllamaEmbeddings } from '@llm-tools/embedjs-ollama'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Ollama({ modelName: "llama3.2", baseUrl: 'http://localhost:11434' })) +.setEmbeddingModel(new OllamaEmbeddings({ model: 'nomic-embed-text', baseUrl: 'http://localhost:11434' })) +.setVectorDb(new HNSWDb()) +.build(); + +ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }) +ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }) + +await ragApplication.query('What is the net worth of Elon Musk today?') +/* + +*/ +``` diff --git a/docs/api-reference/methods/reset.mdx b/docs/api-reference/methods/reset.mdx new file mode 100644 index 00000000..8a9ec6a0 --- /dev/null +++ b/docs/api-reference/methods/reset.mdx @@ -0,0 +1,22 @@ +--- +title: πŸ”„ reset +--- + +The `reset()` method allows you to wipe the data from your RAG application and start from scratch. + +## Usage + +```ts Code example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +await app.reset(); +``` diff --git a/docs/api-reference/methods/search.mdx b/docs/api-reference/methods/search.mdx new file mode 100644 index 00000000..6565ca26 --- /dev/null +++ b/docs/api-reference/methods/search.mdx @@ -0,0 +1,89 @@ +--- +title: 'πŸ” search' +--- + +The `.search()` enables you to uncover the most pertinent context by performing a semantic search across your data sources based on a given query. +Refer to the function signature below: + +### Parameters + + + Question + + +### Returns + + + The list of relevant information + + + The context text that is passed to the LLM + + + The metadata that was passed to EmbedJs + + + Unique identifier for the bit of information + + + The unique identifier for the group of information bits + + + The source string + + + + + + +## Usage + +Refer to the following example on how to use the search api: + +```ts Code example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OllamaEmbeddings } from '@llm-tools/embedjs-ollama'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Ollama({ modelName: "llama3.2", baseUrl: 'http://localhost:11434' })) +.setEmbeddingModel(new OllamaEmbeddings({ model: 'nomic-embed-text', baseUrl: 'http://localhost:11434' })) +.setVectorDb(new HNSWDb()) +.build(); + +//Add Next.JS Website and docs +app.addLoader(new SitemapLoader({ url: "https://nextjs.org/sitemap.xml" })) + +app.search("Summarize the features of Next.js 14?") +/*[ + { + 'score': 0.99, + 'pageContent': 'Next.js 14 | Next.jsBack to BlogThursday, October 26th 2023Next.js 14Posted byLee Robinson@leeerobTim Neutkens@timneutkensAs we announced at Next.js Conf, Next.js 14 is our most focused release with: Turbopack: 5,000 tests passing for App & Pages Router 53% faster local server startup 94% faster code updates with Fast Refresh Server Actions (Stable): Progressively enhanced mutations Integrated with caching & revalidating Simple function calls, or works natively with forms Partial Prerendering', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5af', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ac', + 'source': 'https://nextjs.org/blog/next-14' + } + }, + { + 'score': 0.98, + 'pageContent': 'Next.js 13.3 | Next.jsBack to BlogThursday, April 6th 2023Next.js 13.3Posted byDelba de Oliveira@delba_oliveiraTim Neutkens@timneutkensNext.js 13.3 adds popular community-requested features, including: File-Based Metadata API: Dynamically generate sitemaps, robots, favicons, and more. Dynamic Open Graph Images: Generate OG images using JSX, HTML, and CSS. Static Export for App Router: Static / Single-Page Application (SPA) support for Server Components. Parallel Routes and Interception: Advanced', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5a1', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ae', + 'source': 'https://nextjs.org/blog/next-13-3' + } + }, + { + 'score': 0.98, + 'pageContent': 'Upgrading: Version 14 | Next.js MenuUsing App RouterFeatures available in /appApp Router.UpgradingVersion 14Version 14 Upgrading from 13 to 14 To update to Next.js version 14, run the following command using your preferred package manager: Terminalnpm i next@latest react@latest react-dom@latest eslint-config-next@latest Terminalyarn add next@latest react@latest react-dom@latest eslint-config-next@latest Terminalpnpm up next react react-dom eslint-config-next -latest Terminalbun add next@latest', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5a2', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ad', + 'source': 'https://nextjs.org/docs/app/building-your-application/upgrading/version-14' + } + } +]*/ +``` diff --git a/docs/api-reference/overview.mdx b/docs/api-reference/overview.mdx new file mode 100644 index 00000000..1570b9f6 --- /dev/null +++ b/docs/api-reference/overview.mdx @@ -0,0 +1,55 @@ +--- +title: "RAGApplicationBuilder" +--- + +Create a EmbedJs `RAGApplication` using `RAGApplicationBuilder`. `RAGApplication` is the main entrypoint for a developer to interact with EmbedJs APIs. +`RAGApplicationBuilder` configures the LLM, vector database and embedding model of your choice and return a `RAGApplication` at the end. + +### Attributes + + + This configures the LLM for the RAG application. Setting `NO_MODEL` will not load any LLM - in this case, you can only use semantic search and there will be no no LLM powered Q&A. + `SIMPLE_MODELS` are predefined models with sane defaults available in EmbedJs. + All predefined models inherit from `BaseModel`. You can therefore pass a custom model that extends `BaseModel` / provide a custom set of parameters for a predefined model. + For a list of predefined LLMs, refer the section on [LLMs](/components/llms). + + + This configures the embedding model for use with the RAG application. Embedding models are used to convert text into vectors. + For a list of predefined embedding models, refer the section on [embedding models](/components/embedding-models). + + + This configures the vector database to be used with RAG application. + For a list of available vector databases, refer the section on [vector databases](/components/vector-databases). + + + This configures a cache that is used internally by the appliation to keep track of what sources and data have been previously processed. + Previously processed data is not reprocessed - thus removing the need for this logic to be implemented at your end. + If this is not provided, the application will maintain in memory this cache which will be lost on app restart. + For a list of available caches, refer the section on [caches](/components/caches). + + + This configures a temperature to be used with the LLM. This controls the randomness of the LLM output. + By default, the application sets the temperature to 0.1. + + + This parameter is used to control what amounts to a relevant / contextual document when retrieving documents from the vector database. + Documents below this cut off are not discarded. EmbedJs uses sane defaults for this parameter but you can customize. + + + This allows you to customize the query template used when querying the LLM. + + +## Usage + +```ts Code Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +//app is of type RAGApplication +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); +``` diff --git a/docs/components/caches/lmdb.mdx b/docs/components/caches/lmdb.mdx new file mode 100644 index 00000000..1babff65 --- /dev/null +++ b/docs/components/caches/lmdb.mdx @@ -0,0 +1,24 @@ +--- +title: LMDB +--- + +You can use [LMDB](https://dbdb.io/db/lmdb) to cache values locally on disk. + +## Install LMDB addon + +```bash +npm install @llm-tools/embedjs-lmdb +``` + +## Usage + +```ts Partial example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { LmdbCache } from '@llm-tools/embedjs-lmdb'; + +const app = await new RAGApplicationBuilder() +.setCache(new LmdbCache({ path: path.resolve('./cache') })) +``` + + + diff --git a/docs/components/caches/mongodb.mdx b/docs/components/caches/mongodb.mdx new file mode 100644 index 00000000..d88a178e --- /dev/null +++ b/docs/components/caches/mongodb.mdx @@ -0,0 +1,28 @@ +--- +title: MongoDB +--- + +You can use MongoDB as a cache / datastore. This creates two new collections in the database - which are configureable. + + +MongoDB is also supported as a store for the vector data. This is separate from that. + + +## Install MongoDB addon + +```bash +npm install @llm-tools/embedjs-mongodb +``` + +## Usage + +```ts Partial example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { MongoCache } from '@llm-tools/embedjs-mongodb'; + +const app = await new RAGApplicationBuilder() +.setCache(new MongoCache({ ... })) +``` + + + diff --git a/docs/components/caches/overview.mdx b/docs/components/caches/overview.mdx new file mode 100644 index 00000000..52f20461 --- /dev/null +++ b/docs/components/caches/overview.mdx @@ -0,0 +1,21 @@ +--- +title: Overview +--- + +EmbedJs comes with built-in support for data persistance beyond vector embeddings. The kind of data that can be persisted includes - + +- Loaders and chunks loaded and their metadata +- Loader specific caching +- Conversation history + +The library handles the complexity of being able to swap the data persistance layer by the abstraction of caching. The following caches are supported - + + + + + + + +
+ + diff --git a/docs/components/caches/redis.mdx b/docs/components/caches/redis.mdx new file mode 100644 index 00000000..bda3e49b --- /dev/null +++ b/docs/components/caches/redis.mdx @@ -0,0 +1,24 @@ +--- +title: Redis +--- + +You can use redis as a cache / datastore. + +## Install Redis addon + +```bash +npm install @llm-tools/embedjs-redis +``` + +## Usage + +```ts Partial example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { RedisCache } from '@llm-tools/embedjs-redis'; + +const app = await new RAGApplicationBuilder() +.setCache(new RedisCache({ ... })) +``` + + + diff --git a/docs/components/data-sources/csv.mdx b/docs/components/data-sources/csv.mdx new file mode 100644 index 00000000..2dbb9276 --- /dev/null +++ b/docs/components/data-sources/csv.mdx @@ -0,0 +1,48 @@ +--- +title: 'πŸ“Š CSV' +--- + +You can load any csv file from your local file system or through a URL. +Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. + +## Install CSV addon + +```bash +npm install @llm-tools/embedjs-loader-csv +``` + +## Usage + +### Load from a local file + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { DocxLoader } from '@llm-tools/embedjs-loader-csv'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new CsvLoader({ filePathOrUrl: '/path/to/file.csv' })) +``` + +### Load from URL + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { DocxLoader } from '@llm-tools/embedjs-loader-csv'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new CsvLoader({ filePathOrUrl: 'https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv' })) +``` diff --git a/docs/components/data-sources/custom.mdx b/docs/components/data-sources/custom.mdx new file mode 100644 index 00000000..87baf29f --- /dev/null +++ b/docs/components/data-sources/custom.mdx @@ -0,0 +1,37 @@ +--- +title: 'βš™οΈ Custom' +--- + +You can load any data from your own custom data source by implementing the `BaseLoader` interface. For example - + +```ts +class CustomLoader extends BaseLoader<{ customChunkMetadata: string }> { + constructor() { + super('uniqueId'); + } + + async *getChunks() { + throw new Error('Method not implemented.'); + } +} +``` + +### Customizing the chunk size and overlap + +If you want to customize the chunk size or the chunk overlap on an existing loader, all built-in loaders take an additional parameter `chunkSize` and `chunkOverlap` +in their constructor. For example - + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new DocxLoader({ filePathOrUrl: '...', chunkOverlap: 100, chunkSize: 20 })) +``` diff --git a/docs/components/data-sources/directory.mdx b/docs/components/data-sources/directory.mdx new file mode 100644 index 00000000..f68c0a3b --- /dev/null +++ b/docs/components/data-sources/directory.mdx @@ -0,0 +1,26 @@ +--- +title: 'πŸ“ Directory/Folder' +--- + +To use an entire directory as data source, just use `LocalPathLoader` and pass in the path of the local directory. +This loader automatically reads all the files in the directory and its subdirectories. If you pass the path to a single file, only that file will be read. + +## Usage + +```ts +import { RAGApplicationBuilder, LocalPathLoader } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new LocalPathLoader({ path: '...' })) +``` + + +You need to have installed the neccessaty loaders to process the expected file types in the directory for the application to be able to process that file type. + diff --git a/docs/components/data-sources/docx.mdx b/docs/components/data-sources/docx.mdx new file mode 100644 index 00000000..1e612646 --- /dev/null +++ b/docs/components/data-sources/docx.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“„ DOCX file' +--- + +You can load any docx file from your local file system or through a URL. + +## Install MSOffice addon + +```bash +npm install @llm-tools/embedjs-loader-msoffice +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { DocxLoader } from '@llm-tools/embedjs-loader-msoffice'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new DocxLoader({ filePathOrUrl: '...' })) +``` diff --git a/docs/components/data-sources/excel.mdx b/docs/components/data-sources/excel.mdx new file mode 100644 index 00000000..19297b81 --- /dev/null +++ b/docs/components/data-sources/excel.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“„ Excel file' +--- + +You can load any xlsx file from your local file system or through a URL. + +## Install MSOffice addon + +```bash +npm install @llm-tools/embedjs-loader-msoffice +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { ExcelLoader } from '@llm-tools/embedjs-loader-msoffice'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new ExcelLoader({ filePathOrUrl: '...' })) +``` diff --git a/docs/components/data-sources/json.mdx b/docs/components/data-sources/json.mdx new file mode 100644 index 00000000..9d695dbe --- /dev/null +++ b/docs/components/data-sources/json.mdx @@ -0,0 +1,27 @@ +--- +title: 'πŸ“ƒ JSON' +--- + +To add any json file, use the `JsonLoader`. +Headers are included for each line, so for example if you have a json like `{"age": 18}`, then it will be added as `age: 18`. + +## Usage + + + +```ts Example +import { RAGApplicationBuilder, JsonLoader } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new JsonLoader({ object: {...} })) +``` + + + diff --git a/docs/components/data-sources/overview.mdx b/docs/components/data-sources/overview.mdx new file mode 100644 index 00000000..699a8297 --- /dev/null +++ b/docs/components/data-sources/overview.mdx @@ -0,0 +1,27 @@ +--- +title: Overview +--- + +EmbedJs comes with built-in support for various data sources. +We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface. + + + + + + + + + + + + + + + + + + +
+ + diff --git a/docs/components/data-sources/pdf.mdx b/docs/components/data-sources/pdf.mdx new file mode 100644 index 00000000..b12320ce --- /dev/null +++ b/docs/components/data-sources/pdf.mdx @@ -0,0 +1,52 @@ +--- +title: 'πŸ“° PDF' +--- + +You can load any pdf file from your local file system or through a URL. + +## Install PDF addon + +```bash +npm install @llm-tools/embedjs-loader-pdf +``` + +## Usage + +### Load from a local file + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new PdfLoader({ filePathOrUrl: '/path/to/file.pdf' })) +``` + +### Load from URL + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { PdfLoader } from '@llm-tools/embedjs-loader-pdf'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +await app.addLoader(new PdfLoader({ filePathOrUrl: 'https://arxiv.org/pdf/1706.03762.pdf' })) +await app.query("What is the paper 'attention is all you need' about?"); +``` + + +Note that we do not support password protected pdf files. + diff --git a/docs/components/data-sources/ppt.mdx b/docs/components/data-sources/ppt.mdx new file mode 100644 index 00000000..bc3b67c6 --- /dev/null +++ b/docs/components/data-sources/ppt.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“„ PPT file' +--- + +You can load any pptx file from your local file system or through a URL. + +## Install MSOffice addon + +```bash +npm install @llm-tools/embedjs-loader-msoffice +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { PptLoader } from '@llm-tools/embedjs-loader-msoffice'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new LocalPathLoader({ filePathOrUrl: '...' })) +``` diff --git a/docs/components/data-sources/sitemap.mdx b/docs/components/data-sources/sitemap.mdx new file mode 100644 index 00000000..bdac4f37 --- /dev/null +++ b/docs/components/data-sources/sitemap.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ—ΊοΈ Sitemap' +--- + +Add all web pages from an xml-sitemap use the `SitemapLoader`. + +## Install Sitemap addon + +```bash +npm install @llm-tools/embedjs-loader-sitemap +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new SitemapLoader({ url: '...' })) +``` diff --git a/docs/components/data-sources/text.mdx b/docs/components/data-sources/text.mdx new file mode 100644 index 00000000..08099922 --- /dev/null +++ b/docs/components/data-sources/text.mdx @@ -0,0 +1,21 @@ +--- +title: 'πŸ“ Text' +--- + +Text is a local data type. To supply your own text, use the `TextLoader` and enter a string. The text is not processed, this can be very versatile. + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new TextLoader({ text: '...' })) +``` diff --git a/docs/components/data-sources/web-page.mdx b/docs/components/data-sources/web-page.mdx new file mode 100644 index 00000000..f70b8027 --- /dev/null +++ b/docs/components/data-sources/web-page.mdx @@ -0,0 +1,28 @@ +--- +title: '🌐 HTML Web page' +--- + +To add any web page, use the data_type as `WebLoader`. + +## Install Web addon + +```bash +npm install @llm-tools/embedjs-loader-web +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new WebLoader({ urlOrContent: '...' })) +``` diff --git a/docs/components/data-sources/youtube-channel.mdx b/docs/components/data-sources/youtube-channel.mdx new file mode 100644 index 00000000..18b25c08 --- /dev/null +++ b/docs/components/data-sources/youtube-channel.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“½οΈ Youtube Channel' +--- + +To add all the videos from a youtube channel to your app, use the `YoutubeChannelLoader`. + +## Install Youtube addon + +```bash +npm install @llm-tools/embedjs-loader-youtube +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { YoutubeChannelLoader } from '@llm-tools/embedjs-loader-youtube'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new YoutubeChannelLoader({ youtubeChannelId: '...' })) +``` diff --git a/docs/components/data-sources/youtube-search.mdx b/docs/components/data-sources/youtube-search.mdx new file mode 100644 index 00000000..87041e75 --- /dev/null +++ b/docs/components/data-sources/youtube-search.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ” Youtube Search' +--- + +To automatically perform a youtube search and load the top youtube videos to your app, use the `YoutubeSearchLoader`. + +## Install Youtube addon + +```bash +npm install @llm-tools/embedjs-loader-youtube +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { YoutubeSearchLoader } from '@llm-tools/embedjs-loader-youtube'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new YoutubeSearchLoader({ youtubeSearchString: '...' })) +``` diff --git a/docs/components/data-sources/youtube-video.mdx b/docs/components/data-sources/youtube-video.mdx new file mode 100644 index 00000000..3cea54c0 --- /dev/null +++ b/docs/components/data-sources/youtube-video.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“Ί Youtube Video' +--- + +To add any youtube video to your app, use the `YoutubeLoader`. + +## Install Youtube addon + +```bash +npm install @llm-tools/embedjs-loader-youtube +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; +import { YoutubeLoader } from '@llm-tools/embedjs-loader-youtube'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +app.addLoader(new YoutubeLoader({ videoIdOrUrl: '...' })) +``` diff --git a/docs/components/embeddings/azure-openai.mdx b/docs/components/embeddings/azure-openai.mdx new file mode 100644 index 00000000..c28b5efb --- /dev/null +++ b/docs/components/embeddings/azure-openai.mdx @@ -0,0 +1,24 @@ +--- +title: 'Azure OpenAI' +--- + +In order to be able to use an OpenAI model on Azure, it first needs to be deployed. +Refer to the [Azure OpenAI LLM install documentation](/components/llms/azure-openai) for more information. + +## Install OpenAI addon + +```bash +npm install @llm-tools/embedjs-openai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +``` + diff --git a/docs/components/embeddings/cohere.mdx b/docs/components/embeddings/cohere.mdx new file mode 100644 index 00000000..88568b13 --- /dev/null +++ b/docs/components/embeddings/cohere.mdx @@ -0,0 +1,32 @@ +--- +title: 'Cohere' +--- + +The library supports usage of [Cohere-AI](https://cohere.com) `embed-english-v2.0` embedding model out of the box. This model returns vectors with dimension 4096. + +Here's what you have to do to use it - + +- Sign up for an account with Cohere-AI, go to the [API Keys](https://dashboard.cohere.ai/api-keys) section and copy / create an API_KEY. + +- Set the key in the environment variable `COHERE_API_KEY` + +```bash +COHERE_API_KEY="" +``` + +## Install Cohere addon + +```bash +npm install @llm-tools/embedjs-cohere +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { CohereEmbeddings } from '@llm-tools/embedjs-cohere'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new CohereEmbeddings()) +``` diff --git a/docs/components/embeddings/ollama.mdx b/docs/components/embeddings/ollama.mdx new file mode 100644 index 00000000..018eed54 --- /dev/null +++ b/docs/components/embeddings/ollama.mdx @@ -0,0 +1,27 @@ +--- +title: 'Ollama' +--- + +EmbedJs supports local embeddings via `Ollama`. Read more here [Ollama embeddings](https://ollama.com/blog/embedding-models). + +To use this, you need to setup and have Ollama working locally. Refer to their Github [here](https://github.com/ollama/ollama) to understand how to do this. +Once done, follow these steps - + +## Install Ollama addon + +```bash +npm install @llm-tools/embedjs-ollama +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OllamaEmbeddings } from '@llm-tools/embedjs-ollama'; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OllamaEmbeddings({ + modelName: "...", + baseUrl: 'http://localhost:11434' +})) +``` diff --git a/docs/components/embeddings/openai.mdx b/docs/components/embeddings/openai.mdx new file mode 100644 index 00000000..bc8710c0 --- /dev/null +++ b/docs/components/embeddings/openai.mdx @@ -0,0 +1,36 @@ +--- +title: 'OpenAI' +--- + +To use the OpenAI embedding models, you need a API key from OpenAI. +You can alternatively use Azure OpenAI to run most of the OpenAI embedding models. Read the [Azure OpenAI](/components/llms/azure-openai) section to learn more about this. + +To get started, obtain an API Key from OpenAI. You can do this by visiting their [API Portal](https://platform.openai.com/api-keys). +Once you obtain a key, set it in an environment variable, like so - + +```bash +OPENAI_API_KEY="" +``` + +The EmbedJs OpenAI addon provides the following embedding models - + +- OpenAi3LargeEmbeddings +- OpenAi3SmallEmbeddings +- AdaEmbeddings + +## Install OpenAI addon + +```bash +npm install @llm-tools/embedjs-openai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3LargeEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3LargeEmbeddings()) +``` diff --git a/docs/components/embeddings/overview.mdx b/docs/components/embeddings/overview.mdx new file mode 100644 index 00000000..a7b254a2 --- /dev/null +++ b/docs/components/embeddings/overview.mdx @@ -0,0 +1,17 @@ +--- +title: Overview +--- + +EmbedJs supports several embedding models from the following providers: + + + + + + + + + +
+ + diff --git a/docs/components/embeddings/vertexai.mdx b/docs/components/embeddings/vertexai.mdx new file mode 100644 index 00000000..9e0fb66e --- /dev/null +++ b/docs/components/embeddings/vertexai.mdx @@ -0,0 +1,25 @@ +--- +title: 'Vertex AI' +--- + +The libaray supports the Gecko Embedding model on [VertexAI](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings) with 768 dimensions. + +To use this model, you need to authenticate to Vertex AI on GCP. Refer [here](/components/llms/azure-openai) on how to do this. Once done, follow these steps - + +## Install VertexAi addon + +```bash +npm install @llm-tools/embedjs-vertexai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { GeckoEmbeddings } from '@llm-tools/embedjs-vertexai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new GeckoEmbeddings()) +``` + diff --git a/docs/components/introduction.mdx b/docs/components/introduction.mdx new file mode 100644 index 00000000..c9ddc739 --- /dev/null +++ b/docs/components/introduction.mdx @@ -0,0 +1,13 @@ +--- +title: EmbedJs Components +--- + +## Overview + +You can configure following components in an EmbedJs application - + +* [LLM](/components/llms) +* [Embedding Model](/components/embeddings) +* [Data Source](/components/data-sources) +* [Vector Database](/components/vector-databases) +* [Caches](/components/caches) diff --git a/docs/components/llms/anthropic.mdx b/docs/components/llms/anthropic.mdx new file mode 100644 index 00000000..6e9844d8 --- /dev/null +++ b/docs/components/llms/anthropic.mdx @@ -0,0 +1,26 @@ +--- +title: 'Anthropic' +--- + +To use Anthropic's Claude models, you will need to get an API Key from Anthropic. +You can do this from their [console](https://console.anthropic.com/settings/keys). Once you obtain a key, set it in the environment variable, like so - + +```bash +ANTHROPIC_API_KEY="" +``` + +## Install Anthtopic addon + +```bash +npm install @llm-tools/embedjs-anthropic +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { Anthropic } from '@llm-tools/embedjs-anthropic'; + +const app = await new RAGApplicationBuilder() +.setModel(new Anthropic({ modelName: "..." })) +``` diff --git a/docs/components/llms/azure-openai.mdx b/docs/components/llms/azure-openai.mdx new file mode 100644 index 00000000..d1a35f35 --- /dev/null +++ b/docs/components/llms/azure-openai.mdx @@ -0,0 +1,49 @@ +--- +title: 'Azure OpenAI' +--- + +In order to be able to use an OpenAI model on Azure, it first needs to be deployed. +Please refer to [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/) on how to deploy a model on Azure. +To run this library, you will need to deploy both an LLM and an embedding model. For example - + +- text-embedding-ada (or any of the newer models) +- GPT-3.5-turbo (or the 4 series) + +Once these models are deployed, using Azure OpenAI instead of the regular OpenAI is easy to do. Just follow these steps - + +- Remove the `OPENAI_API_KEY` environment variable if you have set it already. + +- Set the following environment variables - + +```bash +# Set this to `azure` +OPENAI_API_TYPE=azure +# The API version you want to use +AZURE_OPENAI_API_VERSION=2023-05-15 +# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. +export AZURE_OPENAI_BASE_PATH=https://your-resource-name.openai.azure.com/openai/deployments +# The API key1 or key2 for your Azure OpenAI resource +export AZURE_OPENAI_API_KEY= +# The deployment name you used for your embedding model +AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=text-embedding-ada-002 +# The deployment name you used for your llm +AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-35-turbo +``` + +The OpenAi addon can now be used as is and will automatically use the Azure OpenAI endpoint. + +## Install OpenAI addon + +```bash +npm install @llm-tools/embedjs-openai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi } from '@llm-tools/embedjs-openai'; + +const app = await new RAGApplicationBuilder() +.setModel(new OpenAi({ modelName: "gpt-4o" })) +``` diff --git a/docs/components/llms/huggingface.mdx b/docs/components/llms/huggingface.mdx new file mode 100644 index 00000000..dac0c8d5 --- /dev/null +++ b/docs/components/llms/huggingface.mdx @@ -0,0 +1,27 @@ +--- +title: 'Hugging Face' +--- + +Hugging face needs no introduction. They host a variety of open source LLMs and you can use most of them for free. To run hugging face inference based LLMs with this library, you will need a free hugging face token. + +You can get an API Token by signing up for hugging face and generate a token from [this page](https://huggingface.co/settings/tokens). Once you get the token, set it to the environment like so - + +```bash +HUGGINGFACEHUB_API_KEY="" +``` + +## Install Huggingface addon + +```bash +npm install @llm-tools/embedjs-huggingface +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { HuggingFace } from '@llm-tools/embedjs-huggingface'; + +const app = await new RAGApplicationBuilder() +.setModel(new HuggingFace({ modelName: "..." })) +``` diff --git a/docs/components/llms/mistral.mdx b/docs/components/llms/mistral.mdx new file mode 100644 index 00000000..2627477c --- /dev/null +++ b/docs/components/llms/mistral.mdx @@ -0,0 +1,21 @@ +--- +title: 'Mistral' +--- + +To use Mirstal's models, you will need to get an API Key from Mistral. You can do this from their [console](https://console.mistral.ai/user/api-keys/). + +## Install Mistral addon + +```bash +npm install @llm-tools/embedjs-mistral +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { Anthropic } from '@llm-tools/embedjs-mistral'; + +const app = await new RAGApplicationBuilder() +.setModel(new Mistral({ accessToken: "", modelName: "..." })) +``` diff --git a/docs/components/llms/ollama.mdx b/docs/components/llms/ollama.mdx new file mode 100644 index 00000000..5f7ae65c --- /dev/null +++ b/docs/components/llms/ollama.mdx @@ -0,0 +1,25 @@ +--- +title: 'Ollama' +--- + +You can also use locally running Ollama models. Installation instructions for Ollama can be found [here](https://ollama.com/). +Once Ollama is installed, you can start a local LLM by executing `ollama run `. + +## Install Ollama addon + +```bash +npm install @llm-tools/embedjs-ollama +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { Ollama } from '@llm-tools/embedjs-ollama'; + +const app = await new RAGApplicationBuilder() +.setModel(new Ollama({ + modelName: "llama3", + baseUrl: 'http://localhost:11434' +})) +``` diff --git a/docs/components/llms/openai.mdx b/docs/components/llms/openai.mdx new file mode 100644 index 00000000..c06e2356 --- /dev/null +++ b/docs/components/llms/openai.mdx @@ -0,0 +1,29 @@ +--- +title: 'OpenAI' +--- + +To use the OpenAI LLM models, you need a API key from OpenAI. +You can alternatively use Azure OpenAI to run most of the OpenAI LLMs. Read the [Azure OpenAI](/components/llms/azure-openai) section to learn more about this. + +To get started, obtain an API Key from OpenAI. You can do this by visiting their [API Portal](https://platform.openai.com/api-keys). +Once you obtain a key, set it in an environment variable, like so - + +```bash +OPENAI_API_KEY="" +``` + +## Install OpenAI addon + +```bash +npm install @llm-tools/embedjs-openai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi } from '@llm-tools/embedjs-openai'; + +const app = await new RAGApplicationBuilder() +.setModel(new OpenAi({ modelName: "text-davinci-003" })) +``` diff --git a/docs/components/llms/overview.mdx b/docs/components/llms/overview.mdx new file mode 100644 index 00000000..220443b0 --- /dev/null +++ b/docs/components/llms/overview.mdx @@ -0,0 +1,30 @@ +--- +title: Overview +--- + +EmbedJs comes with built-in support for various popular large language models. We handle the complexity of integrating these models for you, allowing you to easily customize your language model interactions through a user-friendly interface. + + + + + + + + + + + +
+ +## Token Usage + +The list of LLMs that support token usage are: + +- OpenAI +- Vertex AI +- Anthropic +- Mistral AI + +
+ + diff --git a/docs/components/llms/vertexai.mdx b/docs/components/llms/vertexai.mdx new file mode 100644 index 00000000..0d7b08f4 --- /dev/null +++ b/docs/components/llms/vertexai.mdx @@ -0,0 +1,38 @@ +--- +title: 'Vertex AI' +--- + +You to use Gemini LLM and other models on Google Cloud Platform via [VertexAI](https://cloud.google.com/vertex-ai?hl=en). +Read more about all the supported [LLMs](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models). + +To get started, you need to set the right access credentials to google cloud. You have two options here - + +- Authenticate by using `gcloud` CLI: + +``` +gcloud auth application-default login +``` + +- Authentication using Service Account with JSON key and environment variable: + +```bash +GOOGLE_APPLICATION_CREDENTIALS="" +``` + +Once this is done, you can configure your embedJs app - + +## Install VertexAi addon + +```bash +npm install @llm-tools/embedjs-vertexai +``` + +## Usage + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { Ollama } from '@llm-tools/embedjs-vertexai'; + +const app = await new RAGApplicationBuilder() +.setModel(new VertexAI({ modelName: 'gemini-1.5-pro-preview-0409'})) +``` diff --git a/docs/components/vector-databases/astradb.mdx b/docs/components/vector-databases/astradb.mdx new file mode 100644 index 00000000..92551667 --- /dev/null +++ b/docs/components/vector-databases/astradb.mdx @@ -0,0 +1,48 @@ +--- +title: AstraDB +--- + +[Astra DB](https://www.datastax.com/products/datastax-astra) is a document database with a highly performant vector index powered by Apache Cassandra and available as a managed service. + +To use Astra DB - + +- [Sign up](https://astra.datastax.com/signup) for an Astra DB account. It is free to sign up and doesn't require a credit card. +- Create a database (this takes a couple of minutes to provision) +- From the database overview page get the API Endpoint and generate an Application Token + +## Install AstraDB addon + +```bash +npm install @llm-tools/embedjs-astradb +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { AstraDb } from '@llm-tools/embedjs-astradb'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setVectorDb( + new AstraDb({ + endpoint: "", + apiKey: "", + collectionName: "documents" + }) +) +.build(); + + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/hnswlib.mdx b/docs/components/vector-databases/hnswlib.mdx new file mode 100644 index 00000000..89f3ef98 --- /dev/null +++ b/docs/components/vector-databases/hnswlib.mdx @@ -0,0 +1,36 @@ +--- +title: HNSWLib +--- + +[HNSWLib](https://github.com/nmslib/hnswlib) is an high performance in-memory vectorstore. +It is great for testing since you do not need access to the file system or a cloud service. + +## Install HNSWLib addon + +```bash +npm install @llm-tools/embedjs-hnswlib +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setVectorDb(new HNSWDb()) +.build(); + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/lancedb.mdx b/docs/components/vector-databases/lancedb.mdx new file mode 100644 index 00000000..9593ac38 --- /dev/null +++ b/docs/components/vector-databases/lancedb.mdx @@ -0,0 +1,42 @@ +--- +title: LanceDB +--- + +LanceDB is a developer-friendly, open source database with great performance. +From hyper scalable vector search and advanced retrieval for RAG, to streaming training data and interactive exploration of large scale AI datasets. +You can run LanceDB locally and it is integrated into the LanceDb addon for EmbedJs. + +## Install LanceDB addon + +```bash +npm install @llm-tools/embedjs-lancedb +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { LanceDb } from '@llm-tools/embedjs-lancedb'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setVectorDb( + new LanceDb({ + path: './lmdb', + }), +) +.build(); + + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/mongodb.mdx b/docs/components/vector-databases/mongodb.mdx new file mode 100644 index 00000000..bd1154c3 --- /dev/null +++ b/docs/components/vector-databases/mongodb.mdx @@ -0,0 +1,53 @@ +--- +title: MongoDB +--- + +[MongoDB](https://www.mongodb.com/products/platform/atlas-vector-search) is an open source document database. +The company behind the open source product offers a managed cloud product **MongoDB Atlas**. +As of right now, only the Atlas version supports vector search while the open source version does not. + +To use MongoDB as your vector database, follow these steps - + +- Sign up for a MongoDB Atlas account if you haven't already. Once you have signed up, you will need to spin up a new cluster (or use an existing one) + + +You will need to provision a M10 (or higher) instance type to use Atlas vector search. Cheaper instance types or the free version (M0) give an error when vector indexes are created programatically. + + +- The cluster creation takes a few minutes. Once the cluster is ready, click on the connect button on the dashboard to get the connection string. + + +You will need to add users separately and allow IP access from your relevant development and production environments. + + +## Install MongoDB addon + +```bash +npm install @llm-tools/embedjs-mongodb +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { MongoDb } from '@llm-tools/embedjs-mongodb'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +new MongoDb({ + connectionString: 'mongodb+srv://:@', +}), +.build(); + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/overview.mdx b/docs/components/vector-databases/overview.mdx new file mode 100644 index 00000000..3c066839 --- /dev/null +++ b/docs/components/vector-databases/overview.mdx @@ -0,0 +1,21 @@ +--- +title: Overview +--- + +## Overview + +Utilizing a vector database alongside EmbedJs is a seamless process. +All you need to do is install the specific addon for the vector database of your choice and configure it. +We've provided examples for each supported database below: + + + + + + + + + + + + diff --git a/docs/components/vector-databases/pinecone.mdx b/docs/components/vector-databases/pinecone.mdx new file mode 100644 index 00000000..9314322a --- /dev/null +++ b/docs/components/vector-databases/pinecone.mdx @@ -0,0 +1,52 @@ +--- +title: Pinecone +--- + +To get started, create an account with [Pinecone](https://www.pinecone.io/) if you don't have one already. There is a free tier. +Once you have an account, get an API key from the **API Keys** section on the Pinecone dashboard. + +Add the API Key to your environment variables. + +```bash +PINECONE_API_KEY= +``` + +## Install Pinecone addon + +```bash +npm install @llm-tools/embedjs-pinecone +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { LanceDb } from '@llm-tools/embedjs-pinecone'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setVectorDb(new PineconeDb({ + projectName: '', + namespace: '', + indexSpec: { + pod: { + podType: 'p1.x1', + environment: 'us-east1-gcp', + }, + }, +})) +.build(); + + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/qdrant.mdx b/docs/components/vector-databases/qdrant.mdx new file mode 100644 index 00000000..9904c5f0 --- /dev/null +++ b/docs/components/vector-databases/qdrant.mdx @@ -0,0 +1,36 @@ +--- +title: Qdrant +--- + +[Qdrant](https://qdrant.tech/) is a Vector Search Engine written in Rust. You will need to sign up to get an API key and URL to use Qdrant. + +## Install Qdrant addon + +```bash +npm install @llm-tools/embedjs-pinecone +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { QdrantDb } from '@llm-tools/embedjs-qdrant'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setVectorDb(new QdrantDb({ apiKey: '...'; url: '...'; clusterName: '...' })) +.build(); + + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/weaviate.mdx b/docs/components/vector-databases/weaviate.mdx new file mode 100644 index 00000000..370d49f1 --- /dev/null +++ b/docs/components/vector-databases/weaviate.mdx @@ -0,0 +1,36 @@ +--- +title: Weaviate +--- + +[Weaviate](https://weaviate.io/) is an open source vector store. You can deploy it locally on docker or use their managed cloud offering. + +## Install Qdrant addon + +```bash +npm install @llm-tools/embedjs-weaviate +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { WeaviateDb } from '@llm-tools/embedjs-weaviate'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new WeaviateDb({ host: '...', apiKey: '...', className: '...', scheme: '...' })) +.build(); + + +//add data source and start query in +await app.add("https://www.forbes.com/profile/elon-musk"); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/contribution/dev.mdx b/docs/contribution/dev.mdx new file mode 100644 index 00000000..07a693ae --- /dev/null +++ b/docs/contribution/dev.mdx @@ -0,0 +1,39 @@ +--- +title: 'πŸ‘¨β€πŸ’» Development' +description: 'Contribute to EmbedJs framework development' +--- + +Thank you for your interest in contributing to the EmbedJs project! We welcome your ideas and contributions to help improve the project. Please follow the instructions below to get started: + +1. **Fork the repository**: Click on the "Fork" button at the top right corner of this repository page. This will create a copy of the repository in your own GitHub account. + +2. **Install the required dependencies**: Ensure that you have the necessary dependencies installed. You can do this by running the following command: + +```bash +npm install +``` + +3. **Make changes in the code**: Create a new branch in your forked repository and make your desired changes in the codebase. + +4. **Create a pull request**: When you are ready to contribute your changes, submit a pull request to the EmbedJs repository. Provide a clear and descriptive title for your pull request, along with a detailed description of the changes you have made. + +## Team + +### Maintainers + +- Adhityan K V ([@kvadhityan](https://twitter.com/kvadhityan)) + +### Citation + +If you utilize this repository, please consider citing it with: + +``` +@misc{embedJs, + author = {Adhityan K V}, + title = {EmbedJs: The Open Source RAG Framework}, + year = {2023}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/llm-tools/embedjs}}, +} +``` diff --git a/docs/contribution/docs.mdx b/docs/contribution/docs.mdx new file mode 100644 index 00000000..2b74a31a --- /dev/null +++ b/docs/contribution/docs.mdx @@ -0,0 +1,28 @@ +--- +title: 'πŸ“ Documentation' +description: 'Contribute to EmbedJs docs' +--- + +Thank you for your interest in contributing to the EmbedJs project documentation! Please follow the instructions below to get started: + +Step 1. Go to the `docs/` directory (where you can find `mint.json`) and run the following command: + +```bash +npx mintlify dev +``` + +The documentation website is now available at `http://localhost:3000`. + +### Custom Ports + +Mintlify uses port 3000 by default. You can use the `--port` flag to customize the port Mintlify runs on. For example, use this command to run in port 3333: + +```bash +mintlify dev --port 3333 +``` + +You will see an error like this if you try to run Mintlify in a port that's already taken: + +```md +Error: listen EADDRINUSE: address already in use :::3000 +``` diff --git a/docs/contribution/guidelines.mdx b/docs/contribution/guidelines.mdx new file mode 100644 index 00000000..68ab5efc --- /dev/null +++ b/docs/contribution/guidelines.mdx @@ -0,0 +1,4 @@ +--- +title: 'πŸ“‹ Guidelines' +url: https://github.com/llm-tools/embedJs/blob/main/CONTRIBUTING.md +--- diff --git a/docs/examples/slack_bot.mdx b/docs/examples/slack_bot.mdx new file mode 100644 index 00000000..78bed2e2 --- /dev/null +++ b/docs/examples/slack_bot.mdx @@ -0,0 +1,36 @@ +--- +title: 'πŸ’Ό Slack Bot' +--- + +### πŸ–ΌοΈ Setup + +1. Create a workspace on Slack if you don't have one already by clicking [here](https://slack.com/intl/en-in/). +2. Create a new App on your Slack account by going [here](https://api.slack.com/apps). +3. Select `From Scratch`, then enter the Bot Name and select your workspace. +4. On the left Sidebar, go to `OAuth and Permissions` and add the following scopes under `Bot Token Scopes`: +```text +app_mentions:read +channels:read +chat:write +``` +5. Now select the option `Install to Workspace` and after it's done, copy the `Bot User OAuth Token` and set it in your secrets as `SLACK_BOT_TOKEN`. +6. Clone the example slack repo from [here](https://github.com/llm-tools/slack-bot-template). +7. Run NPM install and start the server. The application starts and prints by default a `NGROK reverse proxy URL` to the console. +8. On the Slack API website go to `Event Subscriptions` on the left Sidebar and turn on `Enable Events`. +9. In `Request URL`, enter your server or ngrok address. +10. After it gets verified, click on `Subscribe to bot events`, add `message.channels` Bot User Event and click on `Save Changes`. +11. Now go to your workspace, right click on the bot name in the sidebar, click `view app details`, then `add this app to a channel`. + +### πŸš€ Usage Instructions + +- Go to the channel where you have added your bot. +- To add data sources to the bot, use the command: +```text +/learnfrom web +``` +- To ask queries from the bot, just tag the bot and ask your question. +```text +@botname +``` + +πŸŽ‰ Happy Chatting! πŸŽ‰ diff --git a/docs/favicon.png b/docs/favicon.png new file mode 100644 index 00000000..9e25a3d6 Binary files /dev/null and b/docs/favicon.png differ diff --git a/docs/get-started/faq.mdx b/docs/get-started/faq.mdx new file mode 100644 index 00000000..88cdce9c --- /dev/null +++ b/docs/get-started/faq.mdx @@ -0,0 +1,76 @@ +--- +title: ❓ FAQs +description: 'Collections of all the frequently asked questions' +--- + + +Use the model `gpt-4o` provided my openai. + + +```ts index.ts +import 'dotenv/config'; +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { Mistral } from '@llm-tools/embedjs-mistral'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const llmApplication = await new RAGApplicationBuilder() + .setModel(new Mistral({ modelName: 'mistral-medium' })) + .setVectorDb(new HNSWDb()) + .build(); + +await llmApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); + +console.log(await llmApplication.query('Who founded Tesla?')); +``` + + + + + +```ts index.ts +import 'dotenv/config'; +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi, OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const llmApplication = await new RAGApplicationBuilder() + .setModel(new OpenAi({ modelName: 'gpt-4o' })) + .setEmbeddingModel(new OpenAi3SmallEmbeddings()) + .setVectorDb(new HNSWDb()) + .build(); + +await llmApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' })); + +console.log(await llmApplication.query('Who founded Tesla?')); +``` + + + + + +```ts index.ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OllamaEmbeddings } from '@llm-tools/embedjs-ollama'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Ollama({ modelName: "llama3.2", baseUrl: 'http://localhost:11434' })) +.setEmbeddingModel(new OllamaEmbeddings({ model: 'nomic-embed-text', baseUrl: 'http://localhost:11434' })) +.build(); + +ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }) +ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }) + +await ragApplication.query('What is the net worth of Elon Musk today?') +``` + + + + + +#### Still have questions? +If docs aren't sufficient, please feel free to reach out to us using one of the following methods: + + diff --git a/docs/get-started/introduction.mdx b/docs/get-started/introduction.mdx new file mode 100644 index 00000000..b8a4c79a --- /dev/null +++ b/docs/get-started/introduction.mdx @@ -0,0 +1,62 @@ +--- +title: πŸ“š Introduction +--- + +## What is EmbedJs? + +EmbedJs is an Open Source Framework that makes it easy to create and deploy personalized AI apps. At its core, EmbedJs follows the design principle of being _"Conventional but Configurable"_ to serve both software engineers and machine learning engineers. + +EmbedJs streamlines the creation of personalized LLM applications, offering a seamless process for managing various types of unstructured data. It efficiently segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. With a suite of diverse APIs, it enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data. + +## Who is EmbedJs for? + +EmbedJs is designed for a diverse range of users, from AI professionals like Data Scientists and Machine Learning Engineers to those just starting their AI journey, including college students, independent developers, and hobbyists. Essentially, it's for anyone with an interest in AI, regardless of their expertise level. + +Our APIs are user-friendly yet adaptable, enabling beginners to effortlessly create LLM-powered applications with as few as 7 lines of code. At the same time, we offer extensive customization options for every aspect of building a personalized AI application. This includes the choice of LLMs, vector databases, loaders and chunkers, and more. + +Our platform's clear and well-structured abstraction layers ensure that users can tailor the system to meet their specific needs, whether they're crafting a simple project or a complex, nuanced AI application. + +## Why Use EmbedJs? + +Developing a personalized AI application for production use presents numerous complexities, such as: + +- Integrating and indexing data from diverse sources. +- Determining optimal data chunking methods for each source. +- Synchronizing the RAG pipeline with regularly updated data sources. +- Implementing efficient data storage in a vector store. +- Deciding whether to include metadata with document chunks. +- Configuring Large Language Models (LLMs). +- Selecting effective prompts. + +EmbedJs is designed to simplify these tasks, offering conventional yet customizable APIs. Our solution handles the intricate processes of loading, chunking, indexing, and retrieving data. This enables you to concentrate on aspects that are crucial for your specific use case or business objectives, ensuring a smoother and more focused development process. + +## How it works? + +EmbedJs makes it easy to add data to your RAG pipeline with these straightforward steps: + +1. **Automatic Data Handling**: It automatically recognizes the data type and loads it. +2. **Efficient Data Processing**: The system creates embeddings for key parts of your data. +3. **Flexible Data Storage**: You get to choose where to store this processed data in a vector database. + +When a user asks a question, whether for chatting, searching, or querying, EmbedJs simplifies the response process: + +1. **Query Processing**: It turns the user's question into embeddings. +2. **Document Retrieval**: These embeddings are then used to find related documents in the database. +3. **Answer Generation**: The related documents are used by the LLM to craft a precise answer. + +With EmbedJs, you don’t have to worry about the complexities of building a personalized AI application. It offers an easy-to-use interface for developing applications with any kind of data. + +## Getting started + +Checkout our [quickstart guide](/get-started/quickstart) to start your first AI application. + +## Support + +Feel free to reach out to us if you have ideas, feedback or questions that we can help out with. + + + +## Contribute + +- [GitHub](https://github.com/llm-tools/embedjs) +- [Contribution docs](/contribution/dev) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx new file mode 100644 index 00000000..868fa0f3 --- /dev/null +++ b/docs/get-started/quickstart.mdx @@ -0,0 +1,91 @@ +--- +title: '⚑ Quickstart' +description: 'πŸ’‘ Create an AI app on your own data in a minute' +--- + +## Installation + +First install the NodeJs package: + +```bash +npm i @llm-tools/embedjs +``` + +Once you have installed the package, depending upon your preference you can either use: + + + + This includes Open source LLMs like Mistral, Llama, etc. +
+ Free to use, and runs locally on your machine. + + + This includes paid LLMs like GPT 4, Claude, etc. +
+ Cost money and are accessible via an API. + + + +## Open Source Models + +This section provides a quickstart example of using `Llama3.2` as the Open source LLM and `nomic-embed-text` as the Open source embedding model. + +These models are free and can be run locally using Ollama. Read the documentation [here](todo) for more information on getting started with Ollama. + + +```ts ollama_demo.ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OllamaEmbeddings } from '@llm-tools/embedjs-ollama'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Ollama({ modelName: "llama3.2", baseUrl: 'http://localhost:11434' })) +.setEmbeddingModel(new OllamaEmbeddings({ model: 'nomic-embed-text', baseUrl: 'http://localhost:11434' })) +.setVectorDb(new HNSWDb()) +.build(); + +ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }) +ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }) + +await ragApplication.query('What is the net worth of Elon Musk today?') +//Answer: The net worth of Elon Musk today is $258.7 billion. +``` + + +## Paid Models + +In this section, we will use both an LLM and embedding model from OpenAI. + + +```ts openai_demo.ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +//Replace this with your OpenAI key +process.env.OPENAI_API_KEY = "sk-xxxx" + +const ragApplication = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); + +ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }) +ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }) + +await ragApplication.query('What is the net worth of Elon Musk today?') +//Answer: The net worth of Elon Musk today is $258.7 billion. +``` + + +# Next Steps + +Now that you have created your first app, you can follow any of the links: + +- [Introduction](/get-started/introduction) +- [Customization](/components/introduction) +- [Use cases](/use-cases/introduction) +- [Deployment](/get-started/deployment) diff --git a/docs/images/checks-passed.png b/docs/images/checks-passed.png new file mode 100644 index 00000000..3303c773 Binary files /dev/null and b/docs/images/checks-passed.png differ diff --git a/docs/images/langsmith.png b/docs/images/langsmith.png new file mode 100644 index 00000000..5d5ff542 Binary files /dev/null and b/docs/images/langsmith.png differ diff --git a/docs/integration/langsmith.mdx b/docs/integration/langsmith.mdx new file mode 100644 index 00000000..8be12bd0 --- /dev/null +++ b/docs/integration/langsmith.mdx @@ -0,0 +1,60 @@ +--- +title: 'πŸ› οΈ LangSmith' +description: 'Integrate with Langsmith to debug and monitor your LLM app' +--- + +EmbedJs now supports integration with [LangSmith](https://www.langchain.com/langsmith). + +To use LangSmith, you need to do the following steps. + +1. Have an account on LangSmith and keep the environment variables in handy +2. Set the environment variables in your app so that EmbedJs has context about it. +3. Just use EmbedJs and everything will be logged to LangSmith, so that you can better test and monitor your application. + +Let's cover each step in detail. + +* First make sure that you have created a LangSmith account and have all the necessary variables handy. LangSmith has a [good documentation](https://docs.smith.langchain.com/) on how to get started with their service. + +* Once you have setup the account, we will need the following environment variables + +```bash +# Setting environment variable for LangChain Tracing V2 integration. +export LANGCHAIN_TRACING_V2=true + +# Setting the API endpoint for LangChain. +export LANGCHAIN_ENDPOINT=https://api.smith.langchain.com + +# Replace '' with your LangChain API key. +export LANGCHAIN_API_KEY= + +# Replace '' with your LangChain project name, or it defaults to "default". +export LANGCHAIN_PROJECT= # if not specified, defaults to "default" +``` + +* Now create an app using EmbedJs and everything will be automatically visible in LangSmith automatically. + + +```ts +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; + +//Replace this with your OpenAI key +process.env.OPENAI_API_KEY = "sk-xxxx" + +//Build a new application +const ragApplication = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.build(); + +//Add new documents +ragApplication.addLoader('https://www.forbes.com/profile/elon-musk') +ragApplication.addLoader('https://en.wikipedia.org/wiki/Elon_Musk') + +//Query your app +await ragApplication.query('What is the net worth of Elon Musk today?') +``` + +* Now the entire log for this will be visible in langsmith. + + diff --git a/docs/logo/dark.webp b/docs/logo/dark.webp new file mode 100644 index 00000000..51e4c993 Binary files /dev/null and b/docs/logo/dark.webp differ diff --git a/docs/logo/light.webp b/docs/logo/light.webp new file mode 100644 index 00000000..18b36b97 Binary files /dev/null and b/docs/logo/light.webp differ diff --git a/docs/mint.json b/docs/mint.json new file mode 100644 index 00000000..48ff1a12 --- /dev/null +++ b/docs/mint.json @@ -0,0 +1,207 @@ +{ + "$schema": "https://mintlify.com/schema.json", + "name": "EmbedJs", + "logo": { + "dark": "/logo/dark.webp", + "light": "/logo/light.webp", + "href": "https://github.com/llm-tools/embedjs" + }, + "favicon": "/favicon.png", + "colors": { + "primary": "#8B0000", + "light": "#FF7F7F", + "dark": "#8B0000", + "background": { + "dark": "#000", + "light": "#fff" + } + }, + "modeToggle": { + "default": "dark" + }, + "tabs": [ + { + "name": "API Reference", + "url": "api-reference" + }, + { + "name": "Examples", + "url": "examples" + } + ], + "anchors": [], + "topbarLinks": [ + { + "name": "GitHub", + "url": "https://github.com/llm-tools/embedJs" + } + ], + "topbarCtaButton": { + "name": "Install", + "url": "https://www.npmjs.com/package/@llm-tools/embedjs" + }, + "primaryTab": { + "name": "πŸ“˜ Documentation" + }, + "navigation": [ + { + "group": "Get Started", + "pages": ["get-started/introduction", "get-started/quickstart", "get-started/faq"] + }, + { + "group": "Use cases", + "pages": [ + "use-cases/introduction", + "use-cases/chatbots", + "use-cases/question-answering", + "use-cases/semantic-search" + ] + }, + { + "group": "Components", + "pages": [ + "components/introduction", + { + "group": "πŸ—‚οΈ Data sources", + "pages": [ + "components/data-sources/overview", + { + "group": "Built in", + "pages": [ + "components/data-sources/pdf", + "components/data-sources/csv", + "components/data-sources/json", + "components/data-sources/text", + "components/data-sources/web-page", + "components/data-sources/youtube-video", + "components/data-sources/youtube-channel", + "components/data-sources/youtube-search", + "components/data-sources/docx", + "components/data-sources/ppt", + "components/data-sources/excel", + "components/data-sources/sitemap", + "components/data-sources/directory", + "components/data-sources/custom" + ] + } + ] + }, + { + "group": "πŸ—„οΈ Vector databases", + "pages": [ + "components/vector-databases/overview", + { + "group": "Built in", + "pages": [ + "components/vector-databases/lancedb", + "components/vector-databases/mongodb", + "components/vector-databases/pinecone", + "components/vector-databases/hnswlib", + "components/vector-databases/astradb", + "components/vector-databases/qdrant", + "components/vector-databases/weaviate" + ] + } + ] + }, + { + "group": "πŸ€– Large language models", + "pages": [ + "components/llms/overview", + { + "group": "Built in", + "pages": [ + "components/llms/anthropic", + "components/llms/openai", + "components/llms/azure-openai", + "components/llms/huggingface", + "components/llms/mistral", + "components/llms/ollama", + "components/llms/vertexai" + ] + } + ] + }, + { + "group": "🧩 Embedding models", + "pages": [ + "components/embeddings/overview", + { + "group": "Built in", + "pages": [ + "components/embeddings/openai", + "components/embeddings/azure-openai", + "components/embeddings/cohere", + "components/embeddings/ollama", + "components/embeddings/vertexai" + ] + } + ] + }, + { + "group": "⚑ Caches", + "pages": [ + "components/caches/overview", + { + "group": "Built in", + "pages": ["components/caches/lmdb", "components/caches/mongodb", "components/caches/redis"] + } + ] + } + ] + }, + { + "group": "Integrations", + "pages": ["integration/langsmith"] + }, + { + "group": "Chatbots", + "pages": ["examples/slack_bot"] + }, + { + "group": "API Reference", + "pages": [ + "api-reference/overview", + { + "group": "RAGApplication methods", + "pages": [ + "api-reference/methods/query", + "api-reference/methods/search", + "api-reference/methods/add-loader", + "api-reference/methods/get-loaders", + "api-reference/methods/delete-loader", + "api-reference/methods/reset" + ] + } + ] + }, + { + "group": "Contributing", + "pages": ["contribution/guidelines", "contribution/dev", "contribution/docs"] + }, + { + "group": "Product", + "pages": ["product/release-notes"] + } + ], + "footerSocials": { + "website": "https://ragkit.ai", + "github": "https://github.com/llm-tools/embedJs", + "linkedin": "https://www.linkedin.com/in/adhityan/" + }, + "isWhiteLabeled": true, + "feedback": { + "suggestEdit": true, + "raiseIssue": true, + "thumbsRating": true + }, + "search": { + "prompt": "✨ Search embedJs docs..." + }, + "redirects": [ + { + "source": "/changelog/command-line", + "destination": "/get-started/introduction" + } + ] +} diff --git a/docs/product/release-notes.mdx b/docs/product/release-notes.mdx new file mode 100644 index 00000000..974fc382 --- /dev/null +++ b/docs/product/release-notes.mdx @@ -0,0 +1,4 @@ +--- +title: ' πŸ“œ Release Notes' +url: https://github.com/llm-tools/embedJs/releases +--- diff --git a/docs/use-cases/chatbots.mdx b/docs/use-cases/chatbots.mdx new file mode 100644 index 00000000..3b203be2 --- /dev/null +++ b/docs/use-cases/chatbots.mdx @@ -0,0 +1,23 @@ +--- +title: 'πŸ€– Chatbots' +--- + +Chatbots, especially those powered by Large Language Models (LLMs), have a wide range of use cases, significantly enhancing various aspects of business, education, and personal assistance. Here are some key applications: + +- **Customer Service**: Automating responses to common queries and providing 24/7 support. +- **Education**: Offering personalized tutoring and learning assistance. +- **E-commerce**: Assisting in product discovery, recommendations, and transactions. +- **Content Management**: Aiding in writing, summarizing, and organizing content. +- **Data Analysis**: Extracting insights from large datasets. +- **Language Translation**: Providing real-time multilingual support. +- **Mental Health**: Offering preliminary mental health support and conversation. +- **Entertainment**: Engaging users with games, quizzes, and humorous chats. +- **Accessibility Aid**: Enhancing information and service access for individuals with disabilities. + +EmbedJs provides the right set of tools to create chatbots for the above use cases. Refer to the following examples of chatbots on and you can built on top of these examples: + + + + Enhance your Slack workspace with a specialized bot. + + diff --git a/docs/use-cases/introduction.mdx b/docs/use-cases/introduction.mdx new file mode 100644 index 00000000..a6e023c9 --- /dev/null +++ b/docs/use-cases/introduction.mdx @@ -0,0 +1,11 @@ +--- +title: 🧱 Introduction +--- + +## Overview + +You can use EmbedJs to create the following usecases: + +* [Chatbots](/use-cases/chatbots) +* [Question Answering](/use-cases/question-answering) +* [Semantic Search](/use-cases/semantic-search) diff --git a/docs/use-cases/question-answering.mdx b/docs/use-cases/question-answering.mdx new file mode 100644 index 00000000..c25656b1 --- /dev/null +++ b/docs/use-cases/question-answering.mdx @@ -0,0 +1,72 @@ +--- +title: '❓ Question Answering' +--- + +Utilizing large language models (LLMs) for question answering is a transformative application, bringing significant benefits to various real-world situations. EmbedJs extensively supports tasks related to question answering, including summarization, content creation, language translation, and data analysis. The versatility of question answering with LLMs enables solutions for numerous practical applications such as: + +- **Educational Aid**: Enhancing learning experiences and aiding with homework +- **Customer Support**: Addressing and resolving customer queries efficiently +- **Research Assistance**: Facilitating academic and professional research endeavors +- **Healthcare Information**: Providing fundamental medical knowledge +- **Technical Support**: Resolving technology-related inquiries +- **Legal Information**: Offering basic legal advice and information +- **Business Insights**: Delivering market analysis and strategic business advice +- **Language Learning** Assistance: Aiding in understanding and translating languages +- **Travel Guidance**: Supplying information on travel and hospitality +- **Content Development**: Assisting authors and creators with research and idea generation + +## Example: Build a Q&A System with EmbedJs for Next.JS + +Quickly create a RAG pipeline to answer queries about the [Next.JS Framework](https://nextjs.org/) using EmbedJs tools. + +### Step 1: Set Up Your RAG Pipeline + +First, let's create your RAG pipeline. Open your NodeJs application and add the following code: + +```ts Create pipeline +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); +``` + +This initializes your application. + +### Step 2: Populate Your Pipeline with Data + +Now, let's add data to your pipeline. We'll include the Next.JS website and its documentation: + +```ts Ingest data sources +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +//Add Next.JS Website and docs +app.addLoader(new SitemapLoader({ url: "https://nextjs.org/sitemap.xml" })) + +//Add Next.JS Forum data +app.addLoader(new SitemapLoader({ url: "https://nextjs-forum.com/sitemap.xml" })) +``` + +This step incorporates over **15K pages** from the Next.JS website and forum into your pipeline. For more data source options, check the [EmbedJs data sources overview](/components/data-sources/overview). + +### Step 3: Local Testing of Your Pipeline + +Test the pipeline on your local machine: + +```ts Query App +app.query("Summarize the features of Next.js 14?") +``` + +Run this query to see how your pipeline responds with information about Next.js 14. + +## Need help? + +If you are looking to configure the RAG pipeline further, feel free to checkout the [API reference](/api-reference/pipeline/query). + +In case you run into issues, feel free to contact us via any of the following methods: + + diff --git a/docs/use-cases/semantic-search.mdx b/docs/use-cases/semantic-search.mdx new file mode 100644 index 00000000..04c6e500 --- /dev/null +++ b/docs/use-cases/semantic-search.mdx @@ -0,0 +1,104 @@ +--- +title: 'πŸ” Semantic Search' +--- + +Semantic searching, which involves understanding the intent and contextual meaning behind search queries, is yet another popular use-case of RAG. It has several popular use cases across various domains: + +- **Information Retrieval**: Enhances search accuracy in databases and websites +- **E-commerce**: Improves product discovery in online shopping +- **Customer Support**: Powers smarter chatbots for effective responses +- **Content Discovery**: Aids in finding relevant media content +- **Knowledge Management**: Streamlines document and data retrieval in enterprises +- **Healthcare**: Facilitates medical research and literature search +- **Legal Research**: Assists in legal document and case law search +- **Academic Research**: Aids in academic paper discovery +- **Language Processing**: Enables multilingual search capabilities + +EmbedJs offers a simple yet customizable `search()` API that you can use for semantic search. See the example in the next section to know more. + +## Example: Semantic Search over Next.JS Website + Forum + +### Step 1: Set Up Your RAG Pipeline + +First, let's create your RAG pipeline. Open your NodeJs application and add the following code: + +```ts Create pipeline +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai'; +import { HNSWDb } from '@llm-tools/embedjs-hnswlib'; + +const app = await new RAGApplicationBuilder() +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.setEmbeddingModel(new OpenAi3SmallEmbeddings()) +.setVectorDb(new HNSWDb()) +.build(); +``` + +This initializes your application. + +### Step 2: Populate Your Pipeline with Data + +Now, let's add data to your pipeline. We'll include the Next.JS website and its documentation: + +```ts Ingest data sources +import { SitemapLoader } from '@llm-tools/embedjs-loader-sitemap'; + +//Add Next.JS Website and docs +app.addLoader(new SitemapLoader({ url: "https://nextjs.org/sitemap.xml" })) + +//Add Next.JS Forum data +app.addLoader(new SitemapLoader({ url: "https://nextjs-forum.com/sitemap.xml" })) +``` + +This step incorporates over **15K pages** from the Next.JS website and forum into your pipeline. For more data source options, check the [EmbedJs data sources overview](/components/data-sources/overview). + +### Step 3: Local Testing of Your Pipeline + +Test the pipeline on your local machine: + +```ts Search App +app.search("Summarize the features of Next.js 14?") +/*[ + { + 'score': 0.99, + 'pageContent': 'Next.js 14 | Next.jsBack to BlogThursday, October 26th 2023Next.js 14Posted byLee Robinson@leeerobTim Neutkens@timneutkensAs we announced at Next.js Conf, Next.js 14 is our most focused release with: Turbopack: 5,000 tests passing for App & Pages Router 53% faster local server startup 94% faster code updates with Fast Refresh Server Actions (Stable): Progressively enhanced mutations Integrated with caching & revalidating Simple function calls, or works natively with forms Partial Prerendering', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5af', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ac', + 'source': 'https://nextjs.org/blog/next-14' + } + }, + { + 'score': 0.98, + 'pageContent': 'Next.js 13.3 | Next.jsBack to BlogThursday, April 6th 2023Next.js 13.3Posted byDelba de Oliveira@delba_oliveiraTim Neutkens@timneutkensNext.js 13.3 adds popular community-requested features, including: File-Based Metadata API: Dynamically generate sitemaps, robots, favicons, and more. Dynamic Open Graph Images: Generate OG images using JSX, HTML, and CSS. Static Export for App Router: Static / Single-Page Application (SPA) support for Server Components. Parallel Routes and Interception: Advanced', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5a1', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ae', + 'source': 'https://nextjs.org/blog/next-13-3' + } + }, + { + 'score': 0.98, + 'pageContent': 'Upgrading: Version 14 | Next.js MenuUsing App RouterFeatures available in /appApp Router.UpgradingVersion 14Version 14 Upgrading from 13 to 14 To update to Next.js version 14, run the following command using your preferred package manager: Terminalnpm i next@latest react@latest react-dom@latest eslint-config-next@latest Terminalyarn add next@latest react@latest react-dom@latest eslint-config-next@latest Terminalpnpm up next react react-dom eslint-config-next -latest Terminalbun add next@latest', + 'metadata': { + 'id': '6c8d1a7b-ea34-4927-8823-daa29dcfc5a2', + 'uniqueLoaderId': '6c8d1a7b-ea34-4927-8823-xba29dcfc5ad', + 'source': 'https://nextjs.org/docs/app/building-your-application/upgrading/version-14' + } + } +]*/ +``` +The `source` key contains the url of the document that yielded that document chunk. + +If you are interested in configuring the search further, refer to our [API documentation](/api-reference/pipeline/search). + +---- + +This guide will help you swiftly set up a semantic search pipeline with EmbedJs, making it easier to access and analyze specific information from large data sources. + + +## Need help? + +In case you run into issues, feel free to contact us via any of the following methods: + + diff --git a/package-lock.json b/package-lock.json index bf712531..22255318 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,11 +47,11 @@ }, "core/embedjs": { "name": "@llm-tools/embedjs", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "axios": "^1.7.7", "debug": "^4.3.7", "langchain": "^0.3.2", @@ -67,7 +67,7 @@ }, "core/embedjs-interfaces": { "name": "@llm-tools/embedjs-interfaces", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "debug": "^4.3.7", @@ -83,87 +83,87 @@ }, "core/embedjs-utils": { "name": "@llm-tools/embedjs-utils", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2" + "@llm-tools/embedjs-interfaces": "0.1.3" } }, "databases/embedjs-astra": { "name": "@llm-tools/embedjs-astradb", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@datastax/astra-db-ts": "^1.5.0", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "databases/embedjs-cosmos": { "name": "@llm-tools/embedjs-cosmos", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@azure/cosmos": "^4.1.1", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "databases/embedjs-hnswlib": { "name": "@llm-tools/embedjs-hnswlib", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7", "hnswlib-node": "^3.0.0" } }, "databases/embedjs-lancedb": { "name": "@llm-tools/embedjs-lancedb", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@lancedb/lancedb": "^0.10.0", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "compute-cosine-similarity": "^1.1.0" } }, "databases/embedjs-lmdb": { "name": "@llm-tools/embedjs-lmdb", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "lmdb": "^3.1.3" } }, "databases/embedjs-mongodb": { "name": "@llm-tools/embedjs-mongodb", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7", "mongodb": "^6.9.0" } }, "databases/embedjs-pinecone": { "name": "@llm-tools/embedjs-pinecone", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "@pinecone-database/pinecone": "^3.0.3", "debug": "^4.3.7" } }, "databases/embedjs-qdrant": { "name": "@llm-tools/embedjs-qdrant", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "@qdrant/js-client-rest": "^1.11.0", "debug": "^4.3.7", "uuid": "^10.0.0" @@ -171,19 +171,19 @@ }, "databases/embedjs-redis": { "name": "@llm-tools/embedjs-redis", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "ioredis": "^5.4.1" } }, "databases/embedjs-weaviate": { "name": "@llm-tools/embedjs-weaviate", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7", "weaviate-ts-client": "^2.2.0" @@ -191,11 +191,11 @@ }, "loaders/embedjs-loader-confluence": { "name": "@llm-tools/embedjs-loader-confluence", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-loader-web": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-loader-web": "0.1.3", "confluence.js": "^1.7.4", "debug": "^4.3.7", "md5": "^2.3.0" @@ -203,11 +203,11 @@ }, "loaders/embedjs-loader-csv": { "name": "@llm-tools/embedjs-loader-csv", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "axios": "^1.7.7", "csv-parse": "^5.5.6", "debug": "^4.3.7", @@ -216,11 +216,11 @@ }, "loaders/embedjs-loader-msoffice": { "name": "@llm-tools/embedjs-loader-msoffice", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "langchain": "^0.3.2", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" @@ -228,11 +228,11 @@ }, "loaders/embedjs-loader-pdf": { "name": "@llm-tools/embedjs-loader-pdf", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "langchain": "^0.3.2", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" @@ -240,11 +240,11 @@ }, "loaders/embedjs-loader-sitemap": { "name": "@llm-tools/embedjs-loader-sitemap", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-loader-web": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-loader-web": "0.1.3", "debug": "^4.3.7", "md5": "^2.3.0", "sitemapper": "^3.2.12" @@ -252,11 +252,11 @@ }, "loaders/embedjs-loader-web": { "name": "@llm-tools/embedjs-loader-web", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "axios": "^1.7.7", "debug": "^4.3.7", "html-to-text": "^9.0.5", @@ -269,11 +269,11 @@ }, "loaders/embedjs-loader-youtube": { "name": "@llm-tools/embedjs-loader-youtube", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.2", - "@llm-tools/embedjs-utils": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", + "@llm-tools/embedjs-utils": "0.1.3", "debug": "^4.3.7", "md5": "^2.3.0", "usetube": "^2.2.7", @@ -285,33 +285,33 @@ }, "models/embedjs-anthropic": { "name": "@llm-tools/embedjs-anthropic", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/anthropic": "^0.3.3", "@langchain/core": "^0.3.6", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "models/embedjs-cohere": { "name": "@llm-tools/embedjs-cohere", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/cohere": "^0.3.0", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "cohere-ai": "^7.14.0" } }, "models/embedjs-huggingface": { "name": "@llm-tools/embedjs-huggingface", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@huggingface/inference": "^2.8.1", "@langchain/community": "^0.3.4", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, @@ -831,45 +831,45 @@ }, "models/embedjs-mistral": { "name": "@llm-tools/embedjs-mistral", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.6", "@langchain/mistralai": "^0.1.1", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "models/embedjs-ollama": { "name": "@llm-tools/embedjs-ollama", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.6", "@langchain/ollama": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "models/embedjs-openai": { "name": "@llm-tools/embedjs-openai", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.6", "@langchain/openai": "^0.3.5", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, "models/embedjs-vertexai": { "name": "@llm-tools/embedjs-vertexai", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.6", "@langchain/google-vertexai": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.2", + "@llm-tools/embedjs-interfaces": "0.1.3", "debug": "^4.3.7" } }, @@ -11901,7 +11901,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">= 4" @@ -16182,7 +16182,6 @@ "version": "5.6.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc",