Merge pull request #159 from BingKui/feature/llama-cpp

Add Node LLama Cpp Support

docs/components/embeddings/llama-cpp.mdx

@@ -0,0 +1,23 @@
+---
+title: 'LlamaCpp'
+---
+
+EmbedJs supports local embeddings via `LlamaCpp`.
+
+## Install LlamaCpp addon
+
+```bash
+npm install @llm-tools/embedjs-llama-cpp
+```
+
+## Usage
+
+```ts
+import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { LlamaCppEmbeddings } from '@llm-tools/embedjs-llama-cpp';
+
+const app = await new RAGApplicationBuilder()
+.setEmbeddingModel(new LlamaCppEmbeddings({
+    modelPath: "./models/nomic-embed-text-v1.5.f16.gguf",
+}))
+```

docs/components/embeddings/overview.mdx

@@ -11,6 +11,7 @@ EmbedJs supports several embedding models from the following providers:
     <Card title="Cohere" href="/components/embeddings/cohere" />
     <Card title="Huggingface" href="/components/embeddings/huggingface" />
     <Card title="Ollama" href="/components/embeddings/ollama" />
+    <Card title="LlamaCpp" href="/components/embeddings/llama-cpp" />
 </CardGroup>
 
 <br/ >

docs/components/llms/llama-cpp.mdx

@@ -0,0 +1,23 @@
+---
+title: 'LlamaCpp'
+---
+
+You can also use locally running models by `node-llama-cpp`.
+
+## Install LlamaCpp addon
+
+```bash
+npm install @llm-tools/embedjs-llama-cpp
+```
+
+## Usage
+
+```ts
+import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { LlamaCpp } from '@llm-tools/embedjs-llama-cpp';
+
+const app = await new RAGApplicationBuilder()
+.setModel(new LlamaCpp({
+    modelPath: "../models/llama-3.1-8b-instruct-hf-q4_k_m.gguf",
+}))
+```

docs/components/llms/overview.mdx

@@ -9,6 +9,7 @@ EmbedJs comes with built-in support for various popular large language models. W
     <Card title="Azure OpenAI" href="/components/llms/azure-openai" />
     <Card title="Anthropic" href="/components/llms/anthropic" />
     <Card title="Ollama" href="/components/llms/ollama" />
+    <Card title="LlamaCpp" href="/components/llms/llama-cpp" />
     <Card title="Hugging Face" href="/components/llms/huggingface" />
     <Card title="Vertex AI" href="/components/llms/vertexai" />
     <Card title="Mistral AI" href="/components/llms/mistral" />

models/embedjs-llama-cpp/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2024-10-31)
+
+Complate LlamaCppModel & LlamaCppEmbeddings.

models/embedjs-llama-cpp/README.md

@@ -0,0 +1,8 @@
+# embedjs-llama-cpp
+
+<p>
+<a href="https://www.npmjs.com/package/@llm-tools/embedjs"  target="_blank"><img alt="NPM Version" src="https://img.shields.io/npm/v/%40llm-tools/embedjs?style=for-the-badge"></a>
+<a href="https://www.npmjs.com/package/@llm-tools/embedjs"  target="_blank"><img alt="License" src="https://img.shields.io/npm/l/%40llm-tools%2Fembedjs?style=for-the-badge"></a>
+</p>
+
+This package extends and offers additional functionality to [embedJs](https://www.npmjs.com/package/@llm-tools/embedjs). Refer to the documentation there for more details.

models/embedjs-llama-cpp/eslint.config.js

@@ -0,0 +1,20 @@
+import baseConfig from '../../eslint.config.js';
+import parser from '@nx/eslint-plugin';
+
+export default [
+    ...baseConfig,
+    {
+        files: ['**/*.json'],
+        rules: {
+            '@nx/dependency-checks': [
+                'error',
+                {
+                    ignoredFiles: ['{projectRoot}/eslint.config.{js,cjs,mjs}'],
+                },
+            ],
+        },
+        languageOptions: {
+            parser,
+        },
+    },
+];

models/embedjs-llama-cpp/package.json

@@ -0,0 +1,42 @@
+{
+    "name": "@llm-tools/embedjs-llama-cpp",
+    "version": "0.0.1",
+    "description": "Enable usage of Node-Llama-Cpp with embedjs",
+    "dependencies": {
+        "@langchain/community": "^0.3.11",
+        "@langchain/core": "^0.3.15",
+        "@llm-tools/embedjs-interfaces": "0.1.13",
+        "debug": "^4.3.7",
+        "node-llama-cpp": "^3.2.0"
+    },
+    "type": "module",
+    "main": "./src/index.js",
+    "license": "Apache-2.0",
+    "publishConfig": {
+        "access": "public"
+    },
+    "keywords": [
+        "node-llama-cpp",
+        "llm",
+        "ai",
+        "gpt3",
+        "chain",
+        "prompt",
+        "prompt engineering",
+        "chatgpt",
+        "machine learning",
+        "ml",
+        "anthropic",
+        "embeddings",
+        "vectorstores"
+    ],
+    "author": "BingKui",
+    "bugs": {
+        "url": "https://github.com/llm-tools/embedjs/issues"
+    },
+    "homepage": "https://github.com/llm-tools/embedjs#readme",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/llm-tools/embedjs.git"
+    }
+}

models/embedjs-llama-cpp/project.json

@@ -0,0 +1,29 @@
+{
+    "name": "embedjs-llama-cpp",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "models/embedjs-llama-cpp/src",
+    "projectType": "library",
+    "tags": [],
+    "targets": {
+        "build": {
+            "executor": "@nx/js:tsc",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/esm/embedjs-llama-cpp",
+                "main": "models/embedjs-llama-cpp/src/index.ts",
+                "tsConfig": "models/embedjs-llama-cpp/tsconfig.lib.json",
+                "assets": ["models/embedjs-llama-cpp/*.md"]
+            }
+        },
+        "build-cjs": {
+            "executor": "@nx/js:tsc",
+            "outputs": ["{options.outputPath}"],
+            "dependsOn": ["^build-cjs"],
+            "options": {
+                "outputPath": "dist/cjs/embedjs-llama-cpp",
+                "main": "models/embedjs-llama-cpp/src/index.ts",
+                "tsConfig": "models/embedjs-llama-cpp/tsconfig.cjs.json"
+            }
+        }
+    }
+}

models/embedjs-llama-cpp/src/index.ts

@@ -0,0 +1,2 @@
+export * from './llama-cpp-model.js';
+export * from './llama-cpp-embeddings.js';

models/embedjs-llama-cpp/src/llama-cpp-embeddings.ts

@@ -0,0 +1,33 @@
+import { BaseEmbeddings } from '@llm-tools/embedjs-interfaces';
+import { getLlama, Llama, LlamaEmbedding, LlamaEmbeddingContext, LlamaModel } from 'node-llama-cpp';
+
+export class LlamaCppEmbeddings implements BaseEmbeddings {
+    private context: LlamaEmbeddingContext;
+
+    constructor(options: { modelPath: string; }) {
+        getLlama().then((llama: Llama) => {
+            llama.loadModel({ modelPath: options.modelPath }).then((model: LlamaModel) => {
+                model.createEmbeddingContext().then((context: LlamaEmbeddingContext) => {
+                    this.context = context;
+                });
+            });
+        });
+    }
+    async getDimensions(): Promise<number> {
+        const sample = await this.embedDocuments(['sample']);
+        return sample[0].length;
+    }
+
+    async embedDocuments(texts: string[]): Promise<number[][]> {
+        const embeddings = new Map<string, LlamaEmbedding>();
+        await Promise.all(texts.map(async (document) => {
+            const embedding = await this.context.getEmbeddingFor(document);
+            embeddings.set(document, embedding);
+        }));
+        return Array.from(embeddings).map(([_, embedding]) => embedding.vector as number[]);
+    }
+
+    async embedQuery(text: string): Promise<number[]> {
+        return (await this.context.getEmbeddingFor(text)).vector as number[];
+    }
+}

models/embedjs-llama-cpp/src/llama-cpp-model.ts

@@ -0,0 +1,26 @@
+import createDebugMessages from 'debug';
+import { AIMessage, HumanMessage, SystemMessage } from '@langchain/core/messages';
+import { BaseModel, ModelResponse } from '@llm-tools/embedjs-interfaces';
+import { LlamaCpp as ChatLlamaCpp } from '@langchain/community/llms/llama_cpp';
+
+export class LlamaCpp extends BaseModel {
+    private readonly debug = createDebugMessages('embedjs:model:LlamaCpp');
+    private model: ChatLlamaCpp;
+
+    constructor({ temperature, modelPath }: { temperature?: number; modelPath: string }) {
+        super(temperature);
+        this.model = new ChatLlamaCpp({
+            modelPath: modelPath ?? '',
+        });
+    }
+
+    override async runQuery(messages: (AIMessage | SystemMessage | HumanMessage)[]): Promise<ModelResponse> {
+        this.debug(`Executing LlamaCpp model ${this.model} with prompt -`, messages[messages.length - 1].content);
+        const result = await this.model.invoke(messages);
+        this.debug('LlamaCpp response -', result);
+
+        return {
+            result: result.toString(),
+        };
+    }
+}

models/embedjs-llama-cpp/tsconfig.cjs.json

@@ -0,0 +1,7 @@
+{
+    "extends": "./tsconfig.lib.json",
+    "compilerOptions": {
+        "module": "commonjs",
+        "moduleResolution": "Node10"
+    }
+}

models/embedjs-llama-cpp/tsconfig.json

@@ -0,0 +1,26 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "compilerOptions": {
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
+        "module": "NodeNext",
+        "moduleResolution": "nodenext",
+        "esModuleInterop": true,
+        "declaration": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        "noUnusedLocals": true,
+        "noUnusedParameters": true,
+        "useDefineForClassFields": true,
+        "strictPropertyInitialization": false,
+        "allowJs": false,
+        "strict": false
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        }
+    ]
+}

models/embedjs-llama-cpp/tsconfig.lib.json

@@ -0,0 +1,10 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "declaration": true,
+        "types": ["node"]
+    },
+    "include": ["src/**/*.ts"],
+    "exclude": ["src/**/*.spec.ts", "src/**/*.test.ts"]
+}