feat: add image to text interface

toeverything · Apr 2, 2024 · 9a3a73d · 9a3a73d
1 parent 71a9fc3
commit 9a3a73d
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts
@@ -5,6 +5,7 @@ import { ClientOptions, OpenAI } from 'openai';
 import {
  ChatMessageRole,
  CopilotCapability,
+ CopilotImageToTextProvider,
  CopilotProviderType,
  CopilotTextToEmbeddingProvider,
  CopilotTextToImageProvider,
@@ -19,13 +20,15 @@ export class OpenAIProvider
  implements
  CopilotTextToTextProvider,
  CopilotTextToEmbeddingProvider,
- CopilotTextToImageProvider
+ CopilotTextToImageProvider,
+ CopilotImageToTextProvider
 {
  static readonly type = CopilotProviderType.OpenAI;
  static readonly capabilities = [
  CopilotCapability.TextToText,
  CopilotCapability.TextToEmbedding,
  CopilotCapability.TextToImage,
+ CopilotCapability.ImageToText,
  ];
 
  readonly availableModels = [

diff --git a/packages/backend/server/src/plugins/copilot/types.ts b/packages/backend/server/src/plugins/copilot/types.ts
@@ -135,6 +135,7 @@ export enum CopilotCapability {
  TextToEmbedding = 'text-to-embedding',
  TextToImage = 'text-to-image',
  ImageToImage = 'image-to-image',
+ ImageToText = 'image-to-text',
 }
 
 export interface CopilotProvider {
@@ -195,11 +196,35 @@ export interface CopilotTextToImageProvider extends CopilotProvider {
  ): AsyncIterable<string>;
 }
 
+export interface CopilotImageToTextProvider extends CopilotProvider {
+ generateText(
+ messages: PromptMessage[],
+ model: string,
+ options: {
+ temperature?: number;
+ maxTokens?: number;
+ signal?: AbortSignal;
+ user?: string;
+ }
+ ): Promise<string>;
+ generateTextStream(
+ messages: PromptMessage[],
+ model: string,
+ options: {
+ temperature?: number;
+ maxTokens?: number;
+ signal?: AbortSignal;
+ user?: string;
+ }
+ ): AsyncIterable<string>;
+}
+
 export interface CopilotImageToImageProvider extends CopilotProvider {}
 
 export type CapabilityToCopilotProvider = {
  [CopilotCapability.TextToText]: CopilotTextToTextProvider;
  [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
  [CopilotCapability.TextToImage]: CopilotTextToImageProvider;
+ [CopilotCapability.ImageToText]: CopilotImageToTextProvider;
  [CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
 };