increase cosine similarity threshold to 0.55

omnivore-app · May 7, 2024 · 994e66b · 994e66b
1 parent a7760f1
commit 994e66b
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 10 deletions.
diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
@@ -138,8 +138,9 @@ export class LibraryItem {
  @Column('text')
  readableContent!: string
 
+ // typeorm does not support vector type, so we store it as a string
  @Column('text')
- embedding?: number[]
+ embedding?: string
 
  @Column('text', { nullable: true })
  textContentHash?: string | null

diff --git a/packages/api/src/jobs/ai/update_embedding.ts b/packages/api/src/jobs/ai/update_embedding.ts
@@ -44,7 +44,7 @@ export const updateEmbedding = async (data: UpdateEmbeddingJobData) => {
 
  await updateLibraryItem(
  libraryItemId,
- { embedding: `[${embedding.toString()}]` },
+ { embedding: JSON.stringify(embedding) },
  userId,
  undefined,
  true

diff --git a/packages/api/src/jobs/save_page.ts b/packages/api/src/jobs/save_page.ts
@@ -270,13 +270,15 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
  throw new Error(result.message || result.errorCodes[0])
  }
 
- await enqueueUpdateEmbedding({
- libraryItemId: result.clientRequestId,
- userId,
- })
+ if (result.__typename === 'SaveSuccess') {
+ await enqueueUpdateEmbedding({
+ libraryItemId: result.clientRequestId,
+ userId,
+ })
 
- isImported = true
- isSaved = true
+ isImported = true
+ isSaved = true
+ }
  } catch (e) {
  logError(e)
 

diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
@@ -17,7 +17,10 @@ import { env } from './env'
 import { TaskState } from './generated/graphql'
 import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
 import { createDigest, CREATE_DIGEST_JOB } from './jobs/ai/create_digest'
-import { UDPATE_EMBEDDING_JOB_NAME, updateEmbedding } from './jobs/ai/update_embedding'
+import {
+ UDPATE_EMBEDDING_JOB_NAME,
+ updateEmbedding,
+} from './jobs/ai/update_embedding'
 import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
 import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
 import {

diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
@@ -563,7 +563,7 @@ export const buildQueryString = (
 
  orders.push({ by: alias, order: SortOrder.DESCENDING })
 
- return escapeQueryWithParameters(`${similarity} > 0.5`, {
+ return escapeQueryWithParameters(`${similarity} > 0.55`, {
  [param]: value,
  })
  }