From 5d479e28ce0a44b9e2e1511b7cb1b55307991729 Mon Sep 17 00:00:00 2001 From: tea artist Date: Thu, 2 Jan 2025 16:59:11 +0800 Subject: [PATCH] feat: add integrity check --- apps/nestjs-backend/src/app.module.ts | 2 + .../src/db-provider/db.provider.interface.ts | 6 +- .../src/db-provider/postgres.provider.ts | 23 +- .../src/db-provider/sqlite.provider.ts | 23 +- .../features/calculation/reference.service.ts | 12 + .../integrity/integrity.controller.ts | 23 ++ .../features/integrity/integrity.module.ts | 10 + .../integrity/link-integrity.service.ts | 359 ++++++++++++++++++ .../table/open-api/table-open-api.service.ts | 6 +- apps/nestjs-backend/test/link-api.e2e-spec.ts | 6 +- apps/nextjs-app/.env.development | 2 +- .../prisma/postgres/schema.prisma | 6 + .../prisma/sqlite/schema.prisma | 6 + .../db-main-prisma/prisma/template.prisma | 6 + packages/openapi/src/access-token/refresh.ts | 2 +- packages/openapi/src/auth/add-password.ts | 2 +- packages/openapi/src/auth/reset-password.ts | 2 +- .../src/auth/send-reset-password-email.ts | 2 +- packages/openapi/src/comment/create.ts | 2 +- .../src/comment/reaction/create-reaction.ts | 2 +- .../src/comment/subscribe/create-subscribe.ts | 2 +- packages/openapi/src/dashboard/create.ts | 2 +- packages/openapi/src/import/import-table.ts | 2 +- packages/openapi/src/index.ts | 1 + packages/openapi/src/integrity/index.ts | 2 + packages/openapi/src/integrity/link-check.ts | 74 ++++ packages/openapi/src/integrity/link-fix.ts | 31 ++ packages/openapi/src/oauth/revoke.ts | 2 +- packages/openapi/src/plugin/get-auth-code.ts | 2 +- packages/openapi/src/plugin/refresh-token.ts | 2 +- .../openapi/src/plugin/regenerate-secret.ts | 2 +- packages/openapi/src/share/view-auth.ts | 2 +- packages/openapi/src/table/index.ts | 1 - packages/openapi/src/table/sql-query.ts | 45 --- 34 files changed, 603 insertions(+), 69 deletions(-) create mode 100644 apps/nestjs-backend/src/features/integrity/integrity.controller.ts create mode 100644 apps/nestjs-backend/src/features/integrity/integrity.module.ts create mode 100644 apps/nestjs-backend/src/features/integrity/link-integrity.service.ts create mode 100644 packages/openapi/src/integrity/index.ts create mode 100644 packages/openapi/src/integrity/link-check.ts create mode 100644 packages/openapi/src/integrity/link-fix.ts delete mode 100644 packages/openapi/src/table/sql-query.ts diff --git a/apps/nestjs-backend/src/app.module.ts b/apps/nestjs-backend/src/app.module.ts index d50684148f..57c87b979f 100644 --- a/apps/nestjs-backend/src/app.module.ts +++ b/apps/nestjs-backend/src/app.module.ts @@ -18,6 +18,7 @@ import { ExportOpenApiModule } from './features/export/open-api/export-open-api. import { FieldOpenApiModule } from './features/field/open-api/field-open-api.module'; import { HealthModule } from './features/health/health.module'; import { ImportOpenApiModule } from './features/import/open-api/import-open-api.module'; +import { IntegrityModule } from './features/integrity/integrity.module'; import { InvitationModule } from './features/invitation/invitation.module'; import { NextModule } from './features/next/next.module'; import { NotificationModule } from './features/notification/notification.module'; @@ -44,6 +45,7 @@ export const appModules = { NextModule, FieldOpenApiModule, BaseModule, + IntegrityModule, ChatModule, AttachmentsModule, WsModule, diff --git a/apps/nestjs-backend/src/db-provider/db.provider.interface.ts b/apps/nestjs-backend/src/db-provider/db.provider.interface.ts index a289c2188e..588aaa6b9b 100644 --- a/apps/nestjs-backend/src/db-provider/db.provider.interface.ts +++ b/apps/nestjs-backend/src/db-provider/db.provider.interface.ts @@ -1,4 +1,4 @@ -import type { DriverClient, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; +import type { DriverClient, FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi'; import type { Knex } from 'knex'; @@ -72,6 +72,8 @@ export interface IDbProvider { prisma: Prisma.TransactionClient ): Promise; + checkTableExist(tableName: string): string; + dropColumnAndIndex(tableName: string, columnName: string, indexName: string): string[]; modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[]; @@ -165,5 +167,5 @@ export interface IDbProvider { lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string; - optionsQuery(optionsKey: string, value: string): string; + optionsQuery(type: FieldType, optionsKey: string, value: string): string; } diff --git a/apps/nestjs-backend/src/db-provider/postgres.provider.ts b/apps/nestjs-backend/src/db-provider/postgres.provider.ts index 7df49319e8..09fcca324c 100644 --- a/apps/nestjs-backend/src/db-provider/postgres.provider.ts +++ b/apps/nestjs-backend/src/db-provider/postgres.provider.ts @@ -1,6 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { Logger } from '@nestjs/common'; -import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; +import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; import { DriverClient } from '@teable/core'; import type { PrismaClient } from '@teable/db-main-prisma'; import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi'; @@ -75,6 +75,16 @@ export class PostgresProvider implements IDbProvider { return res[0].exists; } + checkTableExist(tableName: string): string { + const [schemaName, dbTableName] = this.splitTableName(tableName); + return this.knex + .raw( + 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = ? AND table_name = ?) AS exists', + [schemaName, dbTableName] + ) + .toQuery(); + } + renameColumn(tableName: string, oldName: string, newName: string): string[] { return this.knex.schema .alterTable(tableName, (table) => { @@ -424,20 +434,29 @@ export class PostgresProvider implements IDbProvider { lookupOptionsQuery(optionsKey: keyof ILookupOptionsVo, value: string): string { return this.knex('field') .select({ + tableId: 'table_id', id: 'id', + type: 'type', + name: 'name', lookupOptions: 'lookup_options', }) + .whereNull('deleted_time') .whereRaw(`lookup_options::json->>'${optionsKey}' = ?`, [value]) .toQuery(); } - optionsQuery(optionsKey: string, value: string): string { + optionsQuery(type: FieldType, optionsKey: string, value: string): string { return this.knex('field') .select({ + tableId: 'table_id', id: 'id', + type: 'type', + name: 'name', options: 'options', }) + .whereNull('deleted_time') .whereRaw(`options::json->>'${optionsKey}' = ?`, [value]) + .where('type', type) .toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts index 03a0113304..4f8053c5de 100644 --- a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts +++ b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts @@ -1,6 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { Logger } from '@nestjs/common'; -import type { IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; +import type { FieldType, IFilter, ILookupOptionsVo, ISortItem } from '@teable/core'; import { DriverClient } from '@teable/core'; import type { PrismaClient } from '@teable/db-main-prisma'; import type { IAggregationField, ISearchIndexByQueryRo } from '@teable/openapi'; @@ -65,6 +65,18 @@ export class SqliteProvider implements IDbProvider { return columns.some((column) => column.name === columnName); } + checkTableExist(tableName: string): string { + return this.knex + .raw( + `SELECT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type='table' AND name = ? + ) as exists`, + [tableName] + ) + .toQuery(); + } + renameColumn(tableName: string, oldName: string, newName: string): string[] { return [ this.knex @@ -382,18 +394,25 @@ export class SqliteProvider implements IDbProvider { return this.knex('field') .select({ id: 'id', + type: 'type', + name: 'name', lookupOptions: 'lookup_options', }) + .whereNull('deleted_time') .whereRaw(`json_extract(lookup_options, '$."${optionsKey}"') = ?`, [value]) .toQuery(); } - optionsQuery(optionsKey: string, value: string): string { + optionsQuery(type: FieldType, optionsKey: string, value: string): string { return this.knex('field') .select({ id: 'id', + type: 'type', + name: 'name', options: 'options', }) + .where('type', type) + .whereNull('deleted_time') .whereRaw(`json_extract(options, '$."${optionsKey}"') = ?`, [value]) .toQuery(); } diff --git a/apps/nestjs-backend/src/features/calculation/reference.service.ts b/apps/nestjs-backend/src/features/calculation/reference.service.ts index 234f2b12c0..872ae0c97a 100644 --- a/apps/nestjs-backend/src/features/calculation/reference.service.ts +++ b/apps/nestjs-backend/src/features/calculation/reference.service.ts @@ -343,6 +343,10 @@ export class ReferenceService { if (!fromRecordIds?.length && !toRecordIds?.length) { continue; } + + console.log('order', JSON.stringify(order, null, 2)); + console.log('order:fromRecordIds', fromRecordIds); + console.log('order:toRecordIds', toRecordIds); const relatedRecordItems = await this.getAffectedRecordItems({ fieldId, fieldMap, @@ -351,6 +355,7 @@ export class ReferenceService { fkRecordMap, tableId2DbTableName, }); + console.log('relatedRecordItems', relatedRecordItems); if (field.lookupOptions || field.type === FieldType.Link) { await this.calculateLinkRelatedRecords({ @@ -583,6 +588,13 @@ export class ReferenceService { : (field.options as ILinkFieldOptions); const { relationship } = lookupOptions; const linkFieldId = field.lookupOptions ? field.lookupOptions.linkFieldId : field.id; + + if (!recordItem.record?.fields) { + console.log('recordItem', JSON.stringify(recordItem, null, 2)); + console.log('recordItem.field', field); + throw new InternalServerErrorException('record fields is undefined'); + } + const cellValue = recordItem.record.fields[linkFieldId]; const dependenciesIndexed = keyBy(dependencies, 'id'); diff --git a/apps/nestjs-backend/src/features/integrity/integrity.controller.ts b/apps/nestjs-backend/src/features/integrity/integrity.controller.ts new file mode 100644 index 0000000000..a72d33c508 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity.controller.ts @@ -0,0 +1,23 @@ +import { Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import type { IIntegrityCheckVo } from '@teable/openapi'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PermissionGuard } from '../auth/guard/permission.guard'; +import { LinkIntegrityService } from './link-integrity.service'; + +@UseGuards(PermissionGuard) +@Controller('api/integrity') +export class IntegrityController { + constructor(private readonly linkIntegrityService: LinkIntegrityService) {} + + @Permissions('table|create') + @Get('base/:baseId/link-check') + async checkBaseIntegrity(@Param('baseId') baseId: string): Promise { + return await this.linkIntegrityService.linkIntegrityCheck(baseId); + } + + @Permissions('table|create') + @Post('base/:baseId/link-fix') + async fixBaseIntegrity(@Param('baseId') baseId: string): Promise { + return await this.linkIntegrityService.linkIntegrityFix(baseId); + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity.module.ts b/apps/nestjs-backend/src/features/integrity/integrity.module.ts new file mode 100644 index 0000000000..a94b5a1ade --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { IntegrityController } from './integrity.controller'; +import { LinkIntegrityService } from './link-integrity.service'; + +@Module({ + controllers: [IntegrityController], + providers: [LinkIntegrityService], + exports: [LinkIntegrityService], +}) +export class IntegrityModule {} diff --git a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts new file mode 100644 index 0000000000..11bcfcb0c0 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts @@ -0,0 +1,359 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { FieldType, type ILinkFieldOptions } from '@teable/core'; +import type { Field } from '@teable/db-main-prisma'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityCheckVo, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; + +@Injectable() +export class LinkIntegrityService { + private readonly logger = new Logger(LinkIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async linkIntegrityCheck(baseId: string): Promise { + const tables = await this.prismaService.tableMeta.findMany({ + where: { baseId }, + select: { + id: true, + name: true, + fields: { + where: { type: FieldType.Link, isLookup: null, deletedTime: null }, + }, + }, + }); + + const crossBaseLinkFieldsQuery = this.dbProvider.optionsQuery(FieldType.Link, 'baseId', baseId); + const crossBaseLinkFieldsRaw = + await this.prismaService.$queryRawUnsafe(crossBaseLinkFieldsQuery); + + const crossBaseLinkFields = crossBaseLinkFieldsRaw.filter( + (field) => !tables.find((table) => table.id === field.tableId) + ); + + const linkFieldIssues: IIntegrityCheckVo['linkFieldIssues'] = []; + + for (const table of tables) { + const tableIssues = await this.checkTableLinkFields(table, baseId); + if (tableIssues.length > 0) { + linkFieldIssues.push({ + tableId: table.id, + fieldId: table.fields[0].id, + issues: tableIssues, + }); + } + } + + for (const field of crossBaseLinkFields) { + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: field.tableId, deletedTime: null }, + select: { id: true, name: true, baseId: true }, + }); + + const tableIssues = await this.checkTableLinkFields( + { + id: table.id, + name: table.name, + fields: [field], + }, + table.baseId + ); + + if (tableIssues.length > 0) { + linkFieldIssues.push({ + baseId: table.baseId, + tableId: field.tableId, + fieldId: field.id, + issues: tableIssues, + }); + } + } + + return { + hasIssues: linkFieldIssues.length > 0, + linkFieldIssues, + }; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private async checkTableLinkFields( + table: { + id: string; + name: string; + fields: Field[]; + }, + baseId: string + ): Promise { + const issues: IIntegrityIssue[] = []; + + for (const field of table.fields) { + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + + const foreignTable = await this.prismaService.tableMeta.findFirst({ + where: { id: options.foreignTableId, deletedTime: null }, + select: { id: true, baseId: true, dbTableName: true }, + }); + + if (!foreignTable) { + issues.push({ + type: IntegrityIssueType.ForeignTableNotFound, + message: `Foreign table with ID ${options.foreignTableId} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + + if (options.baseId === baseId) { + issues.push({ + type: IntegrityIssueType.SameBaseReference, + message: `Link field ${field.name} in table ${table.name} references the same base ID ${baseId}, which should not be configured`, + }); + } + + const tableExistsSql = this.dbProvider.checkTableExist(options.fkHostTableName); + const tableExists = + await this.prismaService.$queryRawUnsafe<{ exists: boolean }[]>(tableExistsSql); + + if (!tableExists[0].exists) { + issues.push({ + type: IntegrityIssueType.ForeignKeyHostTableNotFound, + message: `Foreign key host table ${options.fkHostTableName} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } else { + const selfKeyExists = await this.dbProvider.checkColumnExist( + options.fkHostTableName, + options.selfKeyName, + this.prismaService + ); + + const foreignKeyExists = await this.dbProvider.checkColumnExist( + options.fkHostTableName, + options.foreignKeyName, + this.prismaService + ); + + if (!selfKeyExists) { + issues.push({ + type: IntegrityIssueType.ForeignKeyNotFound, + message: `Self key name "${options.selfKeyName}" is missing for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + + if (!foreignKeyExists) { + issues.push({ + type: IntegrityIssueType.ForeignKeyNotFound, + message: `Foreign key name "${options.foreignKeyName}" is missing for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + } + + if (options.symmetricFieldId) { + const symmetricField = await this.prismaService.field.findFirst({ + where: { id: options.symmetricFieldId, deletedTime: null }, + }); + + if (!symmetricField) { + issues.push({ + type: IntegrityIssueType.SymmetricFieldNotFound, + message: `Symmetric field ID ${options.symmetricFieldId} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + } + + if (foreignTable) { + const invalidReferences = await this.checkInvalidRecordReferences(table.id, field, options); + + if (invalidReferences.length > 0) { + issues.push(...invalidReferences); + } + } + } + + return issues; + } + + private async checkInvalidRecordReferences( + tableId: string, + field: { id: string; name: string }, + options: ILinkFieldOptions + ): Promise { + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; + + const { name: selfTableName, dbTableName: selfTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + const { name: foreignTableName, dbTableName: foreignTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + const issues: IIntegrityIssue[] = []; + + // Check self references + if (selfTableDbTableName !== fkHostTableName) { + const selfIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: selfTableDbTableName, + keyName: selfKeyName, + field, + referencedTableName: selfTableName, + isSelfReference: true, + }); + issues.push(...selfIssues); + } + + // Check foreign references + if (foreignTableDbTableName !== fkHostTableName) { + const foreignIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: foreignTableDbTableName, + keyName: foreignKeyName, + field, + referencedTableName: foreignTableName, + isSelfReference: false, + }); + issues.push(...foreignIssues); + } + + return issues; + } + + private async checkInvalidReferences({ + fkHostTableName, + targetTableName, + keyName, + field, + referencedTableName, + isSelfReference, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + field: { id: string; name: string }; + referencedTableName: string; + isSelfReference: boolean; + }): Promise { + const issues: IIntegrityIssue[] = []; + + const invalidQuery = this.knex(fkHostTableName) + .leftJoin(targetTableName, `${fkHostTableName}.${keyName}`, `${targetTableName}.__id`) + .whereNull(`${targetTableName}.__id`) + .count(`${fkHostTableName}.${keyName} as count`) + .first() + .toQuery(); + + try { + const invalidRefs = + await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(invalidQuery); + const refCount = Number(invalidRefs[0]?.count || 0); + + if (refCount > 0) { + const message = isSelfReference + ? `Found ${refCount} invalid self references in table ${referencedTableName}` + : `Found ${refCount} invalid foreign references to table ${referencedTableName}`; + + issues.push({ + type: IntegrityIssueType.InvalidRecordReference, + message: `${message} (Field Name: ${field.name}, Field ID: ${field.id})`, + }); + } + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { + console.error('error ignored:', error); + } else { + throw error; + } + } + + return issues; + } + + async linkIntegrityFix(baseId: string) { + const checkResult = await this.linkIntegrityCheck(baseId); + + for (const issues of checkResult.linkFieldIssues) { + for (const issue of issues.issues) { + // eslint-disable-next-line sonarjs/no-small-switch + switch (issue.type) { + case IntegrityIssueType.InvalidRecordReference: + await this.fixInvalidRecordReferences(issues.tableId, issues.fieldId); + break; + default: + break; + } + } + } + } + + async fixInvalidRecordReferences(tableId: string, fieldId: string) { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; + + const { dbTableName: selfTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + const { dbTableName: foreignTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + // Delete invalid self references + if (selfTableDbTableName !== fkHostTableName) { + await this.deleteInvalidReferences({ + fkHostTableName, + targetTableName: selfTableDbTableName, + keyName: selfKeyName, + }); + } + + // Delete invalid foreign references + if (foreignTableDbTableName !== fkHostTableName) { + await this.deleteInvalidReferences({ + fkHostTableName, + targetTableName: foreignTableDbTableName, + keyName: foreignKeyName, + }); + } + } + + private async deleteInvalidReferences({ + fkHostTableName, + targetTableName, + keyName, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + }) { + const deleteQuery = this.knex(fkHostTableName) + .whereNotExists( + this.knex + .select('__id') + .from(targetTableName) + .where('__id', this.knex.ref(`${fkHostTableName}.${keyName}`)) + ) + .delete() + .toQuery(); + + const result = await this.prismaService.$executeRawUnsafe(deleteQuery); + this.logger.log('deleteQuery result:', result); + } +} diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts index b449204cc3..9d294c8ca9 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts @@ -469,7 +469,11 @@ export class TableOpenApiService { throw new NotFoundException(`table ${tableId} not found`); }); - const linkFieldsQuery = this.dbProvider.optionsQuery('fkHostTableName', oldDbTableName); + const linkFieldsQuery = this.dbProvider.optionsQuery( + FieldType.Link, + 'fkHostTableName', + oldDbTableName + ); const lookupFieldsQuery = this.dbProvider.lookupOptionsQuery('fkHostTableName', oldDbTableName); await this.prismaService.$tx(async (prisma) => { diff --git a/apps/nestjs-backend/test/link-api.e2e-spec.ts b/apps/nestjs-backend/test/link-api.e2e-spec.ts index 67f33ef30d..28d7b7b04f 100644 --- a/apps/nestjs-backend/test/link-api.e2e-spec.ts +++ b/apps/nestjs-backend/test/link-api.e2e-spec.ts @@ -8,6 +8,7 @@ import type { IFieldRo, IFieldVo, ILinkFieldOptions, ILookupOptionsVo } from '@t import { FieldKeyType, FieldType, NumberFormattingType, Relationship } from '@teable/core'; import type { ITableFullVo } from '@teable/openapi'; import { + checkBaseIntegrity, convertField, createBase, deleteBase, @@ -2899,7 +2900,7 @@ describe('OpenAPI link (e2e)', () => { await permanentDeleteTable(baseId, table2.id); }); - it('should correct update db table name', async () => { + it.only('should correct update db table name', async () => { const table1LinkField = table1.fields[2]; const table2LinkField = table2.fields[2]; expect((table1LinkField.options as ILinkFieldOptions).fkHostTableName).toEqual( @@ -2996,6 +2997,9 @@ describe('OpenAPI link (e2e)', () => { (updatedLinkField.options as ILinkFieldOptions).symmetricFieldId as string ); expect((symUpdatedLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId); + + const integrity = await checkBaseIntegrity(baseId2); + expect(integrity.data.hasIssues).toEqual(false); }); it('should correct update db table name when link field is cross base', async () => { diff --git a/apps/nextjs-app/.env.development b/apps/nextjs-app/.env.development index 27e77e4f41..a86cb70138 100644 --- a/apps/nextjs-app/.env.development +++ b/apps/nextjs-app/.env.development @@ -21,7 +21,7 @@ PUBLIC_ORIGIN=http://127.0.0.1:3000 STORAGE_PREFIX=http://localhost:3000 # DATABASE_URL # @see https://www.prisma.io/docs/reference/database-reference/connection-urls#examples -PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:5432/teable?schema=public&statement_cache_size=1 +PRISMA_DATABASE_URL=postgresql://postgres:tk882hrn@dbconn.sealosgzg.site:38577/teable?schema=public&statement_cache_size=1 PUBLIC_DATABASE_PROXY=127.0.0.1:5432 API_DOC_DISENABLED=false diff --git a/packages/db-main-prisma/prisma/postgres/schema.prisma b/packages/db-main-prisma/prisma/postgres/schema.prisma index 55b50e641c..93be5a5c38 100644 --- a/packages/db-main-prisma/prisma/postgres/schema.prisma +++ b/packages/db-main-prisma/prisma/postgres/schema.prisma @@ -108,6 +108,9 @@ model Field { lastModifiedBy String? @map("last_modified_by") table TableMeta @relation(fields: [tableId], references: [id]) + fromReferences Reference[] @relation("FromFields") + toReferences Reference[] @relation("ToFields") + @@index([lookupLinkedFieldId]) @@map("field") } @@ -170,6 +173,9 @@ model Reference { toFieldId String @map("to_field_id") createdTime DateTime @default(now()) @map("created_time") + fromField Field @relation("FromFields", fields: [fromFieldId], references: [id]) + toField Field @relation("ToFields", fields: [toFieldId], references: [id]) + @@unique([toFieldId, fromFieldId]) @@index([fromFieldId]) @@index([toFieldId]) diff --git a/packages/db-main-prisma/prisma/sqlite/schema.prisma b/packages/db-main-prisma/prisma/sqlite/schema.prisma index 63320be203..5f00fc2709 100644 --- a/packages/db-main-prisma/prisma/sqlite/schema.prisma +++ b/packages/db-main-prisma/prisma/sqlite/schema.prisma @@ -108,6 +108,9 @@ model Field { lastModifiedBy String? @map("last_modified_by") table TableMeta @relation(fields: [tableId], references: [id]) + fromReferences Reference[] @relation("FromFields") + toReferences Reference[] @relation("ToFields") + @@index([lookupLinkedFieldId]) @@map("field") } @@ -170,6 +173,9 @@ model Reference { toFieldId String @map("to_field_id") createdTime DateTime @default(now()) @map("created_time") + fromField Field @relation("FromFields", fields: [fromFieldId], references: [id]) + toField Field @relation("ToFields", fields: [toFieldId], references: [id]) + @@unique([toFieldId, fromFieldId]) @@index([fromFieldId]) @@index([toFieldId]) diff --git a/packages/db-main-prisma/prisma/template.prisma b/packages/db-main-prisma/prisma/template.prisma index 059a0c9bf8..38b75bcd3d 100644 --- a/packages/db-main-prisma/prisma/template.prisma +++ b/packages/db-main-prisma/prisma/template.prisma @@ -108,6 +108,9 @@ model Field { lastModifiedBy String? @map("last_modified_by") table TableMeta @relation(fields: [tableId], references: [id]) + fromReferences Reference[] @relation("FromFields") + toReferences Reference[] @relation("ToFields") + @@index([lookupLinkedFieldId]) @@map("field") } @@ -170,6 +173,9 @@ model Reference { toFieldId String @map("to_field_id") createdTime DateTime @default(now()) @map("created_time") + fromField Field @relation("FromFields", fields: [fromFieldId], references: [id]) + toField Field @relation("ToFields", fields: [toFieldId], references: [id]) + @@unique([toFieldId, fromFieldId]) @@index([fromFieldId]) @@index([toFieldId]) diff --git a/packages/openapi/src/access-token/refresh.ts b/packages/openapi/src/access-token/refresh.ts index b7b73ade4d..2c8d4beeae 100644 --- a/packages/openapi/src/access-token/refresh.ts +++ b/packages/openapi/src/access-token/refresh.ts @@ -37,7 +37,7 @@ export const accessTokenRoute = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Returns access token.', content: { 'application/json': { diff --git a/packages/openapi/src/auth/add-password.ts b/packages/openapi/src/auth/add-password.ts index 7bf029ac09..389686f3ef 100644 --- a/packages/openapi/src/auth/add-password.ts +++ b/packages/openapi/src/auth/add-password.ts @@ -26,7 +26,7 @@ export const addPasswordRoute = registerRoute({ }, tags: ['auth'], responses: { - 200: { + 201: { description: 'Successfully added password', }, }, diff --git a/packages/openapi/src/auth/reset-password.ts b/packages/openapi/src/auth/reset-password.ts index 29a006ab8f..23b9b2c2b4 100644 --- a/packages/openapi/src/auth/reset-password.ts +++ b/packages/openapi/src/auth/reset-password.ts @@ -27,7 +27,7 @@ export const resetPasswordRoute = registerRoute({ }, tags: ['auth'], responses: { - 200: { + 201: { description: 'Successfully reset password', }, }, diff --git a/packages/openapi/src/auth/send-reset-password-email.ts b/packages/openapi/src/auth/send-reset-password-email.ts index df8f3ceae7..32fa91d997 100644 --- a/packages/openapi/src/auth/send-reset-password-email.ts +++ b/packages/openapi/src/auth/send-reset-password-email.ts @@ -25,7 +25,7 @@ export const sendResetPasswordEmailRoute = registerRoute({ }, tags: ['auth'], responses: { - 200: { + 201: { description: 'Successfully sent reset password email', }, }, diff --git a/packages/openapi/src/comment/create.ts b/packages/openapi/src/comment/create.ts index 6c213690fd..0440620d9a 100644 --- a/packages/openapi/src/comment/create.ts +++ b/packages/openapi/src/comment/create.ts @@ -26,7 +26,7 @@ export const CreateCommentRoute: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Successfully create comment.', }, }, diff --git a/packages/openapi/src/comment/reaction/create-reaction.ts b/packages/openapi/src/comment/reaction/create-reaction.ts index 5213401b5d..ba497fc997 100644 --- a/packages/openapi/src/comment/reaction/create-reaction.ts +++ b/packages/openapi/src/comment/reaction/create-reaction.ts @@ -46,7 +46,7 @@ export const CreateCommentReactionRoute: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Successfully create comment reaction.', }, }, diff --git a/packages/openapi/src/comment/subscribe/create-subscribe.ts b/packages/openapi/src/comment/subscribe/create-subscribe.ts index 4a8f7c0b73..554b261e5c 100644 --- a/packages/openapi/src/comment/subscribe/create-subscribe.ts +++ b/packages/openapi/src/comment/subscribe/create-subscribe.ts @@ -16,7 +16,7 @@ export const CreateCommentSubscribeRoute: RouteConfig = registerRoute({ }), }, responses: { - 200: { + 201: { description: 'Successfully subscribe record comment.', }, }, diff --git a/packages/openapi/src/dashboard/create.ts b/packages/openapi/src/dashboard/create.ts index 7b9525b15f..91deae4de0 100644 --- a/packages/openapi/src/dashboard/create.ts +++ b/packages/openapi/src/dashboard/create.ts @@ -35,7 +35,7 @@ export const CreateDashboardRoute: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Returns data about the created dashboard.', content: { 'application/json': { diff --git a/packages/openapi/src/import/import-table.ts b/packages/openapi/src/import/import-table.ts index d32f4a652c..ec552608ff 100644 --- a/packages/openapi/src/import/import-table.ts +++ b/packages/openapi/src/import/import-table.ts @@ -26,7 +26,7 @@ export const ImportTableFromFileRoute: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Returns data about a table without records', content: { 'application/json': { diff --git a/packages/openapi/src/index.ts b/packages/openapi/src/index.ts index 9cb4fccdbe..de669cdb3c 100644 --- a/packages/openapi/src/index.ts +++ b/packages/openapi/src/index.ts @@ -32,3 +32,4 @@ export * from './plugin'; export * from './dashboard'; export * from './comment'; export * from './organization'; +export * from './integrity'; diff --git a/packages/openapi/src/integrity/index.ts b/packages/openapi/src/integrity/index.ts new file mode 100644 index 0000000000..d328b4f88e --- /dev/null +++ b/packages/openapi/src/integrity/index.ts @@ -0,0 +1,2 @@ +export * from './link-check'; +export * from './link-fix'; diff --git a/packages/openapi/src/integrity/link-check.ts b/packages/openapi/src/integrity/link-check.ts new file mode 100644 index 0000000000..80a89c10f7 --- /dev/null +++ b/packages/openapi/src/integrity/link-check.ts @@ -0,0 +1,74 @@ +import type { RouteConfig } from '@asteasolutions/zod-to-openapi'; +import { axios } from '../axios'; +import { registerRoute, urlBuilder } from '../utils'; +import { z } from '../zod'; + +export const CHECK_BASE_INTEGRITY = '/integrity/base/{baseId}/link-check'; + +// Define the issue types enum +export enum IntegrityIssueType { + ForeignTableNotFound = 'ForeignTableNotFound', + ForeignKeyNotFound = 'ForeignKeyNotFound', + SelfKeyNotFound = 'SelfKeyNotFound', + SymmetricFieldNotFound = 'SymmetricFieldNotFound', + InvalidRecordReference = 'InvalidRecordReference', + SameBaseReference = 'SameBaseReference', + ForeignKeyHostTableNotFound = 'ForeignKeyHostTableNotFound', +} + +// Define the schema for a single issue +const integrityIssueSchema = z.object({ + type: z.nativeEnum(IntegrityIssueType), + message: z.string(), +}); + +// Define the schema for a link field check item +const linkFieldCheckItemSchema = z.object({ + baseId: z + .string() + .optional() + .openapi({ description: 'The base id of the link field with is cross-base' }), + fieldId: z.string(), + tableId: z.string(), + issues: z.array(integrityIssueSchema), +}); + +export type IIntegrityIssue = z.infer; + +// Define the response schema +export const integrityCheckVoSchema = z.object({ + hasIssues: z.boolean(), + linkFieldIssues: z.array(linkFieldCheckItemSchema), +}); + +export type IIntegrityCheckVo = z.infer; + +export const IntegrityCheckRoute: RouteConfig = registerRoute({ + method: 'get', + path: CHECK_BASE_INTEGRITY, + description: 'Check integrity of link fields in a base', + request: { + params: z.object({ + baseId: z.string(), + }), + }, + responses: { + 200: { + description: 'Returns integrity check results for the base', + content: { + 'application/json': { + schema: integrityCheckVoSchema, + }, + }, + }, + }, + tags: ['integrity'], +}); + +export const checkBaseIntegrity = async (baseId: string) => { + return axios.get( + urlBuilder(CHECK_BASE_INTEGRITY, { + baseId, + }) + ); +}; diff --git a/packages/openapi/src/integrity/link-fix.ts b/packages/openapi/src/integrity/link-fix.ts new file mode 100644 index 0000000000..6378060b1e --- /dev/null +++ b/packages/openapi/src/integrity/link-fix.ts @@ -0,0 +1,31 @@ +import type { RouteConfig } from '@asteasolutions/zod-to-openapi'; +import { axios } from '../axios'; +import { registerRoute, urlBuilder } from '../utils'; +import { z } from '../zod'; + +export const FIX_BASE_INTEGRITY = '/integrity/base/{baseId}/link-fix'; + +export const IntegrityFixRoute: RouteConfig = registerRoute({ + method: 'post', + path: FIX_BASE_INTEGRITY, + description: 'Fix integrity of link fields in a base', + request: { + params: z.object({ + baseId: z.string(), + }), + }, + responses: { + 201: { + description: 'Success', + }, + }, + tags: ['integrity'], +}); + +export const fixBaseIntegrity = async (baseId: string) => { + return axios.post( + urlBuilder(FIX_BASE_INTEGRITY, { + baseId, + }) + ); +}; diff --git a/packages/openapi/src/oauth/revoke.ts b/packages/openapi/src/oauth/revoke.ts index 4b6b709391..f2e2f9b8db 100644 --- a/packages/openapi/src/oauth/revoke.ts +++ b/packages/openapi/src/oauth/revoke.ts @@ -13,7 +13,7 @@ export const revokeAccessRoute = registerRoute({ }), }, responses: { - 200: { + 201: { description: 'Revoke access permission successfully', }, }, diff --git a/packages/openapi/src/plugin/get-auth-code.ts b/packages/openapi/src/plugin/get-auth-code.ts index 1e15aa779d..7ad8e38a06 100644 --- a/packages/openapi/src/plugin/get-auth-code.ts +++ b/packages/openapi/src/plugin/get-auth-code.ts @@ -25,7 +25,7 @@ export const pluginGetAuthCodeRouter = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Returns auth code.', content: { 'application/json': { diff --git a/packages/openapi/src/plugin/refresh-token.ts b/packages/openapi/src/plugin/refresh-token.ts index dab0efb4c8..6e747b8cb1 100644 --- a/packages/openapi/src/plugin/refresh-token.ts +++ b/packages/openapi/src/plugin/refresh-token.ts @@ -34,7 +34,7 @@ export const PluginRefreshTokenRoute: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Returns token.', content: { 'application/json': { diff --git a/packages/openapi/src/plugin/regenerate-secret.ts b/packages/openapi/src/plugin/regenerate-secret.ts index 2cac2cc6c2..60a9deda14 100644 --- a/packages/openapi/src/plugin/regenerate-secret.ts +++ b/packages/openapi/src/plugin/regenerate-secret.ts @@ -26,7 +26,7 @@ export const pluginRegenerateSecretRoute: RouteConfig = registerRoute({ params: pluginRegenerateSecretRoSchema, }, responses: { - 200: { + 201: { description: 'Returns data about the plugin.', content: { 'application/json': { diff --git a/packages/openapi/src/share/view-auth.ts b/packages/openapi/src/share/view-auth.ts index 844b6203b2..2d633602ff 100644 --- a/packages/openapi/src/share/view-auth.ts +++ b/packages/openapi/src/share/view-auth.ts @@ -31,7 +31,7 @@ export const ShareViewAuthRouter: RouteConfig = registerRoute({ }, }, responses: { - 200: { + 201: { description: 'Successfully authenticated', content: { 'application/json': { diff --git a/packages/openapi/src/table/index.ts b/packages/openapi/src/table/index.ts index 990527212f..937bea8223 100644 --- a/packages/openapi/src/table/index.ts +++ b/packages/openapi/src/table/index.ts @@ -2,7 +2,6 @@ export * from './create'; export * from './delete'; export * from './get-list'; export * from './get'; -export * from './sql-query'; export * from './permanent-delete'; export * from './update-name'; export * from './update-icon'; diff --git a/packages/openapi/src/table/sql-query.ts b/packages/openapi/src/table/sql-query.ts deleted file mode 100644 index 5c7e4d18ec..0000000000 --- a/packages/openapi/src/table/sql-query.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { RouteConfig } from '@asteasolutions/zod-to-openapi'; -import { axios } from '../axios'; -import { registerRoute, urlBuilder } from '../utils'; -import { z } from '../zod'; - -export const sqlQuerySchema = z.object({ - viewId: z.string(), - sql: z.string(), -}); - -export type ISqlQuerySchema = z.infer; - -export const TABLE_SQL_QUERY = '/base/{baseId}/table/{tableId}/sql-query'; - -export const TableSqlQueryRoute: RouteConfig = registerRoute({ - method: 'post', - path: TABLE_SQL_QUERY, - description: 'Query a table by raw sql', - request: { - params: z.object({ - baseId: z.string(), - tableId: z.string(), - }), - query: sqlQuerySchema, - }, - responses: { - 200: { - description: 'Returns sql query result data.', - }, - }, - tags: ['table'], -}); - -export const tableSqlQuery = async (baseId: string, tableId: string, query: ISqlQuerySchema) => { - return axios.post( - urlBuilder(TABLE_SQL_QUERY, { - baseId, - tableId, - }), - undefined, - { - params: query, - } - ); -};