feat: new cron job for duplicates (bloom-housing#4230) (#766)

* feat: new cron job for duplicates (bloom-housing#4230) * fix: add duplicates view * fix: new cron job for duplicates * fix: multiple names flagged usecase * fix: bugs found in qa * fix: better way to connect listings * fix: clean up imports in listing service * fix: alternative way of duplicate grouping * fix: qa fixes * fix: add afs last run timestamp * fix: add additional test * fix: move type * fix: pr fixes * fix: change name of enum to combination * fix: cleanup test * fix: make lottery test more consistent * fix: update to a higher migration number * fix: switch to concat * fix: lottery merge conflicts * fix: remove missed merge conflict --------- Co-authored-by: Jared White <[email protected]>
housingbayarea · Oct 2, 2024 · 10c7233 · 10c7233
1 parent d850244
commit 10c7233
Show file tree

Hide file tree

Showing 17 changed files with 3,630 additions and 1,468 deletions.
diff --git a/api/.env.template b/api/.env.template
@@ -35,7 +35,7 @@ PARTNERS_PORTAL_URL=http://localhost:3001
 # sendgrid email api key
 EMAIL_API_KEY=SG.ExampleApiKey
 # controls the repetition of the afs cron job
-AFS_PROCESSING_CRON_STRING=0,3,15 * * * *
+AFS_PROCESSING_CRON_STRING=0 * * * *
 # controls the repetition of the listing cron job
 LISTING_PROCESSING_CRON_STRING=0 * * * *
 # controls the repetition of the lottery publish cron job
@@ -62,3 +62,7 @@ THROTTLE_LIMIT=100
 API_PASS_KEY="some-key-here"
 # this is used to test the script runner's data transfer job
 TEST_CONNECTION_STRING=""
+# controls the repetition of the new afs cron job
+DUPLICATES_PROCESSING_CRON_STRING=15 * * * *
+# Date the different AFS cron jobs should use to determine which cron job is applicable for the listing
+DUPLICATES_CLOSE_DATE="2024-07-28 00:00 -08:00"
diff --git a/api/prisma/migrations/17_add_duplicates_view/migration.sql b/api/prisma/migrations/17_add_duplicates_view/migration.sql
@@ -0,0 +1,73 @@
+CREATE VIEW "application_flagged_set_possibilities" AS (
+ SELECT
+ CONCAT(
+ LOWER(a.first_name),
+ '-',
+ LOWER(a.last_name),
+ '-',
+ a.birth_month,
+ '-',
+ a.birth_day,
+ '-',
+ a.birth_year
+ ) as "key",
+ app.listing_id,
+ app.id as "application_id",
+ 'nameAndDOB' as "type"
+ FROM
+ applicant a,
+ applications app
+ WHERE
+ a.id = app.applicant_id
+ and app.deleted_at is null
+)
+UNION
+(
+ SELECT
+ a.email_address as "key",
+ app.listing_id,
+ app.id as "application_id",
+ 'email' as "type"
+ FROM
+ applications app,
+ applicant a
+ WHERE
+ a.id = app.applicant_id
+ and a.email_address is not null
+ and app.deleted_at is null
+)
+UNION
+(
+ SELECT
+ CONCAT(
+ LOWER(hm.first_name),
+ '-',
+ LOWER(hm.last_name),
+ '-',
+ hm.birth_month,
+ '-',
+ hm.birth_day,
+ '-',
+ hm.birth_year
+ ) as "key",
+ app.listing_id,
+ app.id as "application_id",
+ 'nameAndDOB' as "type"
+ FROM
+ applications app,
+ household_member hm
+ WHERE
+ hm.application_id = app.id
+ and app.deleted_at is null
+);
+
+ALTER TYPE "rule_enum"
+ADD
+ VALUE 'combination';
+
+ALTER TABLE
+ application_flagged_set DROP CONSTRAINT IF EXISTS "application_flagged_set_rule_key_key";
+
+DROP INDEX IF EXISTS "application_flagged_set_rule_key_key";
+
+CREATE UNIQUE INDEX "application_flagged_set_rule_key_listing_id_key" ON "application_flagged_set"("rule_key", "listing_id");
diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma
@@ -1,6 +1,6 @@
 generator client {
  provider = "prisma-client-js"
- previewFeatures = ["postgresqlExtensions"]
+ previewFeatures = ["postgresqlExtensions", "views"]
 }
 
 datasource db {
@@ -136,7 +136,7 @@ model ApplicationFlaggedSet {
  createdAt DateTime @default(now()) @map("created_at") @db.Timestamp(6)
  updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamp(6)
  rule RuleEnum
- ruleKey String @unique() @map("rule_key") @db.VarChar
+ ruleKey String @map("rule_key") @db.VarChar
  resolvedTime DateTime? @map("resolved_time") @db.Timestamptz(6)
  listingId String @map("listing_id") @db.Uuid
  showConfirmationAlert Boolean @default(false) @map("show_confirmation_alert")
@@ -147,6 +147,7 @@ model ApplicationFlaggedSet {
  applications Applications[]
 
  @@index([listingId])
+ @@unique([ruleKey, listingId])
  @@map("application_flagged_set")
 }
 
@@ -928,6 +929,16 @@ model ScriptRuns {
  @@map("script_runs")
 }
 
+view ApplicationFlaggedSetPossibilities {
+ key String
+ listingId String @map("listing_id") @db.Uuid
+ applicationId String @map("application_id") @db.Uuid
+ type String
+
+ @@id([key, applicationId])
+ @@map("application_flagged_set_possibilities")
+}
+
 enum ApplicationMethodsTypeEnum {
  Internal
  FileDownload
@@ -1010,6 +1021,7 @@ enum YesNoEnum {
 enum RuleEnum {
  nameAndDOB
  email
+ combination
 
  @@map("rule_enum")
 }

diff --git a/api/prisma/seed-helpers/household-member-factory.ts b/api/prisma/seed-helpers/household-member-factory.ts
@@ -11,6 +11,7 @@ import {
 
 export const householdMemberFactorySingle = (
  index: number,
+ overrides?: Prisma.HouseholdMemberCreateWithoutApplicationsInput,
 ): Prisma.HouseholdMemberCreateWithoutApplicationsInput => {
  const firstName = randomNoun();
  const lastName = randomNoun();
@@ -33,6 +34,7 @@ export const householdMemberFactorySingle = (
  create: addressFactory(),
  },
  orderId: index,
+ ...overrides,
  };
 };
 

diff --git a/api/prisma/seed-staging.ts b/api/prisma/seed-staging.ts
@@ -37,6 +37,7 @@ import {
  simplifiedDCMap,
 } from './seed-helpers/map-layer-factory';
 import { ValidationMethod } from '../src/enums/multiselect-questions/validation-method-enum';
+import { householdMemberFactorySingle } from './seed-helpers/household-member-factory';
 
 export const stagingSeed = async (
  prismaClient: PrismaClient,
@@ -505,18 +506,131 @@ export const stagingSeed = async (
  },
  ],
  multiselectQuestions: [cityEmployeeQuestion],
- // has applications that are the same email
+ // has applications that are the same email and also same name/dob
  applications: [
+ await applicationFactory(),
+ await applicationFactory(),
  await applicationFactory({
- applicant: { emailAddress: '[email protected]' },
+ submissionType: ApplicationSubmissionTypeEnum.paper,
  }),
  await applicationFactory({
- applicant: { emailAddress: '[email protected]' },
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first',
+ lastName: 'last',
+ birthDay: 1,
+ birthMonth: 1,
+ birthYear: 1970,
+ },
  }),
- await applicationFactory(),
- await applicationFactory(),
  await applicationFactory({
- submissionType: ApplicationSubmissionTypeEnum.paper,
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first2',
+ lastName: 'last2',
+ birthDay: 2,
+ birthMonth: 2,
+ birthYear: 1992,
+ },
+ }),
+ await applicationFactory({
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first2',
+ lastName: 'last2',
+ birthDay: 2,
+ birthMonth: 2,
+ birthYear: 1992,
+ },
+ }),
+ await applicationFactory({
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first',
+ lastName: 'last',
+ birthDay: 1,
+ birthMonth: 1,
+ birthYear: 1970,
+ },
+ }),
+ await applicationFactory({
+ applicant: { emailAddress: '[email protected]' },
+ }),
+ await applicationFactory({
+ applicant: { emailAddress: '[email protected]' },
+ }),
+ await applicationFactory({
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first3',
+ lastName: 'last3',
+ birthDay: 1,
+ birthMonth: 1,
+ birthYear: 1970,
+ },
+ householdMember: [
+ householdMemberFactorySingle(1, {
+ firstName: 'householdFirst1',
+ lastName: 'householdLast1',
+ birthDay: 5,
+ birthMonth: 5,
+ birthYear: 1950,
+ }),
+ householdMemberFactorySingle(2, {
+ firstName: 'householdFirst2',
+ lastName: 'householdLast2',
+ birthDay: 8,
+ birthMonth: 8,
+ birthYear: 1980,
+ }),
+ ],
+ }),
+ await applicationFactory({
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first3',
+ lastName: 'last3',
+ birthDay: 1,
+ birthMonth: 1,
+ birthYear: 1970,
+ },
+ householdMember: [
+ householdMemberFactorySingle(1, {
+ firstName: 'householdFirst1',
+ lastName: 'householdLast1',
+ birthDay: 5,
+ birthMonth: 5,
+ birthYear: 1950,
+ }),
+ householdMemberFactorySingle(2, {
+ firstName: 'householdFirst2',
+ lastName: 'householdLast2',
+ birthDay: 8,
+ birthMonth: 8,
+ birthYear: 1980,
+ }),
+ ],
+ }),
+ await applicationFactory({
+ applicant: {
+ emailAddress: '[email protected]',
+ firstName: 'first4',
+ lastName: 'last4',
+ birthDay: 2,
+ birthMonth: 2,
+ birthYear: 2002,
+ },
+ }),
+ await applicationFactory({
+ householdMember: [
+ {
+ firstName: 'first4',
+ lastName: 'last4',
+ birthDay: 2,
+ birthMonth: 2,
+ birthYear: 2002,
+ },
+ ],
  }),
  ],
  },
@@ -710,7 +824,7 @@ export const stagingSeed = async (
  customMapPin: false,
  contentUpdatedAt: dayjs(new Date()).subtract(1, 'days').toDate(),
  publishedAt: dayjs(new Date()).subtract(3, 'days').toDate(),
- closedAt: dayjs(new Date()).subtract(1, 'days').toDate(),
+ closedAt: dayjs(new Date()).subtract(5, 'days').toDate(),
  listingsApplicationPickUpAddress: undefined,
  listingsLeasingAgentAddress: undefined,
  listingsApplicationDropOffAddress: undefined,

diff --git a/api/src/controllers/application-flagged-set.controller.ts b/api/src/controllers/application-flagged-set.controller.ts
@@ -26,6 +26,7 @@ import { PaginatedAfsDto } from '../dtos/application-flagged-sets/paginated-afs.
 import { ApplicationFlaggedSet } from '../dtos/application-flagged-sets/application-flagged-set.dto';
 import { AfsResolve } from '../dtos/application-flagged-sets/afs-resolve.dto';
 import { AfsMeta } from '../dtos/application-flagged-sets/afs-meta.dto';
+import { AfsProcessQueryParams } from '../dtos/application-flagged-sets/afs-process-query-params.dto';
 import { AfsQueryParams } from '../dtos/application-flagged-sets/afs-query-params.dto';
 import { User } from '../dtos/users/user.dto';
 import { mapTo } from '../utilities/mapTo';
@@ -105,6 +106,21 @@ export class ApplicationFlaggedSetController {
  return await this.applicationFlaggedSetService.process();
  }
 
+ @Put('process_duplicates')
+ @ApiOperation({
+ summary: 'Trigger the duplicate check process',
+ operationId: 'processDuplicates',
+ })
+ @ApiOkResponse({ type: SuccessDTO })
+ async processDuplicates(
+ @Query() params: AfsProcessQueryParams,
+ ): Promise<SuccessDTO> {
+ return await this.applicationFlaggedSetService.processDuplicates(
+ params?.listingId,
+ params?.force,
+ );
+ }
+
  @Put(':id')
  @ApiOperation({
  summary: 'Reset flagged set confirmation alert',

diff --git a/api/src/dtos/application-flagged-sets/afs-process-query-params.dto.ts b/api/src/dtos/application-flagged-sets/afs-process-query-params.dto.ts
@@ -0,0 +1,15 @@
+import { ApiPropertyOptional } from '@nestjs/swagger';
+import { Expose } from 'class-transformer';
+import { IsUUID } from 'class-validator';
+import { ValidationsGroupsEnum } from '../../enums/shared/validation-groups-enum';
+
+export class AfsProcessQueryParams {
+ @Expose()
+ @ApiPropertyOptional()
+ @IsUUID(4, { groups: [ValidationsGroupsEnum.default] })
+ listingId?: string;
+
+ @Expose()
+ @ApiPropertyOptional()
+ force?: boolean;
+}