From 3e2df78f7deef9e5b59ac55b958003f88ac0fd44 Mon Sep 17 00:00:00 2001 From: Arnaud MENGUS <44212923+isontheline@users.noreply.github.com> Date: Tue, 28 May 2024 17:47:07 +0200 Subject: [PATCH] Store vector init (#24) * API CRUD Refactoring * Backend API Refactoring * Adding vectorStoreClass and embeddingModelClass to SiloEntity * Begin of Scheduling Configuration * Fix checkstyle fail * Removing AsyncConfig and SchedulingConfig in order to replace them with JobRunr * Init JobRunr * Updating default port of JobRunr from 8000 to 1984 * refactor: Rename SecurityConfiguration to SecurityConfig on-behalf-of: @dRAGon-Okinawa team@dragon.okinawa" * chore: Update OpenAPI mappings to use single quotes instead of double quotes on-behalf-of: @ORG team@dragon.okinawa * chore: Disable JobRunr background job server, job scheduler, and dashboard for Profile("test") This commit disables the JobRunr background job server, job scheduler, and dashboard by setting the corresponding properties to false in the application-test.yaml file. This change is necessary to prevent unnecessary resource usage during testing. Note: This commit message follows the established convention of using the "chore" type for non-functional changes. * refactor: Update ProviderType enum values and add fromString method This commit updates the ProviderType enum in the `ProviderType.java` file. It adds two new enum values, `ONNX` and `OpenAI`, and modifies the existing `OPENAI` value to `OpenAI`. Additionally, a new static method `fromString` is added to the enum to retrieve the corresponding ProviderType based on a given string value. Note: This commit message follows the established convention of using the "refactor" type for code restructuring or refactoring. * refactor: Add unique index for name field in IngestorEntity and ProviderEntity This commit adds a unique index for the `name` field in both the `IngestorEntity` and `ProviderEntity` classes. The unique index ensures that the `name` field values are unique within their respective collections. Note: This commit message follows the established convention of using the "refactor" type for code restructuring or refactoring. * feat: Add create method with beforeSaveCallback to AbstractCrudBackendApiController This commit adds a new overloaded `create` method to the `AbstractCrudBackendApiController` class. The new method accepts an additional parameter `beforeSaveCallback`, which is a function that can be used to modify the entity before saving it to the repository. This allows for more flexibility in customizing the creation process of entities. Note: This commit message follows the established convention of using the "feat" type for new feature additions. * feat: Handle unique key constraint violation in AbstractCrudBackendApiController This commit modifies the `AbstractCrudBackendApiController` class to handle unique key constraint violations when saving entities to the repository. It adds a try-catch block around the `repository.save(entity)` method and throws a `ResponseStatusException` with a `CONFLICT` status and a descriptive error message when a `UniqueConstraintException` is caught. Note: This commit message follows the established convention of using the "feat" type for new feature additions. * Fix checkstyleMain + checkstyleTest * feat: Update dRAGon description to mention RAG pipeline This commit updates the dRAGon description in the `HomepageFeatures` component to mention the RAG pipeline instead of just "your docs". This change provides more clarity and aligns with the actual functionality of the software. * refactor: Update primary color variables in custom.css This commit updates the primary color variables in the custom.css file. The new color values are #8b5cf6 for the primary color, #743cf4 for the dark shade, #692cf3 for the darker shade, #4d0ce0 for the darkest shade, #a27cf8 for the light shade, #ad8cf9 for the lighter shade, and #cfbcfb for the lightest shade. This change ensures consistency and aligns with the desired color scheme. * refactor: Update ProviderEntity settings field description * refactor: Update SiloEntity vectorStoreClass field to vectorStoreType This commit updates the `SiloEntity` class by renaming the `vectorStoreClass` field to `vectorStoreType`. This change provides a more accurate and descriptive name for the field, improving code readability and maintainability. * refactor: Update EmbeddingModelType enum with dimensions and maxTokens This commit updates the `EmbeddingModelType` enum in the `EmbeddingModelType.java` file. It adds the `dimensions` and `maxTokens` fields to the `EmbeddingModelDefinition` class, allowing for more flexibility in defining the dimensions and maximum tokens for each embedding model. This change enhances the configurability of the embedding models and improves the overall functionality of the application. * refactor: Add IngestorEntity settings field for key-value pairs This commit adds a new field `settings` to the `IngestorEntity` class. The `settings` field is a map of key-value pairs and allows for linking additional settings to the Ingestor. This change enhances the flexibility and configurability of the IngestorEntity. * refactor: Rename SiloEntity vectorStoreClass field to vectorStoreType * Fix checkstyleMain * refactor: Update AbstractCrudBackendApiController to use ObjectMapper for field updates This commit refactors the `AbstractCrudBackendApiController` class to use the `ObjectMapper` from the Jackson library for updating entity fields. Instead of manually iterating over the fields and using reflection to set the values, the code now serializes the `fields` map to JSON using the `ObjectMapper`, and then uses the `ObjectReader` to update the entity fields. This change improves code readability and maintainability by leveraging the built-in functionality of the `ObjectMapper` and reduces the risk of errors when updating fields. * refactor: Add ingestors field to SiloEntity This commit adds a new field `ingestors` to the `SiloEntity` class. The `ingestors` field is a list of Ingestor UUIDs that are linked to the Silo. When linked, the Ingestors will be run in order to add documents into the Silo. This change enhances the functionality of the SiloEntity by allowing it to manage and track the associated Ingestors. * refactor: Add showAppURLs and configureJobs methods to StartupApplicationListener * Begin JobRunr Ingestor Job * refactor: Remove @Profile("!test") annotation from JobRunrConfig The @Profile("!test") annotation was removed from the JobRunrConfig class. This change ensures that the configuration is applied regardless of the active profile, improving consistency and eliminating potential issues related to test environments. --- .devcontainer/devcontainer.json | 1 + backend/build.gradle | 27 ++++-- .../main/java/ai/dragon/config/AppConfig.java | 1 - .../java/ai/dragon/config/JobRunrConfig.java | 17 ++++ ...Configuration.java => SecurityConfig.java} | 2 +- .../AbstractCrudBackendApiController.java | 74 +++++++++++++++ .../IngestorBackendApiController.java | 49 +++------- .../ProviderBackendApiController.java | 48 +++------- .../repository/SiloBackendApiController.java | 46 +++------ .../java/ai/dragon/entity/IngestorEntity.java | 14 ++- .../java/ai/dragon/entity/ProviderEntity.java | 14 +-- .../java/ai/dragon/entity/SiloEntity.java | 26 ++++- .../enumeration/EmbeddingModelType.java | 95 +++++++++++++++++++ .../ai/dragon/enumeration/IngestorType.java | 25 +++++ .../ai/dragon/enumeration/ProviderType.java | 12 ++- .../dragon/enumeration/VectorStoreType.java | 25 +++++ .../listener/StartupApplicationListener.java | 43 +++++++++ .../dragon/repository/AbstractRepository.java | 4 +- .../ai/dragon/service/IngestorService.java | 23 +++++ .../java/ai/dragon/service/JobService.java | 40 ++++++++ .../src/main/resources/application-prod.yaml | 22 +++++ .../src/main/resources/application-test.yaml | 8 ++ backend/src/main/resources/application.yaml | 22 +++++ .../ai/dragon/DragonApplicationTests.java | 2 + .../enumeration/EmbeddingModelTypeTest.java | 29 ++++++ .../dragon/enumeration/ProviderTypeTest.java | 19 ++++ .../repository/ProviderRepositoryTest.java | 54 ++++------- .../src/components/HomepageFeatures/index.tsx | 2 +- docs/src/css/custom.css | 32 +++---- 29 files changed, 603 insertions(+), 173 deletions(-) create mode 100644 backend/src/main/java/ai/dragon/config/JobRunrConfig.java rename backend/src/main/java/ai/dragon/config/{SecurityConfiguration.java => SecurityConfig.java} (96%) create mode 100644 backend/src/main/java/ai/dragon/controller/api/backendapi/repository/AbstractCrudBackendApiController.java create mode 100644 backend/src/main/java/ai/dragon/enumeration/EmbeddingModelType.java create mode 100644 backend/src/main/java/ai/dragon/enumeration/IngestorType.java create mode 100644 backend/src/main/java/ai/dragon/enumeration/VectorStoreType.java create mode 100644 backend/src/main/java/ai/dragon/listener/StartupApplicationListener.java create mode 100644 backend/src/main/java/ai/dragon/service/IngestorService.java create mode 100644 backend/src/main/java/ai/dragon/service/JobService.java create mode 100644 backend/src/test/java/ai/dragon/enumeration/EmbeddingModelTypeTest.java create mode 100644 backend/src/test/java/ai/dragon/enumeration/ProviderTypeTest.java diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 80cba70a..a36f2ab2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,6 +13,7 @@ "postStartCommand": ".devcontainer/postStart.sh", "postCreateCommand": ".devcontainer/postCreate.sh", "forwardPorts": [ + 1984, 1985, 3000, 4173 diff --git a/backend/build.gradle b/backend/build.gradle index 1634e12b..98a1f240 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -20,26 +20,37 @@ repositories { } dependencies { - compileOnly 'org.projectlombok:lombok:1.18.32' - annotationProcessor 'org.projectlombok:lombok:1.18.32' annotationProcessor "org.springframework.boot:spring-boot-configuration-processor" implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-thymeleaf:3.2.5' implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.hibernate.validator:hibernate-validator' - implementation 'dev.langchain4j:langchain4j-open-ai:0.30.0' + developmentOnly 'org.springframework.boot:spring-boot-devtools' + implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.5.0' + + implementation 'dev.langchain4j:langchain4j-open-ai:0.31.0' + implementation 'dev.langchain4j:langchain4j:0.31.0' + implementation 'dev.langchain4j:langchain4j-core:0.31.0' + implementation 'dev.langchain4j:langchain4j-embeddings:0.31.0' + implementation 'dev.langchain4j:langchain4j-open-ai:0.31.0' + implementation 'dev.langchain4j:langchain4j-easy-rag:0.31.0' + implementation platform('org.dizitart:nitrite-bom:4.2.1') implementation 'org.dizitart:nitrite' implementation 'org.dizitart:nitrite-support' implementation 'org.dizitart:nitrite-mvstore-adapter' implementation 'org.dizitart:nitrite-jackson-mapper' - implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.5.0' + implementation 'net.sourceforge.argparse4j:argparse4j:0.9.0' - developmentOnly("org.springframework.boot:spring-boot-devtools") + implementation 'org.jobrunr:jobrunr-spring-boot-3-starter:7.1.2' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.security:spring-security-test' + + compileOnly 'org.projectlombok:lombok:1.18.32' + annotationProcessor 'org.projectlombok:lombok:1.18.32' testCompileOnly 'org.projectlombok:lombok:1.18.32' testAnnotationProcessor 'org.projectlombok:lombok:1.18.32' } @@ -49,12 +60,12 @@ application { } openApi { - groupedApiMappings.set(["http://localhost:1985/api/docs/backendapi": "swagger-backendapi.json", - "http://localhost:1985/api/docs/publicapi": "swagger-publicapi.json"]) + groupedApiMappings.set(['http://localhost:1985/api/docs/backendapi': 'swagger-backendapi.json', + 'http://localhost:1985/api/docs/publicapi': 'swagger-publicapi.json']) outputDir.set(file("$buildDir/docs")) waitTimeInSeconds.set(10) customBootRun { - args.set(["--spring.profiles.active=test"]) + args.set(['--spring.profiles.active=test']) } } diff --git a/backend/src/main/java/ai/dragon/config/AppConfig.java b/backend/src/main/java/ai/dragon/config/AppConfig.java index e4589eb0..be23dbd8 100644 --- a/backend/src/main/java/ai/dragon/config/AppConfig.java +++ b/backend/src/main/java/ai/dragon/config/AppConfig.java @@ -8,5 +8,4 @@ @Configuration @EnableConfigurationProperties(DataProperties.class) public class AppConfig { - } diff --git a/backend/src/main/java/ai/dragon/config/JobRunrConfig.java b/backend/src/main/java/ai/dragon/config/JobRunrConfig.java new file mode 100644 index 00000000..61730b78 --- /dev/null +++ b/backend/src/main/java/ai/dragon/config/JobRunrConfig.java @@ -0,0 +1,17 @@ +package ai.dragon.config; + +import org.jobrunr.jobs.mappers.JobMapper; +import org.jobrunr.storage.InMemoryStorageProvider; +import org.jobrunr.storage.StorageProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobRunrConfig { + @Bean + public StorageProvider storageProvider(JobMapper jobMapper) { + InMemoryStorageProvider storageProvider = new InMemoryStorageProvider(); + storageProvider.setJobMapper(jobMapper); + return storageProvider; + } +} diff --git a/backend/src/main/java/ai/dragon/config/SecurityConfiguration.java b/backend/src/main/java/ai/dragon/config/SecurityConfig.java similarity index 96% rename from backend/src/main/java/ai/dragon/config/SecurityConfiguration.java rename to backend/src/main/java/ai/dragon/config/SecurityConfig.java index 2a901a32..a4a93019 100644 --- a/backend/src/main/java/ai/dragon/config/SecurityConfiguration.java +++ b/backend/src/main/java/ai/dragon/config/SecurityConfig.java @@ -8,7 +8,7 @@ import org.springframework.security.web.SecurityFilterChain; @Configuration(proxyBeanMethods = false) -public class SecurityConfiguration { +public class SecurityConfig { @Bean @Order(Ordered.HIGHEST_PRECEDENCE) SecurityFilterChain securityFilterChain(HttpSecurity http) throws Exception { diff --git a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/AbstractCrudBackendApiController.java b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/AbstractCrudBackendApiController.java new file mode 100644 index 00000000..a6bcf229 --- /dev/null +++ b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/AbstractCrudBackendApiController.java @@ -0,0 +1,74 @@ +package ai.dragon.controller.api.backendapi.repository; + +import java.lang.reflect.Constructor; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import org.dizitart.no2.exceptions.UniqueConstraintException; +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; + +import ai.dragon.entity.IAbstractEntity; +import ai.dragon.repository.AbstractRepository; + +abstract class AbstractCrudBackendApiController { + public T update(String uuid, Map fields, AbstractRepository repository) { + T entityToUpdate = repository.getByUuid(uuid); + if (entityToUpdate == null) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Entity not found"); + } + try { + ObjectMapper objectMapper = new ObjectMapper(); + String fieldsAsJson = objectMapper.writeValueAsString(fields); + ObjectReader objectReader = objectMapper.readerForUpdating(entityToUpdate); + entityToUpdate = objectReader.readValue(fieldsAsJson); + } catch (Exception ex) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Error parsing fields as JSON", ex); + } + if (!uuid.equals(entityToUpdate.getUuid().toString())) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "UUID must not be changed."); + } + repository.save(entityToUpdate); + return entityToUpdate; + } + + public List list(AbstractRepository repository) { + return repository.find().toList(); + } + + public T create(AbstractRepository repository) throws Exception { + return create(repository, null); + } + + public T create(AbstractRepository repository, Function beforeSaveCallback) throws Exception { + Class clazz = repository.getGenericSuperclass(); + Constructor cons = clazz.getConstructor(); + T entity = cons.newInstance(); + if (beforeSaveCallback != null) { + entity = beforeSaveCallback.apply(entity); + } + try { + repository.save(entity); + } catch (UniqueConstraintException ex) { + throw new ResponseStatusException(HttpStatus.CONFLICT, "Unique key constraint violation", ex); + } + return entity; + } + + public T get(String uuid, AbstractRepository repository) { + return Optional.ofNullable(repository.getByUuid(uuid)) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Entity not found.")); + } + + public void delete(String uuid, AbstractRepository repository) { + if (!repository.exists(uuid)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Entity not found."); + } + repository.delete(uuid); + } +} diff --git a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/IngestorBackendApiController.java b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/IngestorBackendApiController.java index c051f886..8b0d2e75 100644 --- a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/IngestorBackendApiController.java +++ b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/IngestorBackendApiController.java @@ -1,22 +1,19 @@ package ai.dragon.controller.api.backendapi.repository; -import java.time.LocalDateTime; import java.util.List; -import java.util.Optional; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; import ai.dragon.entity.IngestorEntity; import ai.dragon.repository.IngestorRepository; @@ -29,7 +26,7 @@ @RestController @RequestMapping("/api/backendapi/repository/ingestor") @Tag(name = "Ingestor Repository", description = "Ingestor Repository Management API Endpoints") -public class IngestorBackendApiController { +public class IngestorBackendApiController extends AbstractCrudBackendApiController { @Autowired private IngestorRepository ingestorRepository; @@ -37,47 +34,34 @@ public class IngestorBackendApiController { @ApiResponse(responseCode = "200", description = "List has been successfully retrieved.") @Operation(summary = "List all Ingestors", description = "Returns all Ingestor entities stored in the database.") public List list() { - return ingestorRepository.find().toList(); + return super.list(ingestorRepository); } @PostMapping("/") @ApiResponse(responseCode = "200", description = "Ingestor has been successfully created.") + @ApiResponse(responseCode = "409", description = "Constraint violation.", content = @Content) @Operation(summary = "Create a new Ingestor", description = "Creates one Ingestor entity in the database.") - public IngestorEntity create() { - String ingestorName = String.format("Ingestor %s", - LocalDateTime.now().format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"))); - - IngestorEntity ingestor = new IngestorEntity(); - ingestor.setName(ingestorName); - ingestorRepository.save(ingestor); - return ingestor; + public IngestorEntity create() throws Exception { + return super.create(ingestorRepository); } @GetMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @ApiResponse(responseCode = "200", description = "Ingestor has been successfully retrieved.") @ApiResponse(responseCode = "404", description = "Ingestor not found.", content = @Content) @Operation(summary = "Retrieve one Ingestor", description = "Returns one Ingestor entity from its UUID stored in the database.") - public IngestorEntity get(@PathVariable("uuid") @Parameter(description = "Identifier of the Ingestor") String uuid) { - return Optional.ofNullable(ingestorRepository.getByUuid(uuid)) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Ingestor not found.")); + public IngestorEntity get( + @PathVariable("uuid") @Parameter(description = "Identifier of the Ingestor") String uuid) { + return super.get(uuid, ingestorRepository); } @PatchMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @ApiResponse(responseCode = "200", description = "Ingestor has been successfully created.") @ApiResponse(responseCode = "404", description = "Ingestor not found.", content = @Content) @Operation(summary = "Update a Ingestor", description = "Updates one Ingestor entity in the database.") - public IngestorEntity update(@PathVariable("uuid") @Parameter(description = "Identifier of the Ingestor", required = true) String uuid, - IngestorEntity ingestor) throws JsonMappingException { - if (!ingestorRepository.exists(uuid) || ingestor == null || !uuid.equals(ingestor.getUuid().toString())) { - throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Ingestor not found"); - } - - IngestorEntity ingestorToUpdate = ingestorRepository.getByUuid(uuid); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.updateValue(ingestorToUpdate, ingestor); - ingestorRepository.save(ingestorToUpdate); - - return ingestorToUpdate; + public IngestorEntity update( + @PathVariable("uuid") @Parameter(description = "Identifier of the Ingestor", required = true) String uuid, + @RequestBody Map fields) throws JsonMappingException { + return super.update(uuid, fields, ingestorRepository); } @DeleteMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -85,9 +69,6 @@ public IngestorEntity update(@PathVariable("uuid") @Parameter(description = "Ide @ApiResponse(responseCode = "404", description = "Ingestor not found.", content = @Content) @Operation(summary = "Delete a Ingestor", description = "Deletes one Ingestor entity from its UUID stored in the database.") public void delete(@PathVariable("uuid") @Parameter(description = "Identifier of the Ingestor") String uuid) { - if (!ingestorRepository.exists(uuid)) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Ingestor not found."); - } - ingestorRepository.delete(uuid); + super.delete(uuid, ingestorRepository); } } diff --git a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/ProviderBackendApiController.java b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/ProviderBackendApiController.java index bde7f93b..41219cbf 100644 --- a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/ProviderBackendApiController.java +++ b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/ProviderBackendApiController.java @@ -1,23 +1,20 @@ package ai.dragon.controller.api.backendapi.repository; -import java.time.LocalDateTime; import java.util.List; -import java.util.Optional; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; import ai.dragon.entity.ProviderEntity; import ai.dragon.enumeration.ProviderType; @@ -31,7 +28,7 @@ @RestController @RequestMapping("/api/backendapi/repository/provider") @Tag(name = "Provider Repository", description = "Provider Repository Management API Endpoints") -public class ProviderBackendApiController { +public class ProviderBackendApiController extends AbstractCrudBackendApiController { @Autowired private ProviderRepository providerRepository; @@ -39,22 +36,20 @@ public class ProviderBackendApiController { @ApiResponse(responseCode = "200", description = "List has been successfully retrieved.") @Operation(summary = "List all Providers", description = "Returns all Provider entities stored in the database.") public List list() { - return providerRepository.find().toList(); + return super.list(providerRepository); } @PostMapping("/") @ApiResponse(responseCode = "200", description = "Provider has been successfully created.") + @ApiResponse(responseCode = "409", description = "Constraint violation.", content = @Content) @Operation(summary = "Create a new Provider", description = "Creates one Provider entity in the database.") public ProviderEntity create( - @RequestParam(name = "type", required = true) @Parameter(description = "Type of the provider") String providerType) { - String providerName = String.format("Provider %s", - LocalDateTime.now().format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"))); - - ProviderEntity provider = new ProviderEntity(); - provider.setName(providerName); - provider.setType(ProviderType.valueOf(providerType)); - providerRepository.save(provider); - return provider; + @RequestParam(name = "type", required = true) @Parameter(description = "Type of the provider") String providerType) + throws Exception { + return super.create(providerRepository, provider -> { + provider.setType(ProviderType.fromString(providerType)); + return provider; + }); } @GetMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -63,8 +58,7 @@ public ProviderEntity create( @Operation(summary = "Retrieve one Provider", description = "Returns one Provider entity from its UUID stored in the database.") public ProviderEntity get( @PathVariable("uuid") @Parameter(description = "Identifier of the Provider") String uuid) { - return Optional.ofNullable(providerRepository.getByUuid(uuid)) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Provider not found.")); + return super.get(uuid, providerRepository); } @PatchMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -73,17 +67,8 @@ public ProviderEntity get( @Operation(summary = "Update a Provider", description = "Updates one Provider entity in the database.") public ProviderEntity update( @PathVariable("uuid") @Parameter(description = "Identifier of the Provider", required = true) String uuid, - ProviderEntity provider) throws JsonMappingException { - if (!providerRepository.exists(uuid) || provider == null || !uuid.equals(provider.getUuid().toString())) { - throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Provider not found"); - } - - ProviderEntity providerToUpdate = providerRepository.getByUuid(uuid); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.updateValue(providerToUpdate, provider); - providerRepository.save(providerToUpdate); - - return providerToUpdate; + @RequestBody Map fields) throws JsonMappingException { + return super.update(uuid, fields, providerRepository); } @DeleteMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -91,9 +76,6 @@ public ProviderEntity update( @ApiResponse(responseCode = "404", description = "Provider not found.", content = @Content) @Operation(summary = "Delete a Provider", description = "Deletes one Provider entity from its UUID stored in the database.") public void delete(@PathVariable("uuid") @Parameter(description = "Identifier of the Provider") String uuid) { - if (!providerRepository.exists(uuid)) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Provider not found."); - } - providerRepository.delete(uuid); + super.delete(uuid, providerRepository); } } diff --git a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/SiloBackendApiController.java b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/SiloBackendApiController.java index d5c91591..318f8bc0 100644 --- a/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/SiloBackendApiController.java +++ b/backend/src/main/java/ai/dragon/controller/api/backendapi/repository/SiloBackendApiController.java @@ -1,22 +1,19 @@ package ai.dragon.controller.api.backendapi.repository; -import java.time.LocalDateTime; import java.util.List; -import java.util.Optional; +import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PatchMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.server.ResponseStatusException; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; import ai.dragon.entity.SiloEntity; import ai.dragon.repository.SiloRepository; @@ -29,7 +26,7 @@ @RestController @RequestMapping("/api/backendapi/repository/silo") @Tag(name = "Silo Repository", description = "Silo Repository Management API Endpoints") -public class SiloBackendApiController { +public class SiloBackendApiController extends AbstractCrudBackendApiController { @Autowired private SiloRepository siloRepository; @@ -37,20 +34,15 @@ public class SiloBackendApiController { @ApiResponse(responseCode = "200", description = "List has been successfully retrieved.") @Operation(summary = "List all Silos", description = "Returns all Silo entities stored in the database.") public List list() { - return siloRepository.find().toList(); + return super.list(siloRepository); } @PostMapping("/") @ApiResponse(responseCode = "200", description = "Silo has been successfully created.") + @ApiResponse(responseCode = "409", description = "Constraint violation.", content = @Content) @Operation(summary = "Create a new Silo", description = "Creates one Silo entity in the database.") - public SiloEntity create() { - String siloName = String.format("Silo %s", - LocalDateTime.now().format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"))); - - SiloEntity silo = new SiloEntity(); - silo.setName(siloName); - siloRepository.save(silo); - return silo; + public SiloEntity create() throws Exception { + return super.create(siloRepository); } @GetMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -58,26 +50,17 @@ public SiloEntity create() { @ApiResponse(responseCode = "404", description = "Silo not found.", content = @Content) @Operation(summary = "Retrieve one Silo", description = "Returns one Silo entity from its UUID stored in the database.") public SiloEntity get(@PathVariable("uuid") @Parameter(description = "Identifier of the Silo") String uuid) { - return Optional.ofNullable(siloRepository.getByUuid(uuid)) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Silo not found.")); + return super.get(uuid, siloRepository); } @PatchMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @ApiResponse(responseCode = "200", description = "Silo has been successfully created.") @ApiResponse(responseCode = "404", description = "Silo not found.", content = @Content) @Operation(summary = "Update a Silo", description = "Updates one Silo entity in the database.") - public SiloEntity update(@PathVariable("uuid") @Parameter(description = "Identifier of the Silo", required = true) String uuid, - SiloEntity silo) throws JsonMappingException { - if (!siloRepository.exists(uuid) || silo == null || !uuid.equals(silo.getUuid().toString())) { - throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Silo not found"); - } - - SiloEntity siloToUpdate = siloRepository.getByUuid(uuid); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.updateValue(siloToUpdate, silo); - siloRepository.save(siloToUpdate); - - return siloToUpdate; + public SiloEntity update( + @PathVariable("uuid") @Parameter(description = "Identifier of the Silo", required = true) String uuid, + @RequestBody Map fields) throws JsonMappingException { + return super.update(uuid, fields, siloRepository); } @DeleteMapping("/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}") @@ -85,9 +68,6 @@ public SiloEntity update(@PathVariable("uuid") @Parameter(description = "Identif @ApiResponse(responseCode = "404", description = "Silo not found.", content = @Content) @Operation(summary = "Delete a Silo", description = "Deletes one Silo entity from its UUID stored in the database.") public void delete(@PathVariable("uuid") @Parameter(description = "Identifier of the Silo") String uuid) { - if (!siloRepository.exists(uuid)) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Silo not found."); - } - siloRepository.delete(uuid); + super.delete(uuid, siloRepository); } } diff --git a/backend/src/main/java/ai/dragon/entity/IngestorEntity.java b/backend/src/main/java/ai/dragon/entity/IngestorEntity.java index 3043cebb..34dc6991 100644 --- a/backend/src/main/java/ai/dragon/entity/IngestorEntity.java +++ b/backend/src/main/java/ai/dragon/entity/IngestorEntity.java @@ -1,10 +1,15 @@ package ai.dragon.entity; +import java.util.Map; import java.util.UUID; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; +import org.dizitart.no2.repository.annotations.Indices; +import ai.dragon.enumeration.IngestorType; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -12,6 +17,7 @@ @Entity(value = "ingestor") @Schema(name = "Ingestor", description = "Ingestor Entity") +@Indices({ @Index(fields = "name", type = IndexType.UNIQUE) }) @Getter @Setter public class IngestorEntity implements IAbstractEntity { @@ -21,9 +27,15 @@ public class IngestorEntity implements IAbstractEntity { private UUID uuid; @NotNull - @Schema(description = "Name of the Ingestor") + @Schema(description = "Name of the Ingestor. Must be unique.") private String name; + @Schema(description = "Type of the Ingestor") + private IngestorType type; + + @Schema(description = "Settings to be linked to the Ingestor (if applicable) in the form of key-value pairs.") + private Map settings; + public IngestorEntity() { this.uuid = UUID.randomUUID(); this.name = String.format("Ingestor %s", this.uuid.toString()); diff --git a/backend/src/main/java/ai/dragon/entity/ProviderEntity.java b/backend/src/main/java/ai/dragon/entity/ProviderEntity.java index ec95f4e2..72d479ad 100644 --- a/backend/src/main/java/ai/dragon/entity/ProviderEntity.java +++ b/backend/src/main/java/ai/dragon/entity/ProviderEntity.java @@ -3,8 +3,11 @@ import java.util.Map; import java.util.UUID; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; +import org.dizitart.no2.repository.annotations.Indices; import ai.dragon.enumeration.ProviderType; import io.swagger.v3.oas.annotations.media.Schema; @@ -15,6 +18,7 @@ @Entity(value = "provider") @Schema(name = "Provider", description = "Provider Entity") +@Indices({ @Index(fields = "name", type = IndexType.UNIQUE), @Index(fields = "type", type = IndexType.UNIQUE) }) @Getter @Setter public class ProviderEntity implements IAbstractEntity { @@ -25,17 +29,15 @@ public class ProviderEntity implements IAbstractEntity { @NotNull @NotBlank - @Schema(description = "Name of the Provider") + @Schema(description = "Name of the Provider. Must be unique.") private String name; @NotNull - @Schema(description = "Type of the Provider") + @Schema(description = "Type of the Provider. Must be unique.") private ProviderType type; - @Schema(description = """ - Headers to be sent to the Provider (if applicable) in the form of key-value pairs. - Could be used for authentication with API keys, tokens, etc.""") - private Map httpHeaders; + @Schema(description = "Settings to be linked to the Provider (if applicable) in the form of key-value pairs.") + private Map settings; public ProviderEntity() { this.uuid = UUID.randomUUID(); diff --git a/backend/src/main/java/ai/dragon/entity/SiloEntity.java b/backend/src/main/java/ai/dragon/entity/SiloEntity.java index d155333b..a2ecfafd 100644 --- a/backend/src/main/java/ai/dragon/entity/SiloEntity.java +++ b/backend/src/main/java/ai/dragon/entity/SiloEntity.java @@ -1,10 +1,17 @@ package ai.dragon.entity; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; +import org.dizitart.no2.index.IndexType; import org.dizitart.no2.repository.annotations.Entity; import org.dizitart.no2.repository.annotations.Id; +import org.dizitart.no2.repository.annotations.Index; +import org.dizitart.no2.repository.annotations.Indices; +import ai.dragon.enumeration.EmbeddingModelType; +import ai.dragon.enumeration.VectorStoreType; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -12,6 +19,7 @@ @Entity(value = "silo") @Schema(name = "Silo", description = "Silo Entity") +@Indices({ @Index(fields = "name", type = IndexType.UNIQUE) }) @Getter @Setter public class SiloEntity implements IAbstractEntity { @@ -21,11 +29,27 @@ public class SiloEntity implements IAbstractEntity { private UUID uuid; @NotNull - @Schema(description = "Name of the Silo") + @Schema(description = "Name of the Silo. Must be unique.") private String name; + @NotNull + @Schema(description = "Java Class to be used for the Vector Store", example = "InMemoryEmbeddingStore") + private VectorStoreType vectorStoreType; + + @NotNull + @Schema(description = "Type to be used for the Embedding Model", example = "BgeSmallEnV15QuantizedEmbeddingModel") + private EmbeddingModelType embeddingModelType; + + @Schema(description = """ + List of Ingestor UUIDs to be linked to the Silo. When linked to a Silo, + Ingestors will be run in order to add documents into the Silo.""") + private List ingestors; + public SiloEntity() { this.uuid = UUID.randomUUID(); this.name = String.format("Silo %s", this.uuid.toString()); + this.vectorStoreType = VectorStoreType.InMemoryEmbeddingStore; + this.embeddingModelType = EmbeddingModelType.BgeSmallEnV15QuantizedEmbeddingModel; + this.ingestors = new ArrayList(); } } diff --git a/backend/src/main/java/ai/dragon/enumeration/EmbeddingModelType.java b/backend/src/main/java/ai/dragon/enumeration/EmbeddingModelType.java new file mode 100644 index 00000000..2f57adb1 --- /dev/null +++ b/backend/src/main/java/ai/dragon/enumeration/EmbeddingModelType.java @@ -0,0 +1,95 @@ +package ai.dragon.enumeration; + +import java.util.List; + +import dev.langchain4j.model.openai.OpenAiEmbeddingModelName; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +public enum EmbeddingModelType { + BgeSmallEnV15QuantizedEmbeddingModel("BgeSmallEnV15QuantizedEmbeddingModel"), + OpenAiEmbeddingAda002Model("OpenAiEmbeddingAda002Model"), + OpenAiEmbedding3SmallModel("OpenAiEmbedding3SmallModel"), + OpenAiEmbedding3LargeModel("OpenAiEmbedding3LargeModel"); + + private String value; + + EmbeddingModelType(String value) { + this.value = value; + } + + public static EmbeddingModelType fromString(String text) { + for (EmbeddingModelType b : EmbeddingModelType.values()) { + if (b.value.equalsIgnoreCase(text)) { + return b; + } + } + return null; + } + + public EmbeddingModelDefinition getModelDefinition() throws ClassNotFoundException { + switch (this) { + case BgeSmallEnV15QuantizedEmbeddingModel: + return EmbeddingModelDefinition + .newInstance() + .embeddingModelClassName( + "dev.langchain4j.model.embedding.bge.small.en.v15.BgeSmallEnV15QuantizedEmbeddingModel") + .embeddingModelName("BgeSmallEnV15QuantizedEmbeddingModel") + .providerType(ProviderType.ONNX) + .dimensions(384) + .maxTokens(512); + case OpenAiEmbeddingAda002Model: + return EmbeddingModelDefinition + .newInstance() + .embeddingModelClassName("dev.langchain4j.model.openai.OpenAiEmbeddingModel") + .embeddingModelName(OpenAiEmbeddingModelName.TEXT_EMBEDDING_ADA_002.toString()) + .providerType(ProviderType.OpenAI) + .dimensions(1536) + .maxTokens(8191); + case OpenAiEmbedding3SmallModel: + return EmbeddingModelDefinition + .newInstance() + .embeddingModelClassName("dev.langchain4j.model.openai.OpenAiEmbeddingModel") + .embeddingModelName(OpenAiEmbeddingModelName.TEXT_EMBEDDING_3_SMALL.toString()) + .providerType(ProviderType.OpenAI) + .dimensions(1536) + .maxTokens(8191); + case OpenAiEmbedding3LargeModel: + return EmbeddingModelDefinition + .newInstance() + .embeddingModelClassName("dev.langchain4j.model.openai.OpenAiEmbeddingModel") + .embeddingModelName(OpenAiEmbeddingModelName.TEXT_EMBEDDING_3_LARGE.toString()) + .providerType(ProviderType.OpenAI) + .dimensions(3072) + .maxTokens(8191); + default: + throw new ClassNotFoundException("Model not found"); + } + } + + @Override + public String toString() { + return value; + } +} + +@Getter +@Setter +@Accessors(fluent = true) +class EmbeddingModelDefinition { + private String embeddingModelClassName; + private String embeddingModelName; + private List languages; + private ProviderType providerType; + private int dimensions; + private int maxTokens; + + private EmbeddingModelDefinition() { + this.languages = List.of("en"); + } + + public static EmbeddingModelDefinition newInstance() { + return new EmbeddingModelDefinition(); + } +} diff --git a/backend/src/main/java/ai/dragon/enumeration/IngestorType.java b/backend/src/main/java/ai/dragon/enumeration/IngestorType.java new file mode 100644 index 00000000..c865b6f2 --- /dev/null +++ b/backend/src/main/java/ai/dragon/enumeration/IngestorType.java @@ -0,0 +1,25 @@ +package ai.dragon.enumeration; + +public enum IngestorType { + LOCAL("LOCAL"); + + private String value; + + IngestorType(String value) { + this.value = value; + } + + public static IngestorType fromString(String text) { + for (IngestorType ingestor : IngestorType.values()) { + if (ingestor.value.equalsIgnoreCase(text)) { + return ingestor; + } + } + return null; + } + + @Override + public String toString() { + return value; + } +} diff --git a/backend/src/main/java/ai/dragon/enumeration/ProviderType.java b/backend/src/main/java/ai/dragon/enumeration/ProviderType.java index 634d551c..d1cec135 100644 --- a/backend/src/main/java/ai/dragon/enumeration/ProviderType.java +++ b/backend/src/main/java/ai/dragon/enumeration/ProviderType.java @@ -1,7 +1,8 @@ package ai.dragon.enumeration; public enum ProviderType { - OPENAI("openai"); + ONNX("ONNX"), + OpenAI("OpenAI"); private String value; @@ -9,6 +10,15 @@ public enum ProviderType { this.value = value; } + public static ProviderType fromString(String text) { + for (ProviderType provider : ProviderType.values()) { + if (provider.value.equalsIgnoreCase(text)) { + return provider; + } + } + return null; + } + @Override public String toString() { return value; diff --git a/backend/src/main/java/ai/dragon/enumeration/VectorStoreType.java b/backend/src/main/java/ai/dragon/enumeration/VectorStoreType.java new file mode 100644 index 00000000..5533184b --- /dev/null +++ b/backend/src/main/java/ai/dragon/enumeration/VectorStoreType.java @@ -0,0 +1,25 @@ +package ai.dragon.enumeration; + +public enum VectorStoreType { + InMemoryEmbeddingStore("InMemoryEmbeddingStore"); + + private String value; + + VectorStoreType(String value) { + this.value = value; + } + + public static VectorStoreType fromString(String text) { + for (VectorStoreType vectorStore : VectorStoreType.values()) { + if (vectorStore.value.equalsIgnoreCase(text)) { + return vectorStore; + } + } + return null; + } + + @Override + public String toString() { + return value; + } +} diff --git a/backend/src/main/java/ai/dragon/listener/StartupApplicationListener.java b/backend/src/main/java/ai/dragon/listener/StartupApplicationListener.java new file mode 100644 index 00000000..8bac78f8 --- /dev/null +++ b/backend/src/main/java/ai/dragon/listener/StartupApplicationListener.java @@ -0,0 +1,43 @@ +package ai.dragon.listener; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Profile; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +import ai.dragon.service.JobService; + +@Component +@Profile("!test") +public class StartupApplicationListener implements ApplicationListener { + @Autowired + private ServletWebServerApplicationContext webServerAppCtxt; + + @Autowired + private JobService jobService; + + @Override + public void onApplicationEvent(@NonNull ContextRefreshedEvent event) { + showAppURLs(); + configureJobs(); + } + + private void showAppURLs() { + String scheme = "http"; + String host = "localhost"; + int port = webServerAppCtxt.getWebServer().getPort(); + + System.out.println("================================================"); + System.out.println(String.format("APP URL\t\t : %s://%s:%d/", scheme, host, port)); + System.out.println(String.format("Swagger UI\t : %s://%s:%d/api/swagger-ui.html", scheme, host, port)); + System.out.println(String.format("JobRunr\t\t : %s://%s:%d/", scheme, host, 1984)); + System.out.println("================================================"); + } + + private void configureJobs() { + jobService.onApplicationStartup(); + } +} diff --git a/backend/src/main/java/ai/dragon/repository/AbstractRepository.java b/backend/src/main/java/ai/dragon/repository/AbstractRepository.java index 84bd701e..de526b23 100644 --- a/backend/src/main/java/ai/dragon/repository/AbstractRepository.java +++ b/backend/src/main/java/ai/dragon/repository/AbstractRepository.java @@ -21,7 +21,7 @@ import jakarta.validation.Validator; @Component -abstract class AbstractRepository { +public abstract class AbstractRepository { @Autowired private DatabaseService databaseService; @@ -121,7 +121,7 @@ public void unsubscribe(CollectionEventListener listener) { } @SuppressWarnings("unchecked") - private Class getGenericSuperclass() { + public Class getGenericSuperclass() { ParameterizedType superclass = (ParameterizedType) getClass().getGenericSuperclass(); return (Class) superclass.getActualTypeArguments()[0]; diff --git a/backend/src/main/java/ai/dragon/service/IngestorService.java b/backend/src/main/java/ai/dragon/service/IngestorService.java new file mode 100644 index 00000000..084c7e0c --- /dev/null +++ b/backend/src/main/java/ai/dragon/service/IngestorService.java @@ -0,0 +1,23 @@ +package ai.dragon.service; + +import org.jobrunr.jobs.annotations.Job; +import org.jobrunr.jobs.annotations.Recurring; +import org.jobrunr.jobs.context.JobContext; +import org.jobrunr.jobs.context.JobRunrDashboardLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +@Service +public class IngestorService { + private final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(this.getClass())); + + public static final String INGESTOR_RECURRING_JOB_ID = "ingestor-recurring-job"; + + @Recurring(id = INGESTOR_RECURRING_JOB_ID, cron = "* * * * *") + @Job(name = "Ingestor Recurring Job") + public void executeSampleJob(JobContext jobContext) { + logger.info("testinfo"); + jobContext.logger().info("new info"); + } +} diff --git a/backend/src/main/java/ai/dragon/service/JobService.java b/backend/src/main/java/ai/dragon/service/JobService.java new file mode 100644 index 00000000..4253627f --- /dev/null +++ b/backend/src/main/java/ai/dragon/service/JobService.java @@ -0,0 +1,40 @@ +package ai.dragon.service; + +import org.jobrunr.jobs.Job; +import org.jobrunr.jobs.RecurringJob; +import org.jobrunr.storage.JobNotFoundException; +import org.jobrunr.storage.RecurringJobsResult; +import org.jobrunr.storage.StorageProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class JobService { + private RecurringJobsResult recurringJobsResult; + + @Autowired + private StorageProvider storageProvider; + + public void onApplicationStartup() { + this.triggerRecurringJob(IngestorService.INGESTOR_RECURRING_JOB_ID); + } + + public void triggerRecurringJob(String id) { + final RecurringJob recurringJob = recurringJobResults() + .stream() + .filter(rj -> rj.getId().equals(id)) + .findFirst() + .orElseThrow(() -> new JobNotFoundException(id)); + + final Job job = recurringJob.toEnqueuedJob(); + storageProvider.save(job); + } + + private RecurringJobsResult recurringJobResults() { + if (recurringJobsResult == null + || storageProvider.recurringJobsUpdated(recurringJobsResult.getLastModifiedHash())) { + recurringJobsResult = storageProvider.getRecurringJobs(); + } + return recurringJobsResult; + } +} diff --git a/backend/src/main/resources/application-prod.yaml b/backend/src/main/resources/application-prod.yaml index fb5050c3..a55d578a 100644 --- a/backend/src/main/resources/application-prod.yaml +++ b/backend/src/main/resources/application-prod.yaml @@ -8,6 +8,28 @@ springdoc: path: "/api/swagger-ui.html" api-docs: path: "/api/docs" +org: + jobrunr: + background-job-server: + enabled: true + #worker-count: 4 # Number of CPU cores + poll-interval-in-seconds: 5 # Check for new work every 5 seconds + delete-succeeded-jobs-after: 1 # Succeeded jobs will go to the deleted state after 1 hour + permanently-delete-deleted-jobs-after: 24 # Deleted jobs will be deleted permanently after 24 hours + metrics: + enabled: false + job-scheduler: + enabled: true + dashboard: + enabled: true + port: 1984 + jobs: + default-number-of-retries: 10 # Number of retries for a failing job + retry-back-off-time-seed: 3 # Time seed for the exponential back-off policy + metrics: + enabled: false + miscellaneous: + allow-anonymous-data-usage: false dragon: data: path: "/data" \ No newline at end of file diff --git a/backend/src/main/resources/application-test.yaml b/backend/src/main/resources/application-test.yaml index 823add99..2772de4a 100644 --- a/backend/src/main/resources/application-test.yaml +++ b/backend/src/main/resources/application-test.yaml @@ -8,6 +8,14 @@ springdoc: path: "/api/swagger-ui.html" api-docs: path: "/api/docs" +org: + jobrunr: + background-job-server: + enabled: false + job-scheduler: + enabled: false + dashboard: + enabled: false dragon: data: path: ":temp:" diff --git a/backend/src/main/resources/application.yaml b/backend/src/main/resources/application.yaml index d68254f2..5941a338 100644 --- a/backend/src/main/resources/application.yaml +++ b/backend/src/main/resources/application.yaml @@ -18,6 +18,28 @@ springdoc: path: "/api/swagger-ui.html" api-docs: path: "/api/docs" +org: + jobrunr: + background-job-server: + enabled: true + #worker-count: 4 # Number of CPU cores + poll-interval-in-seconds: 5 # Check for new work every 5 seconds + delete-succeeded-jobs-after: 1 # Succeeded jobs will go to the deleted state after 1 hour + permanently-delete-deleted-jobs-after: 24 # Deleted jobs will be deleted permanently after 24 hours + metrics: + enabled: false + job-scheduler: + enabled: true + dashboard: + enabled: true + port: 1984 + jobs: + default-number-of-retries: 10 # Number of retries for a failing job + retry-back-off-time-seed: 3 # Time seed for the exponential back-off policy + metrics: + enabled: false + miscellaneous: + allow-anonymous-data-usage: false dragon: data: path: "/tmp/dragon_data" \ No newline at end of file diff --git a/backend/src/test/java/ai/dragon/DragonApplicationTests.java b/backend/src/test/java/ai/dragon/DragonApplicationTests.java index 60896218..41971a89 100644 --- a/backend/src/test/java/ai/dragon/DragonApplicationTests.java +++ b/backend/src/test/java/ai/dragon/DragonApplicationTests.java @@ -2,8 +2,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; @SpringBootTest +@ActiveProfiles("test") class DragonApplicationTests { @Test void contextLoads() { diff --git a/backend/src/test/java/ai/dragon/enumeration/EmbeddingModelTypeTest.java b/backend/src/test/java/ai/dragon/enumeration/EmbeddingModelTypeTest.java new file mode 100644 index 00000000..e31f3546 --- /dev/null +++ b/backend/src/test/java/ai/dragon/enumeration/EmbeddingModelTypeTest.java @@ -0,0 +1,29 @@ +package ai.dragon.enumeration; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +@SpringBootTest +@ActiveProfiles("test") +public class EmbeddingModelTypeTest { + @Test + void testModelsExist() throws ClassNotFoundException { + EmbeddingModelType[] models = EmbeddingModelType.values(); + for (EmbeddingModelType model : models) { + EmbeddingModelDefinition embeddingModelDefinition = model.getModelDefinition(); + assertNotNull(Class.forName(embeddingModelDefinition.embeddingModelClassName())); + assertNotNull(embeddingModelDefinition.embeddingModelClassName()); + assertNotNull(embeddingModelDefinition.providerType()); + assertNotNull(embeddingModelDefinition.dimensions()); + assertNotEquals(0, embeddingModelDefinition.dimensions()); + assertNotNull(embeddingModelDefinition.maxTokens()); + assertNotEquals(0, embeddingModelDefinition.maxTokens()); + assertNotNull(embeddingModelDefinition.languages()); + assertNotEquals(0, embeddingModelDefinition.languages().size()); + } + } +} diff --git a/backend/src/test/java/ai/dragon/enumeration/ProviderTypeTest.java b/backend/src/test/java/ai/dragon/enumeration/ProviderTypeTest.java new file mode 100644 index 00000000..cc3afed0 --- /dev/null +++ b/backend/src/test/java/ai/dragon/enumeration/ProviderTypeTest.java @@ -0,0 +1,19 @@ +package ai.dragon.enumeration; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +@SpringBootTest +@ActiveProfiles("test") +public class ProviderTypeTest { + @Test + void testModelsExist() { + ProviderType[] models = ProviderType.values(); + for (ProviderType model : models) { + assertNotNull(ProviderType.fromString(model.toString())); + } + } +} diff --git a/backend/src/test/java/ai/dragon/repository/ProviderRepositoryTest.java b/backend/src/test/java/ai/dragon/repository/ProviderRepositoryTest.java index a543891b..30602c4f 100644 --- a/backend/src/test/java/ai/dragon/repository/ProviderRepositoryTest.java +++ b/backend/src/test/java/ai/dragon/repository/ProviderRepositoryTest.java @@ -28,7 +28,7 @@ void insertProviders() { ProviderEntity provider = new ProviderEntity(); provider.setUuid(UUID.randomUUID()); provider.setName(providerName); - provider.setType(ProviderType.OPENAI); + provider.setType(ProviderType.ONNX); providerRepository.save(provider); ProviderEntity retrievedProvider = providerRepository.getByUuid(provider.getUuid()); @@ -39,55 +39,41 @@ void insertProviders() { @Test void findAllProviders() { providerRepository.deleteAll(); - int nbProvidersToInsert = 3; - - for (int i = 0; i < nbProvidersToInsert; i++) { - ProviderEntity provider = new ProviderEntity(); - provider.setName(String.format("Provider %d", i)); - provider.setType(ProviderType.OPENAI); - providerRepository.save(provider); - } - - assertEquals(nbProvidersToInsert, providerRepository.countAll()); + ProviderEntity provider = new ProviderEntity(); + provider.setName("ONNX Provider"); + provider.setType(ProviderType.ONNX); + providerRepository.save(provider); + assertEquals(1, providerRepository.countAll()); - providerRepository.find().forEach(provider -> { - assertNotNull(provider.getUuid()); + providerRepository.find().forEach(providerFetched -> { + assertNotNull(providerFetched.getUuid()); }); } @Test void findProvidersByName() { providerRepository.deleteAll(); - int nbProvidersToInsert = 3; - - for (int i = 0; i < nbProvidersToInsert; i++) { - ProviderEntity provider = new ProviderEntity(); - provider.setName(String.format("Provider %d", i)); - provider.setType(ProviderType.OPENAI); - providerRepository.save(provider); - } - - assertEquals(nbProvidersToInsert, providerRepository.countAll()); + ProviderEntity provider = new ProviderEntity(); + provider.setName("OpenAI Provider"); + provider.setType(ProviderType.OpenAI); + providerRepository.save(provider); - Cursor cursor = providerRepository.findByFieldValue("name", "Provider 1"); + Cursor cursor = providerRepository.findByFieldValue("name", "OpenAI Provider"); assertEquals(1, cursor.size()); } @Test void deleteProviders() { providerRepository.deleteAll(); - int nbProvidersToInsert = 3; - - for (int i = 0; i < nbProvidersToInsert; i++) { - ProviderEntity provider = new ProviderEntity(); - provider.setType(ProviderType.OPENAI); - providerRepository.save(provider); - } + ProviderEntity provider = new ProviderEntity(); + provider.setName("OpenAI Provider"); + provider.setType(ProviderType.OpenAI); + providerRepository.save(provider); - assertEquals(nbProvidersToInsert, providerRepository.countAll()); + assertEquals(1, providerRepository.countAll()); - providerRepository.find().forEach(provider -> { - providerRepository.delete(provider.getUuid()); + providerRepository.find().forEach(providerFetched -> { + providerRepository.delete(providerFetched.getUuid()); }); assertEquals(0, providerRepository.countAll()); diff --git a/docs/src/components/HomepageFeatures/index.tsx b/docs/src/components/HomepageFeatures/index.tsx index e64b1382..a9e21009 100644 --- a/docs/src/components/HomepageFeatures/index.tsx +++ b/docs/src/components/HomepageFeatures/index.tsx @@ -21,7 +21,7 @@ const FeatureList: FeatureItem[] = [ title: 'Focus on What Matters', description: ( <> - dRAGon lets you focus on your docs, and we'll do the chores. Go + dRAGon lets you focus on your RAG pipeline, and we'll do the chores. Go ahead and import your docs into the pipeline and let dRAGon do the rest. ), diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index 2bc6a4cf..88572088 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -6,25 +6,23 @@ /* You can override the default Infima variables here. */ :root { - --ifm-color-primary: #2e8555; - --ifm-color-primary-dark: #29784c; - --ifm-color-primary-darker: #277148; - --ifm-color-primary-darkest: #205d3b; - --ifm-color-primary-light: #33925d; - --ifm-color-primary-lighter: #359962; - --ifm-color-primary-lightest: #3cad6e; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); + --ifm-color-primary: #8b5cf6; + --ifm-color-primary-dark: #743cf4; + --ifm-color-primary-darker: #692cf3; + --ifm-color-primary-darkest: #4d0ce0; + --ifm-color-primary-light: #a27cf8; + --ifm-color-primary-lighter: #ad8cf9; + --ifm-color-primary-lightest: #cfbcfb; } /* For readability concerns, you should choose a lighter palette in dark mode. */ [data-theme='dark'] { - --ifm-color-primary: #25c2a0; - --ifm-color-primary-dark: #21af90; - --ifm-color-primary-darker: #1fa588; - --ifm-color-primary-darkest: #1a8870; - --ifm-color-primary-light: #29d5b0; - --ifm-color-primary-lighter: #32d8b4; - --ifm-color-primary-lightest: #4fddbf; + --ifm-color-primary: #8b5cf6; + --ifm-color-primary-dark: #743cf4; + --ifm-color-primary-darker: #692cf3; + --ifm-color-primary-darkest: #4d0ce0; + --ifm-color-primary-light: #a27cf8; + --ifm-color-primary-lighter: #ad8cf9; + --ifm-color-primary-lightest: #cfbcfb; --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); -} +} \ No newline at end of file