Local Ingestor Loader (#26)

* 🚧 Using LangChain4J Document class instead of IngestorDocument

* 🚧 Improving FileSystemIngestor

* refactor: Update jobrunr-spring-boot-3-starter dependency to version 7.2.0

* 🚧 Improving FileSystemIngestor

* 🚧 Improving FileSystemIngestor

* 🚧 Improving FileSystemIngestor

* ❌ Begin RAG Api Ask endpoint

* feat: Include dRAGon picture to README.md

* 🚧 Test entrypoint for querying the embedding store

* feat: Add Swagger configuration for RAG API

* Refactoring directories locations

* Refactoring directories locations

* Fixing ingest : only one document at a time

* Remove Silo link inside Farm when deleted

* Fix build test

* Ability to rebuild Silo from API Command

* Renaming AskRagApi to SearchRagApi

README.md

@@ -3,4 +3,6 @@
 
 dRAGon is a multi-modal RAG engine which can search over multiple kind of documents to generate good quality answers for LLMs.
 
-[Your journey to the dRAGon starts here!](https://dragon.okinawa)
+[Your journey to the dRAGon starts here!](https://dragon.okinawa)
+
+![dRAGon](https://dragon.okinawa/img/dragon_okinawa.jpg)

backend/build.gradle

@@ -43,9 +43,10 @@ dependencies {
 	implementation 'org.dizitart:nitrite-jackson-mapper'
 
 	implementation 'net.sourceforge.argparse4j:argparse4j:0.9.0'
-	implementation 'org.jobrunr:jobrunr-spring-boot-3-starter:7.1.2'
+	implementation 'org.jobrunr:jobrunr-spring-boot-3-starter:7.2.0'
 
 	testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+	testImplementation 'org.awaitility:awaitility:4.2.1'
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
 	testImplementation 'org.springframework.security:spring-security-test'
 
@@ -61,7 +62,8 @@ application {
 
 openApi {
 	groupedApiMappings.set(['http://localhost:1985/api/docs/backendapi': 'swagger-backendapi.json',
-                            'http://localhost:1985/api/docs/publicapi': 'swagger-publicapi.json'])
+                            'http://localhost:1985/api/docs/publicapi': 'swagger-publicapi.json',
+							'http://localhost:1985/api/docs/ragapi': 'swagger-ragapi.json'])
     outputDir.set(file("$buildDir/docs"))
     waitTimeInSeconds.set(10)
     customBootRun {

backend/src/main/java/ai/dragon/command/DatabaseExportCommand.java

@@ -30,6 +30,6 @@ public CommandLineExecutionResultType runCommand(String... args) throws Exceptio
         databaseService.exportDatabase(ns.getString("output"));
         logger.info("Database export completed to : " + ns.getString("output"));
 
-        return CommandLineExecutionResultType.EXECUTED;
+        return CommandLineExecutionResultType.Executed;
     }
 }

backend/src/main/java/ai/dragon/config/SwaggerConfig.java

@@ -29,8 +29,15 @@ public GroupedOpenApi backendApi() {
     }
 
     @Bean
-    public OpenAPI backendOpenAPI() {
+    public GroupedOpenApi ragApi() {
+        return GroupedOpenApi.builder()
+                .group("ragapi")
+                .pathsToMatch("/api/ragapi/**")
+                .build();
+    }
 
+    @Bean
+    public OpenAPI openAPI() {
         return new OpenAPI()
                 .info(new Info().title("dRAGon API")
                         .description("dRAGon API Reference for Developers")

backend/src/main/java/ai/dragon/controller/api/backendapi/command/DatabaseCommandBackendApiController.java

@@ -18,7 +18,7 @@ public class DatabaseCommandBackendApiController {
     @Autowired
     private DatabaseService databaseService;
 
-    @PostMapping("/database/export")
+    @PostMapping("/export")
     @ApiResponse(responseCode = "200", description = "Database dump has been successfully created.")
     @Operation(summary = "Create a database export", description = "Creates a JSON database dump.")
     public void export(HttpServletResponse response) throws Exception {

backend/src/main/java/ai/dragon/controller/api/backendapi/command/SiloCommandBackendApiController.java

@@ -0,0 +1,30 @@
+package ai.dragon.controller.api.backendapi.command;
+
+import java.util.UUID;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import ai.dragon.service.SiloService;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@RequestMapping("/api/backendapi/command/silo")
+@Tag(name = "Silo Command", description = "Silo Command API Endpoints")
+public class SiloCommandBackendApiController {
+    @Autowired
+    private SiloService siloService;
+
+    @PostMapping("/rebuild/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}")
+    @ApiResponse(responseCode = "200", description = "Silo is being rebuilt.")
+    @Operation(summary = "Rebuild Silo", description = "This will recompute the embeddings of the Silo.")
+    public void export(@PathVariable("uuid") @Parameter(description = "Identifier of the Silo") UUID uuid) throws Exception {
+        siloService.rebuildSilo(uuid);
+    }
+}

backend/src/main/java/ai/dragon/controller/api/ragapi/SearchRagApiController.java

@@ -0,0 +1,35 @@
+package ai.dragon.controller.api.ragapi;
+
+import java.util.UUID;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import ai.dragon.service.EmbeddingStoreService;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@RequestMapping("/api/ragapi/search")
+@Tag(name = "Ask", description = "Search API Endpoints")
+public class SearchRagApiController {
+    @Autowired
+    private EmbeddingStoreService embeddingStoreService;
+
+    // TODO Silo OR Farm
+    @PostMapping("/documents/silo/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}")
+    @ApiResponse(responseCode = "200", description = "Documents have been successfully retrieved.")
+    @Operation(summary = "Search documents inside a Silo", description = "Search documents from the Silo.")
+    public void searchDocumentsInSilo(
+            @PathVariable("uuid") @Parameter(description = "Identifier of the Silo") UUID uuid,
+            @RequestBody String query)
+            throws Exception {
+        embeddingStoreService.query(uuid, query);
+    }
+}

backend/src/main/java/ai/dragon/entity/SiloEntity.java

@@ -10,7 +10,7 @@
 import org.dizitart.no2.repository.annotations.Indices;
 
 import ai.dragon.enumeration.EmbeddingModelType;
-import ai.dragon.enumeration.IngestorType;
+import ai.dragon.enumeration.IngestorLoaderType;
 import ai.dragon.enumeration.VectorStoreType;
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.constraints.NotNull;
@@ -43,22 +43,28 @@ public class SiloEntity implements AbstractEntity {
     private EmbeddingModelType embeddingModelType;
 
     @NotNull
-    @Schema(description = "Type of the Silo's Ingestor Type")
-    private IngestorType ingestorType;
+    @Schema(description = "Type of the Silo's Ingestor Loader Type")
+    private IngestorLoaderType ingestorLoaderType;
 
     @NotNull
     @Schema(description = "Cron Expression for the Silo's Ingestor Job", example = "Launch the Silo ingestor every 15 minutes : */15 * * * *")
     private String ingestorSchedule;
 
-    @Schema(description = "Settings to be linked to the Silo's Ingestor in the form of `key = value` pairs.")
-    private List<String> ingestorSettings;
+    @Schema(description = "Settings to be linked to the Silo's Vector Store in the form of `key = value` pairs.")
+    private List<String> vectorStoreSettings;
+
+    @Schema(description = "Settings to be linked to the Silo's Embedding Model in the form of `key = value` pairs.")
+    private List<String> embeddingModelSettings;
+
+    @Schema(description = "Settings to be linked to the Silo's Ingestor Loader in the form of `key = value` pairs.")
+    private List<String> ingestorLoaderSettings;
 
     public SiloEntity() {
         this.uuid = UUID.randomUUID();
         this.name = String.format("Silo %s", this.uuid.toString());
         this.vectorStoreType = VectorStoreType.InMemoryEmbeddingStore;
         this.embeddingModelType = EmbeddingModelType.BgeSmallEnV15QuantizedEmbeddingModel;
-        this.ingestorType = IngestorType.LOCAL;
+        this.ingestorLoaderType = IngestorLoaderType.FileSystem;
         this.ingestorSchedule = DEFAULT_CRON_EXPRESSION;
     }
 }

backend/src/main/java/ai/dragon/enumeration/CommandLineExecutionResultType.java

@@ -1,7 +1,7 @@
 package ai.dragon.enumeration;
 
 public enum CommandLineExecutionResultType {
-    EXECUTED,
-    BYPASS,
-    ERROR
+    Executed,
+    Bypass,
+    Error
 }