EGDM-379 : DataPull - Build index of production job configuration

CHANGELOG.md

@@ -4,6 +4,35 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html)
+## [0.1.48] - 2021-11-30
+
+The new end point added. **/getSampleInputJson**
+This endpoint help the user to fetch the existing input configuration based on source and destination.
+
+### Theory
+
+At the start of application all the input json will be pulled in the background asynchronously. 
+The json will be indexed in the cache based on source , destination and source/destination.
+Every time a new input is received the cache is updated as it saved in s3. 
+
+### Changed
+* api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java 
+* api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java 
+* api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java 
+* api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java 
+* api/src/main/java/com/homeaway/datapullclient/input/Destination.java 
+* api/src/main/java/com/homeaway/datapullclient/input/Migration.java 
+* api/src/main/java/com/homeaway/datapullclient/input/Source.java 
+* api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java 
+* docs/mkdocs.yml
+
+### Created New
+* api/src/main/java/com/homeaway/datapullclient/config/StoredInputProviderConfig.java
+* api/src/main/java/com/homeaway/datapullclient/input/InputConfiguration.java
+* api/src/main/java/com/homeaway/datapullclient/process/SampleInputFetchImpl.java
+* api/src/main/java/com/homeaway/datapullclient/service/SampleInputFetchService.java
+* api/src/main/java/com/homeaway/datapullclient/util/InputJsonHelper.java
+* docs/docs/fetching_existing_json.md
 
 ## [0.1.47] - 2021-11-17
 

api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java

@@ -79,4 +79,15 @@ public interface DataPullClientApi {
  @ApiImplicitParam(name = "pipelinename", value = "pipelinename", required = true, dataType = "String", paramType = "query")
  })
  SimpleResponseEntity startSimpleDataPull(@RequestParam("pipelinename") String pipelinename , @RequestParam("awsenv") String awenv);
+
+ @ApiOperation(value = "Given source or destination platform the api returns the existing ran sample config json.",
+ produces = MediaType.APPLICATION_JSON_VALUE, response = ResponseEntity.class
+ , notes = "Given either source/s or destination Or both we can fetch the existing json. ", nickname = "sampleInput")
+ @ResponseStatus(value = HttpStatus.ACCEPTED)
+ @RequestMapping(value = "/getSampleInputJson", method = GET, produces = MediaType.APPLICATION_JSON_VALUE)
+ @ApiImplicitParams({
+ @ApiImplicitParam(name = "sources", value = "Specify one or more comma seperated sources for the config being search.", required = false, dataType = "String", paramType = "query"),
+ @ApiImplicitParam(name = "destination", value = "Specify the destination for the config to be searched", required = false, dataType = "String", paramType = "query")
+ })
+ ResponseEntity getSampleInputJson(@RequestParam("sources") String sources, @RequestParam("destination") String destination);
 }

api/src/main/java/com/homeaway/datapullclient/config/StoredInputProviderConfig.java

@@ -0,0 +1,94 @@
+package com.homeaway.datapullclient.config;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.homeaway.datapullclient.exception.InputException;
+import com.homeaway.datapullclient.input.InputConfiguration;
+import com.homeaway.datapullclient.util.InputJsonHelper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.Async;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+
+@Slf4j
+@Configuration
+public class StoredInputProviderConfig {
+ private Map<String, Set<InputConfiguration>> source;
+ private Map<String, Set<InputConfiguration>> destination;
+ private Map<String, Set<InputConfiguration>> srcDestMap;
+ private final DataPullProperties dataPullProperties;
+ private final AmazonS3 amazonS3;
+ private final String s3Bucket;
+ private static final String DATAPULL_HISTORY_FOLDER = "datapull-opensource/history";
+ private final ObjectMapper mapper;
+
+ @Autowired
+ public StoredInputProviderConfig(
+ ObjectMapper mapper, DataPullProperties dataPullProperties, AmazonS3 amazonS3) {
+ this.mapper = Objects.requireNonNull(mapper, "ObjectMapper can't be null.");
+ this.dataPullProperties =
+ Objects.requireNonNull(dataPullProperties, "dataPullProperties can't be null.");
+ this.amazonS3 = Objects.requireNonNull(amazonS3, "amazonS3 can't be null.");
+ this.s3Bucket = dataPullProperties.getS3BucketName();
+ this.source = new HashMap<>();
+ this.destination = new HashMap<>();
+ this.srcDestMap = new HashMap<>();
+ }
+
+ @PostConstruct
+ public void initMaps() {
+ log.info("Existing input getting cache.");
+ populateMaps(InputJsonHelper.fetchAllInputConf(amazonS3, s3Bucket, DATAPULL_HISTORY_FOLDER));
+ }
+
+ public Set<InputConfiguration> getSample(String inputKeyType, String key) {
+ switch (inputKeyType) {
+ case "srcdest":
+ return srcDestMap.getOrDefault(key, new HashSet<>());
+ case "src":
+ return source.getOrDefault(key, new HashSet<>());
+ case "dest":
+ return destination.getOrDefault(key, new HashSet<>());
+ default:
+ throw new InputException("Asked key or key type not found.");
+ }
+ }
+
+ public static boolean isJSONValid(ObjectMapper mapper, String json) {
+ try {
+ mapper.readTree(json);
+ return true;
+ } catch (JsonMappingException e) {
+ log.error("Error : Invalid json :\n{}", json);
+ log.error("Error : Invalid json :", e);
+ return false;
+ } catch (JsonProcessingException e) {
+ log.error("Error : Invalid json :\n{}", json);
+ log.error("Error : Invalid json :", e);
+ return false;
+ }
+ }
+
+ private void populateMaps(List<String> jsons) {
+ jsons.stream()
+ .parallel()
+ .filter(j -> isJSONValid(mapper, j))
+ .forEach(e -> processCacheStore(e));
+ }
+
+ @Async
+ public void processCacheStore(String json) {
+ try {
+ InputJsonHelper.process(mapper, json, source, destination, srcDestMap);
+ } catch (JsonProcessingException e) {
+ log.error("Error json:: {}", json);
+ log.error("Error while creating index. ", e);
+ throw new RuntimeException("error while creating index. ");
+ }
+ }
+}

api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java

@@ -21,23 +21,25 @@
 import org.springframework.http.HttpStatus;
 
 @ApiModel(value = "Response entity for DataPull")
-public class ResponseEntity {
+public class ResponseEntity<T> {
 
  @ApiModelProperty("HTTP code for response")
  private int statusCode = HttpStatus.OK.value();
 
- @ApiModelProperty("Response Message")
- private String message;
+ @ApiModelProperty("Response Body")
+ private T t;
 
- public ResponseEntity(int status, String statusCode) {
+
+ public ResponseEntity(int status, T t) {
  this.statusCode = status;
- this.message = statusCode;
+ this.t = t;
  }
+
  public int getStatus() {
  return statusCode;
  }
 
- public String getMessage() {
- return message;
+ public T getMessage() {
+ return t;
  }
 }

api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java

@@ -21,7 +21,9 @@
 import com.homeaway.datapullclient.data.SimpleResponseEntity;
 import com.homeaway.datapullclient.exception.InputException;
 import com.homeaway.datapullclient.exception.ProcessingException;
+import com.homeaway.datapullclient.input.InputConfiguration;
 import com.homeaway.datapullclient.service.DataPullClientService;
+import com.homeaway.datapullclient.service.SampleInputFetchService;
 import io.swagger.annotations.ApiResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,13 +33,16 @@
 import org.springframework.http.RequestEntity;
 import org.springframework.web.bind.annotation.RestController;
 import java.util.Map;
+import java.util.Set;
 
 @Slf4j
 @RestController
 public class DataPullRequestHandler implements DataPullClientApi {
 
  @Autowired
  private DataPullClientService service;
+ @Autowired
+ private SampleInputFetchService inputFetchService;
 
  @Override
  public ResponseEntity startDataPull(HttpEntity<String> inputJson) {
@@ -83,6 +88,25 @@ public SimpleResponseEntity startSimpleDataPull(String pipelinename, String awse
 
  return entity;
  }
+ @Override
+ public ResponseEntity<?> getSampleInputJson(String sources, String destination) {
+ log.info("Source : {} && Destination : {}", sources, destination);
+ ResponseEntity entity = null;
+ try {
+ Set<InputConfiguration> response = inputFetchService.getSampleInputConfs(sources, destination);
+ entity =
+ new ResponseEntity(
+ response != null && !response.isEmpty()
+ ? HttpStatus.OK.value()
+ : HttpStatus.NO_CONTENT.value(),
+ response);
+ } catch (InputException exception) {
+ entity = new ResponseEntity(HttpStatus.BAD_REQUEST.value(), exception.getMessage());
+ } catch (Exception exception) {
+ entity = new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), exception.getMessage());
+ }
+ return entity;
+ }
 }
 
 

api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java

@@ -16,15 +16,18 @@
 
 package com.homeaway.datapullclient.input;
 
-import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 import java.util.HashMap;
 import java.util.Map;
 
 @Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class ClusterProperties {
 
@@ -111,7 +114,19 @@ public class ClusterProperties {
 
  private String env;
 
- @Override
+ @JsonIgnore
+ private Map<String, Object> additionalProperties = new HashMap<>();
+ @JsonAnyGetter
+ public Map<String, Object> otherFields() {
+ return additionalProperties;
+ }
+
+ @JsonAnySetter
+ public void setOtherField(String name, Object value) {
+ additionalProperties.put(name, value);
+ }
+
+ /* @Override
  public String toString() {
  return "ClusterProperties{" +
  "pipelineName='" + pipelineName + '\'' +
@@ -140,5 +155,5 @@ public String toString() {
  ", forcerestart='" + forceRestart + '\'' +
  ", application='" + application + '\'' +
  '}';
- }
+ }*/
 }

api/src/main/java/com/homeaway/datapullclient/input/Destination.java

@@ -16,11 +16,18 @@
 
 package com.homeaway.datapullclient.input;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import java.util.HashMap;
+import java.util.Map;
 
 @Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class Destination {
  @JsonProperty("platform")
@@ -58,8 +65,18 @@ public class Destination {
 
  @JsonProperty("jksfiles")
  private String[] jksfiles;
+ @JsonIgnore
+ private Map<String, Object> additionalProperties = new HashMap<>();
+ @JsonAnyGetter
+ public Map<String, Object> otherFields() {
+ return additionalProperties;
+ }
 
- @Override
+ @JsonAnySetter
+ public void setOtherField(String name, Object value) {
+ additionalProperties.put(name, value);
+ }
+ /*@Override
  public String toString() {
  return "Destination{" +
  "platform='" + platform + '\'' +
@@ -74,5 +91,5 @@ public String toString() {
  ", database='" + database + '\'' +
  ", jksfilepath='" + jksfiles + '\'' +
  '}';
- }
+ }*/
 }

api/src/main/java/com/homeaway/datapullclient/input/InputConfiguration.java

@@ -0,0 +1,39 @@
+package com.homeaway.datapullclient.input;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+@Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InputConfiguration {
+ @JsonProperty("useremailaddress")
+ private String userEmailAddress;
+ @JsonProperty("failureemailaddress")
+ private String failureEmailAddress;
+ @JsonProperty("migration_failure_threshold")
+ private String migrationFailureThreshold;
+ @JsonProperty("parallelmigrations")
+ private Boolean parallelMigrations;
+ @JsonProperty("no_of_retries")
+ private String noOfRetries;
+ @JsonProperty("precisecounts")
+ private Boolean preciseCounts;
+ @JsonProperty("migrations")
+ private Set<Migration> migrations;
+ @JsonProperty("cluster")
+ private ClusterProperties clusterProperties;
+ @JsonIgnore
+ private Map<String, Object> additionalProperties = new HashMap<>();
+}