EGDM-379 : DataPull - Build index of production job configuration

CHANGELOG.md

@@ -4,6 +4,35 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html)
+## [0.1.48] - 2021-11-30
+
+The new end point added. **/getSampleInputJson**
+This endpoint help the user to fetch the existing input configuration based on source and destination.
+
+### Theory
+
+At the start of application all the input json will be pulled in the background asynchronously. 
+The json will be indexed in the cache based on source , destination and source/destination.
+Every time a new input is received the cache is updated as it saved in s3. 
+
+### Changed
+* api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java            
+* api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java              
+* api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java  
+* api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java          
+* api/src/main/java/com/homeaway/datapullclient/input/Destination.java                
+* api/src/main/java/com/homeaway/datapullclient/input/Migration.java                  
+* api/src/main/java/com/homeaway/datapullclient/input/Source.java                     
+* api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java 
+* docs/mkdocs.yml
+
+### Created New
+* api/src/main/java/com/homeaway/datapullclient/config/StoredInputProviderConfig.java
+* api/src/main/java/com/homeaway/datapullclient/input/InputConfiguration.java
+* api/src/main/java/com/homeaway/datapullclient/process/SampleInputFetchImpl.java
+* api/src/main/java/com/homeaway/datapullclient/service/SampleInputFetchService.java
+* api/src/main/java/com/homeaway/datapullclient/util/InputJsonHelper.java
+* docs/docs/fetching_existing_json.md
 
 ## [0.1.47] - 2021-11-17
 

api/src/main/java/com/homeaway/datapullclient/api/DataPullClientApi.java

@@ -79,4 +79,15 @@ public interface DataPullClientApi {
             @ApiImplicitParam(name = "pipelinename", value = "pipelinename", required = true, dataType = "String", paramType = "query")
     })
     SimpleResponseEntity startSimpleDataPull(@RequestParam("pipelinename") String pipelinename , @RequestParam("awsenv") String  awenv);
+
+    @ApiOperation(value = "Given source or destination platform the api returns the existing ran sample config json.",
+            produces = MediaType.APPLICATION_JSON_VALUE, response = ResponseEntity.class
+            , notes = "Given either source/s or destination Or both we can fetch the existing json. ", nickname = "sampleInput")
+    @ResponseStatus(value = HttpStatus.ACCEPTED)
+    @RequestMapping(value = "/getSampleInputJson", method = GET, produces = MediaType.APPLICATION_JSON_VALUE)
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "sources", value = "Specify one or more comma seperated sources for the config being search.", required = false, dataType = "String", paramType = "query"),
+            @ApiImplicitParam(name = "destination", value = "Specify the destination for the config to be searched", required = false, dataType = "String", paramType = "query")
+    })
+    ResponseEntity getSampleInputJson(@RequestParam("sources") String sources, @RequestParam("destination") String destination);
 }

api/src/main/java/com/homeaway/datapullclient/config/StoredInputProviderConfig.java

@@ -0,0 +1,94 @@
+package com.homeaway.datapullclient.config;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.homeaway.datapullclient.exception.InputException;
+import com.homeaway.datapullclient.input.InputConfiguration;
+import com.homeaway.datapullclient.util.InputJsonHelper;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.Async;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+
+@Slf4j
+@Configuration
+public class StoredInputProviderConfig {
+    private Map<String, Set<InputConfiguration>> source;
+    private Map<String, Set<InputConfiguration>> destination;
+    private Map<String, Set<InputConfiguration>> srcDestMap;
+    private final DataPullProperties dataPullProperties;
+    private final AmazonS3 amazonS3;
+    private final String s3Bucket;
+    private static final String DATAPULL_HISTORY_FOLDER = "datapull-opensource/history";
+    private final ObjectMapper mapper;
+
+    @Autowired
+    public StoredInputProviderConfig(
+            ObjectMapper mapper, DataPullProperties dataPullProperties, AmazonS3 amazonS3) {
+        this.mapper = Objects.requireNonNull(mapper, "ObjectMapper can't be null.");
+        this.dataPullProperties =
+                Objects.requireNonNull(dataPullProperties, "dataPullProperties can't be null.");
+        this.amazonS3 = Objects.requireNonNull(amazonS3, "amazonS3 can't be null.");
+        this.s3Bucket = dataPullProperties.getS3BucketName();
+        this.source = new HashMap<>();
+        this.destination = new HashMap<>();
+        this.srcDestMap = new HashMap<>();
+    }
+
+    @PostConstruct
+    public void initMaps() {
+        log.info("Existing input getting cache.");
+        populateMaps(InputJsonHelper.fetchAllInputConf(amazonS3, s3Bucket, DATAPULL_HISTORY_FOLDER));
+    }
+
+    public Set<InputConfiguration> getSample(String inputKeyType, String key) {
+        switch (inputKeyType) {
+            case "srcdest":
+                return srcDestMap.getOrDefault(key, new HashSet<>());
+            case "src":
+                return source.getOrDefault(key, new HashSet<>());
+            case "dest":
+                return destination.getOrDefault(key, new HashSet<>());
+            default:
+                throw new InputException("Asked key or key type not found.");
+        }
+    }
+
+    public static boolean isJSONValid(ObjectMapper mapper, String json) {
+        try {
+            mapper.readTree(json);
+            return true;
+        } catch (JsonMappingException e) {
+            log.error("Error : Invalid json :\n{}", json);
+            log.error("Error : Invalid json :", e);
+            return false;
+        } catch (JsonProcessingException e) {
+            log.error("Error : Invalid json :\n{}", json);
+            log.error("Error : Invalid json :", e);
+            return false;
+        }
+    }
+
+    private void populateMaps(List<String> jsons) {
+        jsons.stream()
+                .parallel()
+                .filter(j -> isJSONValid(mapper, j))
+                .forEach(e -> processCacheStore(e));
+    }
+
+    @Async
+    public void processCacheStore(String json) {
+        try {
+            InputJsonHelper.process(mapper, json, source, destination, srcDestMap);
+        } catch (JsonProcessingException e) {
+            log.error("Error json:: {}", json);
+            log.error("Error while creating index. ", e);
+            throw new RuntimeException("error while creating index. ");
+        }
+    }
+}

api/src/main/java/com/homeaway/datapullclient/data/ResponseEntity.java

@@ -21,23 +21,25 @@
 import org.springframework.http.HttpStatus;
 
 @ApiModel(value = "Response entity for DataPull")
-public class ResponseEntity {
+public class ResponseEntity<T> {
 
     @ApiModelProperty("HTTP code for response")
     private int statusCode = HttpStatus.OK.value();
 
-    @ApiModelProperty("Response Message")
-    private String message;
+    @ApiModelProperty("Response Body")
+    private T t;
 
-    public ResponseEntity(int status, String statusCode) {
+
+    public ResponseEntity(int status, T t) {
         this.statusCode = status;
-        this.message = statusCode;
+        this.t = t;
     }
+
     public int getStatus() {
         return statusCode;
     }
 
-    public String getMessage() {
-        return message;
+    public T getMessage() {
+        return t;
     }
 }

api/src/main/java/com/homeaway/datapullclient/handlers/DataPullRequestHandler.java

@@ -21,7 +21,9 @@
 import com.homeaway.datapullclient.data.SimpleResponseEntity;
 import com.homeaway.datapullclient.exception.InputException;
 import com.homeaway.datapullclient.exception.ProcessingException;
+import com.homeaway.datapullclient.input.InputConfiguration;
 import com.homeaway.datapullclient.service.DataPullClientService;
+import com.homeaway.datapullclient.service.SampleInputFetchService;
 import io.swagger.annotations.ApiResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,13 +33,16 @@
 import org.springframework.http.RequestEntity;
 import org.springframework.web.bind.annotation.RestController;
 import java.util.Map;
+import java.util.Set;
 
 @Slf4j
 @RestController
 public class DataPullRequestHandler implements DataPullClientApi {
 
     @Autowired
     private DataPullClientService service;
+    @Autowired
+    private SampleInputFetchService inputFetchService;
 
     @Override
     public ResponseEntity startDataPull(HttpEntity<String> inputJson) {
@@ -83,6 +88,25 @@ public SimpleResponseEntity startSimpleDataPull(String pipelinename, String awse
 
         return entity;
     }
+    @Override
+    public ResponseEntity<?> getSampleInputJson(String sources, String destination) {
+        log.info("Source : {} && Destination : {}", sources, destination);
+        ResponseEntity entity = null;
+        try {
+            Set<InputConfiguration> response = inputFetchService.getSampleInputConfs(sources, destination);
+            entity =
+                    new ResponseEntity(
+                            response != null && !response.isEmpty()
+                                    ? HttpStatus.OK.value()
+                                    : HttpStatus.NO_CONTENT.value(),
+                            response);
+        } catch (InputException exception) {
+            entity = new ResponseEntity(HttpStatus.BAD_REQUEST.value(), exception.getMessage());
+        } catch (Exception exception) {
+            entity = new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), exception.getMessage());
+        }
+        return entity;
+    }
 }
 
 

api/src/main/java/com/homeaway/datapullclient/input/ClusterProperties.java

@@ -16,15 +16,18 @@
 
 package com.homeaway.datapullclient.input;
 
-import com.fasterxml.jackson.annotation.JsonAlias;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 import java.util.HashMap;
 import java.util.Map;
 
 @Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class ClusterProperties {
 
@@ -111,7 +114,19 @@ public class ClusterProperties {
 
     private String env;
 
-    @Override
+    @JsonIgnore
+    private Map<String, Object> additionalProperties = new HashMap<>();
+    @JsonAnyGetter
+    public Map<String, Object> otherFields() {
+        return additionalProperties;
+    }
+
+    @JsonAnySetter
+    public void setOtherField(String name, Object value) {
+        additionalProperties.put(name, value);
+    }
+
+   /* @Override
     public String toString() {
         return "ClusterProperties{" +
                 "pipelineName='" + pipelineName + '\'' +
@@ -140,5 +155,5 @@ public String toString() {
                 ", forcerestart='" + forceRestart + '\'' +
                 ", application='" + application + '\'' +
                 '}';
-    }
+    }*/
 }

api/src/main/java/com/homeaway/datapullclient/input/Destination.java

@@ -16,11 +16,18 @@
 
 package com.homeaway.datapullclient.input;
 
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import java.util.HashMap;
+import java.util.Map;
 
 @Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class Destination {
     @JsonProperty("platform")
@@ -58,8 +65,18 @@ public class Destination {
 
     @JsonProperty("jksfiles")
     private String[] jksfiles;
+    @JsonIgnore
+    private Map<String, Object> additionalProperties = new HashMap<>();
+    @JsonAnyGetter
+    public Map<String, Object> otherFields() {
+        return additionalProperties;
+    }
 
-    @Override
+    @JsonAnySetter
+    public void setOtherField(String name, Object value) {
+        additionalProperties.put(name, value);
+    }
+    /*@Override
     public String toString() {
         return "Destination{" +
                 "platform='" + platform + '\'' +
@@ -74,5 +91,5 @@ public String toString() {
                 ", database='" + database + '\'' +
                 ", jksfilepath='" + jksfiles + '\'' +
                 '}';
-    }
+    }*/
 }

api/src/main/java/com/homeaway/datapullclient/input/InputConfiguration.java

@@ -0,0 +1,39 @@
+package com.homeaway.datapullclient.input;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+@Data
+@EqualsAndHashCode
+@ToString
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class InputConfiguration {
+    @JsonProperty("useremailaddress")
+    private String userEmailAddress;
+    @JsonProperty("failureemailaddress")
+    private String failureEmailAddress;
+    @JsonProperty("migration_failure_threshold")
+    private String migrationFailureThreshold;
+    @JsonProperty("parallelmigrations")
+    private Boolean parallelMigrations;
+    @JsonProperty("no_of_retries")
+    private String noOfRetries;
+    @JsonProperty("precisecounts")
+    private Boolean preciseCounts;
+    @JsonProperty("migrations")
+    private Set<Migration> migrations;
+    @JsonProperty("cluster")
+    private ClusterProperties clusterProperties;
+    @JsonIgnore
+    private Map<String, Object> additionalProperties = new HashMap<>();
+}