Datapull emailnotification feature

CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html)
 
+## [0.1.79] - 2024-04-25
+Implemented a new functionality to send email notifications regarding the initiation of EMR cluster spin-up during the start of datapull jobs, covering both success and failure scenarios.
+### Changed
+- api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java
+- api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java
+- api/src/main/java/com/homeaway/datapullclient/config/DataPullClientConfig.java
+### Added
+- api/src/main/java/com/homeaway/datapullclient/config/SESProperties.java
+- api/src/main/java/com/homeaway/datapullclient/config/SMTPProperties.java
+- api/src/main/java/com/homeaway/datapullclient/utils/EmailNotification.java
+
+## [0.1.78] - 2024-04-22
+Mandating the Linux flavor of the docker image that's been used in ECR.
+### Changed
+- API/terraform/datapull_task/ecs_deploy.sh
+
+## [0.1.77] - 2024-04-10
+Fixing the null subnetId issue by adding StringUtils.isNotBlank(dataPullProperties.getApplicationSubnet3())
+### Changed
+- api/src/main/java/com/homeaway/datapullclient/process/DataPullTask.java
 
 ## [0.1.76] - 2024-01-08
 Fixing the issue when the key is not provided by the user. and reverting chnages w.r.t insertinto hive table for partitioned table. 

api/src/main/java/com/homeaway/datapullclient/config/DataPullClientConfig.java

@@ -24,17 +24,23 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.homeaway.datapullclient.process.DataPullTask;
+import com.homeaway.datapullclient.utils.EmailNotification;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.context.annotation.*;
 import org.springframework.core.env.Environment;
 
 @Slf4j
 @Data
 @Configuration("dataPullConfig")
 @PropertySources(@PropertySource("classpath:application.yml"))
+//
+//@SpringBootApplication
+//@ComponentScan("com.homeaway.datapullclient")
+
 public class DataPullClientConfig {
 
  @Autowired
@@ -71,6 +77,18 @@ public EMRProperties getEmrProperties(){
  return new EMRProperties();
  }
 
+
+ @Bean
+ public SMTPProperties getSMTPProperties(){
+ return new SMTPProperties();
+ }
+
+
+ @Bean
+ public SESProperties getSESProperties(){
+ return new SESProperties();
+ }
+
  @Bean
  public AmazonElasticMapReduce getEMRClient(){
  AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
@@ -79,4 +97,5 @@ public AmazonElasticMapReduce getEMRClient(){
  .withCredentials(credentialsProvider)
  .build());
  }
+
 }

api/src/main/java/com/homeaway/datapullclient/config/SESProperties.java

@@ -0,0 +1,29 @@
+
+package com.homeaway.datapullclient.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix="datapull.email.ses")
+@EnableConfigurationProperties
+@Data
+
+public class SESProperties {
+
+ @Value("${region:}")
+ private String region;
+
+ @Value("${email:}")
+ private String email;
+
+ @Value("${access_key:}")
+ private String accessKey;
+
+ @Value("${secret_key:}")
+ private String secretKey;
+
+}

api/src/main/java/com/homeaway/datapullclient/config/SMTPProperties.java

@@ -0,0 +1,29 @@
+package com.homeaway.datapullclient.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix="datapull.email.smtp")
+@EnableConfigurationProperties
+@Data
+
+public class SMTPProperties {
+
+ @Value("${emailaddress:}")
+ private String emailaddress;
+
+ @Value("${smtpserveraddress:}")
+ private String smtpserveraddress;
+
+ @Value("${port:}")
+ private String smtpport;
+
+ @Value("${starttls:}")
+ private String smtpstarttls;
+
+
+}

api/src/main/java/com/homeaway/datapullclient/process/DataPullRequestProcessor.java

@@ -34,6 +34,7 @@
 import com.homeaway.datapullclient.input.Source;
 import com.homeaway.datapullclient.service.DataPullClientService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.everit.json.schema.Schema;
 import org.everit.json.schema.ValidationException;
 import org.everit.json.schema.loader.SchemaLoader;
@@ -84,6 +85,9 @@ public class DataPullRequestProcessor implements DataPullClientService {
 
  private final ThreadPoolTaskScheduler scheduler;
 
+ HashMap<String,String> successEmails = new HashMap<>();
+
+ HashMap<String,String> failureEmails = new HashMap<>();
  public DataPullRequestProcessor(){
  scheduler = new ThreadPoolTaskScheduler();
  scheduler.setPoolSize(POOL_SIZE);
@@ -123,6 +127,10 @@ public void runSimpleDataPull(String awsenv, String pipelinename) {
  }
 
  private void runDataPull(String json, boolean isStart, boolean validateJson) throws ProcessingException {
+
+ String userEmail;
+ String failureEmail;
+
  String originalInputJson = json;
  json = extractUserJsonFromS3IfProvided(json, isStart);
 
@@ -138,6 +146,26 @@ private void runDataPull(String json, boolean isStart, boolean validateJson) thr
 
  log.info("Running datapull for json : " + json + " cron expression = " + isStart + "env =" + env);
  final ObjectNode node = new ObjectMapper().readValue(json, ObjectNode.class);
+
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ JsonNode jsonNode = objectMapper.readTree(json);
+ userEmail = null != jsonNode.get("useremailaddress") ? jsonNode.get("useremailaddress").asText(): "";
+
+ String taskId = jsonNode.get("cluster").get("awsenv").asText().concat("-emr-").concat(jsonNode.get("cluster").get("pipelinename").asText()).concat("-pipeline");
+ successEmails.put(taskId,userEmail);
+
+ JsonNode failureEmailNode = jsonNode.get("failureemailaddress");
+ if(StringUtils.isNotEmpty(userEmail)){
+ failureEmail = (failureEmailNode != null) ? userEmail.concat(",").concat(failureEmailNode.asText()): userEmail;
+
+ }else {
+ failureEmail = (failureEmailNode != null) ? (failureEmailNode.asText()): "";
+
+ }
+ failureEmails.put(taskId,failureEmail);
+
+
  List<Map.Entry<String, JsonNode>> result = new LinkedList<Map.Entry<String, JsonNode>>();
  Iterator<Map.Entry<String, JsonNode>> nodes = node.fields();
  while(nodes.hasNext()){
@@ -197,6 +225,16 @@ private void runDataPull(String json, boolean isStart, boolean validateJson) thr
  log.debug("runDataPull <- return");
  }
 
+ public HashMap<String, String> successMailAddress() throws ProcessingException {
+ return successEmails;
+ }
+
+
+ public HashMap<String, String> failureMailAddress() throws ProcessingException {
+ return failureEmails;
+ }
+
+
  private StringBuilder createBootstrapString(Object[] paths, String bootstrapActionStringFromUser) throws ProcessingException {
 
  StringBuilder stringBuilder = new StringBuilder();