use of com.homeaway.datapullclient.input.ClusterProperties in project datapull by homeaway.
the class DataPullRequestProcessor method runDataPull.
private void runDataPull(String json, boolean isStart, boolean validateJson) throws ProcessingException {
String originalInputJson = json;
json = extractUserJsonFromS3IfProvided(json, isStart);
final EMRProperties emrProperties = this.config.getEmrProperties();
if (log.isDebugEnabled())
log.debug("runDataPull -> json = " + json + " isStart = " + isStart);
try {
if (validateJson) {
json = validateAndEnrich(json);
}
log.info("Running datapull for json : " + json + " cron expression = " + isStart + "env =" + env);
final ObjectNode node = new ObjectMapper().readValue(json, ObjectNode.class);
List<Map.Entry<String, JsonNode>> result = new LinkedList<Map.Entry<String, JsonNode>>();
Iterator<Map.Entry<String, JsonNode>> nodes = node.fields();
while (nodes.hasNext()) {
result.add(nodes.next());
}
JsonNode clusterNode = result.stream().filter(y -> y.getKey().equalsIgnoreCase("cluster")).map(x -> x.getValue()).findAny().get();
JsonNode migrationsNode = result.stream().filter(y -> y.getKey().equalsIgnoreCase("migrations")).map(x -> x.getValue()).findAny().get();
if (clusterNode == null)
throw new ProcessingException("Invalid Json!!! Cluster properties cannot be null");
String creator = node.has(CREATOR) ? node.findValue(CREATOR).asText() : "";
ObjectMapper mapper = new ObjectMapper();
ClusterProperties reader = mapper.treeToValue(clusterNode, ClusterProperties.class);
Migration[] myObjects = mapper.treeToValue(migrationsNode, Migration[].class);
String cronExp = Objects.toString(reader.getCronExpression(), "");
if (!cronExp.isEmpty())
cronExp = validateAndProcessCronExpression(cronExp);
String pipeline = Objects.toString(reader.getPipelineName(), UUID.randomUUID().toString());
String pipelineEnv = Objects.toString(reader.getAwsEnv(), env);
DataPullProperties dataPullProperties = config.getDataPullProperties();
String applicationHistoryFolder = dataPullProperties.getApplicationHistoryFolder();
String s3RepositoryBucketName = dataPullProperties.getS3BucketName();
String jobName = pipelineEnv + PIPELINE_NAME_DELIMITER + EMR + PIPELINE_NAME_DELIMITER + pipeline + PIPELINE_NAME_DELIMITER + PIPELINE_NAME_SUFFIX;
String applicationHistoryFolderPath = applicationHistoryFolder == null || applicationHistoryFolder.isEmpty() ? s3RepositoryBucketName + "/" + DATAPULL_HISTORY_FOLDER : applicationHistoryFolder;
String bootstrapFilePath = s3RepositoryBucketName + "/" + BOOTSTRAP_FOLDER;
String filePath = applicationHistoryFolderPath + "/" + jobName;
String bootstrapFile = jobName + ".sh";
String jksFilePath = bootstrapFilePath + "/" + bootstrapFile;
String bootstrapActionStringFromUser = Objects.toString(reader.getBootstrapactionstring(), "");
String defaultBootstrapString = emrProperties.getDefaultBootstrapString();
Boolean haveBootstrapAction = createBootstrapScript(myObjects, bootstrapFile, bootstrapFilePath, bootstrapActionStringFromUser, defaultBootstrapString);
DataPullTask task = createDataPullTask(filePath, jksFilePath, reader, jobName, creator, node.path("sparkjarfile").asText(), haveBootstrapAction);
if (!isStart) {
json = originalInputJson.equals(json) ? json : originalInputJson;
saveConfig(applicationHistoryFolderPath, jobName + ".json", json);
}
if (!isStart && tasksMap.containsKey(jobName))
cancelExistingTask(jobName);
if (!(isStart && cronExp.isEmpty())) {
Future<?> future = !cronExp.isEmpty() ? scheduler.schedule(task, new CronTrigger(cronExp)) : scheduler.schedule(task, new Date(System.currentTimeMillis() + 1 * 1000));
tasksMap.put(jobName, future);
}
} catch (IOException e) {
throw new ProcessingException("exception while starting datapull " + e.getLocalizedMessage());
}
if (log.isDebugEnabled())
log.debug("runDataPull <- return");
}
Aggregations