use of com.epam.pipeline.exception.CmdExecutionException in project cloud-pipeline by epam.
the class AutoscaleManager method createNodeForRun.
private void createNodeForRun(List<CompletableFuture<Void>> tasks, String runId, RunInstance requiredInstance) {
long longId = Long.parseLong(runId);
addNodeUpTask(longId);
tasks.add(CompletableFuture.runAsync(() -> {
Instant start = Instant.now();
// save required instance
pipelineRunManager.updateRunInstance(longId, requiredInstance);
RunInstance instance = clusterManager.scaleUp(runId, requiredInstance);
// save instance ID and IP
pipelineRunManager.updateRunInstance(longId, instance);
Instant end = Instant.now();
removeNodeUpTask(longId);
LOGGER.debug("Time to create a node for run {} : {} s.", runId, Duration.between(start, end).getSeconds());
}, executorService.getExecutorService()).exceptionally(e -> {
LOGGER.error(e.getMessage(), e);
if (e.getCause() instanceof CmdExecutionException && Objects.equals(NODEUP_SPOT_FAILED_EXIT_CODE, ((CmdExecutionException) e.getCause()).getExitCode())) {
spotNodeUpAttempts.merge(longId, 1, (oldVal, newVal) -> oldVal + 1);
}
removeNodeUpTask(longId, false);
return null;
}));
}
use of com.epam.pipeline.exception.CmdExecutionException in project cloud-pipeline by epam.
the class ClusterManagerImpl method isNodeExpired.
@Override
public boolean isNodeExpired(String runId) {
Integer keepAliveMinutes = preferenceManager.getPreference(SystemPreferences.CLUSTER_KEEP_ALIVE_MINUTES);
if (keepAliveMinutes == null) {
return true;
}
String command = buildRunIdArgCommand(runId, nodeLaunchTimeScript);
try {
LOGGER.debug("Getting node launch time. Command: {}.", command);
String result = cmdExecutor.executeCommand(command).trim();
LOGGER.debug("Node {} launch time {}.", runId, result);
LocalDateTime launchTime = LocalDateTime.parse(result, formatter);
LocalDateTime now = LocalDateTime.now(Clock.systemUTC());
long aliveTime = Duration.between(launchTime, now).getSeconds() / TIME_DELIMITER;
LOGGER.debug("Node {} is alive for {} minutes.", runId, aliveTime);
long minutesToWholeHour = aliveTime % TIME_DELIMITER;
long minutesLeft = TIME_DELIMITER - minutesToWholeHour;
LOGGER.debug("Node {} has {} minutes left until next hour.", runId, minutesLeft);
return minutesLeft <= keepAliveMinutes && minutesLeft > TIME_TO_SHUT_DOWN_NODE;
} catch (DateTimeParseException | CmdExecutionException e) {
LOGGER.error(e.getMessage(), e);
return true;
}
}
Aggregations