Search in sources :

Example 6 with CmdExecutionException

use of com.epam.pipeline.exception.CmdExecutionException in project cloud-pipeline by epam.

the class AutoscaleManager method createNodeForRun.

private void createNodeForRun(List<CompletableFuture<Void>> tasks, String runId, RunInstance requiredInstance) {
    long longId = Long.parseLong(runId);
    addNodeUpTask(longId);
    tasks.add(CompletableFuture.runAsync(() -> {
        Instant start = Instant.now();
        // save required instance
        pipelineRunManager.updateRunInstance(longId, requiredInstance);
        RunInstance instance = clusterManager.scaleUp(runId, requiredInstance);
        // save instance ID and IP
        pipelineRunManager.updateRunInstance(longId, instance);
        Instant end = Instant.now();
        removeNodeUpTask(longId);
        LOGGER.debug("Time to create a node for run {} : {} s.", runId, Duration.between(start, end).getSeconds());
    }, executorService.getExecutorService()).exceptionally(e -> {
        LOGGER.error(e.getMessage(), e);
        if (e.getCause() instanceof CmdExecutionException && Objects.equals(NODEUP_SPOT_FAILED_EXIT_CODE, ((CmdExecutionException) e.getCause()).getExitCode())) {
            spotNodeUpAttempts.merge(longId, 1, (oldVal, newVal) -> oldVal + 1);
        }
        removeNodeUpTask(longId, false);
        return null;
    }));
}
Also used : GitClientException(com.epam.pipeline.exception.git.GitClientException) Date(java.util.Date) PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) LoggerFactory(org.slf4j.LoggerFactory) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) Autowired(org.springframework.beans.factory.annotation.Autowired) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) ParallelExecutorService(com.epam.pipeline.manager.parallel.ParallelExecutorService) Service(org.springframework.stereotype.Service) Duration(java.time.Duration) Map(java.util.Map) ConditionalOnProperty(org.springframework.boot.autoconfigure.condition.ConditionalOnProperty) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) Node(io.fabric8.kubernetes.api.model.Node) PodCondition(io.fabric8.kubernetes.api.model.PodCondition) KubernetesClientException(io.fabric8.kubernetes.client.KubernetesClientException) NodeCondition(io.fabric8.kubernetes.api.model.NodeCondition) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Logger(org.slf4j.Logger) PipelineRunParameter(com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) TaskStatus(com.epam.pipeline.entity.pipeline.TaskStatus) AbstractSchedulingManager(com.epam.pipeline.manager.scheduling.AbstractSchedulingManager) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Pod(io.fabric8.kubernetes.api.model.Pod) Set(java.util.Set) Instant(java.time.Instant) HasMetadata(io.fabric8.kubernetes.api.model.HasMetadata) Collectors(java.util.stream.Collectors) NodeList(io.fabric8.kubernetes.api.model.NodeList) Objects(java.util.Objects) Config(io.fabric8.kubernetes.client.Config) List(java.util.List) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException) PodList(io.fabric8.kubernetes.api.model.PodList) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) NumberUtils(org.apache.commons.lang3.math.NumberUtils) PostConstruct(javax.annotation.PostConstruct) Optional(java.util.Optional) Collections(java.util.Collections) Instant(java.time.Instant) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException)

Example 7 with CmdExecutionException

use of com.epam.pipeline.exception.CmdExecutionException in project cloud-pipeline by epam.

the class ClusterManagerImpl method isNodeExpired.

@Override
public boolean isNodeExpired(String runId) {
    Integer keepAliveMinutes = preferenceManager.getPreference(SystemPreferences.CLUSTER_KEEP_ALIVE_MINUTES);
    if (keepAliveMinutes == null) {
        return true;
    }
    String command = buildRunIdArgCommand(runId, nodeLaunchTimeScript);
    try {
        LOGGER.debug("Getting node launch time. Command: {}.", command);
        String result = cmdExecutor.executeCommand(command).trim();
        LOGGER.debug("Node {} launch time {}.", runId, result);
        LocalDateTime launchTime = LocalDateTime.parse(result, formatter);
        LocalDateTime now = LocalDateTime.now(Clock.systemUTC());
        long aliveTime = Duration.between(launchTime, now).getSeconds() / TIME_DELIMITER;
        LOGGER.debug("Node {} is alive for {} minutes.", runId, aliveTime);
        long minutesToWholeHour = aliveTime % TIME_DELIMITER;
        long minutesLeft = TIME_DELIMITER - minutesToWholeHour;
        LOGGER.debug("Node {} has {} minutes left until next hour.", runId, minutesLeft);
        return minutesLeft <= keepAliveMinutes && minutesLeft > TIME_TO_SHUT_DOWN_NODE;
    } catch (DateTimeParseException | CmdExecutionException e) {
        LOGGER.error(e.getMessage(), e);
        return true;
    }
}
Also used : LocalDateTime(java.time.LocalDateTime) DateTimeParseException(java.time.format.DateTimeParseException) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException)

Aggregations

CmdExecutionException (com.epam.pipeline.exception.CmdExecutionException)7 IOException (java.io.IOException)3 PipelineConfiguration (com.epam.pipeline.entity.configuration.PipelineConfiguration)2 PipelineRun (com.epam.pipeline.entity.pipeline.PipelineRun)2 RunInstance (com.epam.pipeline.entity.pipeline.RunInstance)2 TaskStatus (com.epam.pipeline.entity.pipeline.TaskStatus)2 PipelineRunParameter (com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter)2 GitClientException (com.epam.pipeline.exception.git.GitClientException)2 ParallelExecutorService (com.epam.pipeline.manager.parallel.ParallelExecutorService)2 PipelineRunManager (com.epam.pipeline.manager.pipeline.PipelineRunManager)2 PreferenceManager (com.epam.pipeline.manager.preference.PreferenceManager)2 SystemPreferences (com.epam.pipeline.manager.preference.SystemPreferences)2 AbstractSchedulingManager (com.epam.pipeline.manager.scheduling.AbstractSchedulingManager)2 HasMetadata (io.fabric8.kubernetes.api.model.HasMetadata)2 Node (io.fabric8.kubernetes.api.model.Node)2 NodeCondition (io.fabric8.kubernetes.api.model.NodeCondition)2 NodeList (io.fabric8.kubernetes.api.model.NodeList)2 Pod (io.fabric8.kubernetes.api.model.Pod)2 PodCondition (io.fabric8.kubernetes.api.model.PodCondition)2 PodList (io.fabric8.kubernetes.api.model.PodList)2