Search in sources :

Example 1 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class PipelineRunManager method createRestartRun.

private PipelineRun createRestartRun(final PipelineRun run) {
    PipelineRun restartedRun = new PipelineRun();
    Long runId = pipelineRunDao.createRunId();
    restartedRun.setId(runId);
    restartedRun.setStartDate(DateUtils.now());
    Optional<Pipeline> pipeline = Optional.ofNullable(run.getPipelineId()).map(pipelineId -> pipelineManager.load(pipelineId));
    pipeline.ifPresent(p -> {
        restartedRun.setPipelineName(p.getName());
        restartedRun.setRepository(p.getRepository());
        restartedRun.setPipelineId(p.getId());
        restartedRun.setVersion(run.getVersion());
        restartedRun.setRevisionName(gitManager.getRevisionName(run.getVersion()));
    });
    if (!pipeline.isPresent()) {
        fillMissingPipelineFields(restartedRun);
    }
    restartedRun.setStatus(TaskStatus.RUNNING);
    restartedRun.setCommitStatus(CommitStatus.NOT_COMMITTED);
    restartedRun.setLastChangeCommitTime(DateUtils.now());
    restartedRun.setPodId(getRootPodIDFromPipeline(restartedRun));
    restartedRun.setParams(run.getParams());
    restartedRun.parseParameters();
    restartedRun.setTimeout(run.getTimeout());
    restartedRun.setDockerImage(run.getDockerImage());
    restartedRun.setCmdTemplate(run.getCmdTemplate());
    restartedRun.setNodeCount(run.getNodeCount());
    RunInstance instance = copyInstance(run.getInstance());
    restartedRun.setInstance(instance);
    setRunPrice(instance, restartedRun);
    restartedRun.setSshPassword(PasswordGenerator.generatePassword());
    restartedRun.setOwner(run.getOwner());
    restartedRun.setEntitiesIds(run.getEntitiesIds());
    restartedRun.setConfigurationId(run.getConfigurationId());
    restartedRun.setExecutionPreferences(run.getExecutionPreferences());
    restartedRun.setRunSids(run.getRunSids());
    return restartedRun;
}
Also used : PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) Pipeline(com.epam.pipeline.entity.pipeline.Pipeline)

Example 2 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class PipelineRunDaoTest method createPipelineRun.

private PipelineRun createPipelineRun(Long pipelineId, String params, TaskStatus status, Long parentRunId, Long entitiesId, Boolean isSpot, Long configurationId, List<RunSid> runSids) {
    PipelineRun run = new PipelineRun();
    run.setPipelineId(pipelineId);
    run.setVersion(TEST_REVISION_1);
    run.setStartDate(new Date());
    run.setEndDate(new Date());
    run.setStatus(status);
    run.setCommitStatus(CommitStatus.NOT_COMMITTED);
    run.setLastChangeCommitTime(new Date());
    run.setPodId(TEST_POD_ID);
    run.setParams(params);
    run.setOwner(USER);
    run.setParentRunId(parentRunId);
    run.setRunSids(runSids);
    run.setServiceUrl(TEST_SERVICE_URL);
    RunInstance instance = new RunInstance();
    instance.setSpot(isSpot);
    instance.setNodeId("1");
    run.setInstance(instance);
    run.setEntitiesIds(Collections.singletonList(entitiesId));
    run.setConfigurationId(configurationId);
    pipelineRunDao.createPipelineRun(run);
    return run;
}
Also used : PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) Date(java.util.Date) LocalDate(java.time.LocalDate)

Example 3 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class ObjectCreatorUtils method createPipelineRun.

public static PipelineRun createPipelineRun(Long runId, Long pipelineId, Long parentRunId) {
    PipelineRun run = new PipelineRun();
    run.setId(runId);
    run.setPipelineId(pipelineId);
    run.setVersion(TEST_REVISION_1);
    run.setStartDate(new Date());
    run.setEndDate(new Date());
    run.setStatus(TaskStatus.RUNNING);
    run.setCommitStatus(CommitStatus.NOT_COMMITTED);
    run.setLastChangeCommitTime(new Date());
    run.setPodId(TEST_POD_ID);
    run.setOwner(TEST_NAME);
    run.setParentRunId(parentRunId);
    run.setServiceUrl(TEST_SERVICE_URL);
    RunInstance instance = new RunInstance();
    instance.setSpot(true);
    instance.setNodeId("1");
    run.setInstance(instance);
    return run;
}
Also used : PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) Date(java.util.Date)

Example 4 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class AutoscaleManager method processPod.

private void processPod(Pod pod, KubernetesClient client, Set<String> scheduledRuns, List<CompletableFuture<Void>> tasks, Set<String> allPods, Set<String> nodes, Set<String> reassignedNodes) {
    LOGGER.debug("Found an unscheduled pod: {}.", pod.getMetadata().getName());
    Map<String, String> labels = pod.getMetadata().getLabels();
    String runId = labels.get(KubernetesConstants.RUN_ID_LABEL);
    long longId = Long.parseLong(runId);
    if (nodeUpTaskInProgress.contains(longId)) {
        LOGGER.debug("Nodeup task for ID {} is already in progress.", runId);
        return;
    }
    // Check whether node with required RunID is available
    if (nodes.contains(runId)) {
        LOGGER.debug("Node with required ID {} already exists.", runId);
        return;
    }
    // check max nodeup retry count
    // TODO: should we lock here?
    int retryCount = nodeUpAttempts.getOrDefault(longId, 0);
    int nodeUpRetryCount = preferenceManager.getPreference(SystemPreferences.CLUSTER_NODEUP_RETRY_COUNT);
    if (retryCount >= nodeUpRetryCount) {
        LOGGER.debug("Exceeded max nodeup attempts ({}) for run ID {}. Setting run status 'FAILURE'.", retryCount, runId);
        pipelineRunManager.updatePipelineStatusIfNotFinal(longId, TaskStatus.FAILURE, new Date());
        removeNodeUpTask(longId);
        return;
    }
    try {
        RunInstance requiredInstance = getNewRunInstance(runId);
        // check whether aws instance already exists
        RunInstance awsInstance = clusterManager.describeInstance(runId, requiredInstance);
        if (awsInstance != null && awsInstance.getNodeId() != null) {
            LOGGER.debug("Found {} instance for run ID {}.", awsInstance.getNodeId(), runId);
            createNodeForRun(tasks, runId, requiredInstance);
            return;
        }
        List<String> freeNodes = nodes.stream().filter(nodeId -> !allPods.contains(nodeId) && !reassignedNodes.contains(nodeId) && isNodeAvailable(client, nodeId)).collect(Collectors.toList());
        LOGGER.debug("Found {} free nodes.", freeNodes.size());
        // Try to reassign one of idle nodes
        for (String previousId : freeNodes) {
            LOGGER.debug("Found free node ID {}.", previousId);
            RunInstance previousInstance = getPreviousRunInstance(previousId);
            if (clusterManager.requirementsMatch(requiredInstance, previousInstance)) {
                LOGGER.debug("Reassigning node ID {} to run {}.", previousId, runId);
                boolean successfullyReassigned = clusterManager.reassignNode(previousId, runId);
                if (successfullyReassigned) {
                    scheduledRuns.add(runId);
                    pipelineRunManager.updateRunInstance(longId, previousInstance);
                    reassignedNodes.add(previousId);
                    return;
                }
            }
        }
        // Check max cluster capacity
        int currentClusterSize = getCurrentClusterSize(client);
        NodeList nodeList = getAvailableNodes(client);
        Integer maxClusterSize = preferenceManager.getPreference(SystemPreferences.CLUSTER_MAX_SIZE);
        if (currentClusterSize > maxClusterSize) {
            LOGGER.debug("Exceeded maximum cluster size {} - current size {}.", maxClusterSize, currentClusterSize);
            return;
        }
        if (currentClusterSize == maxClusterSize && preferenceManager.getPreference(SystemPreferences.CLUSTER_KILL_NOT_MATCHING_NODES)) {
            LOGGER.debug("Current cluster size {} has reached limit {}. Checking free nodes.", currentClusterSize, maxClusterSize);
            List<String> nonMatchingFreeNodes = freeNodes.stream().filter(id -> !reassignedNodes.contains(id)).collect(Collectors.toList());
            if (!CollectionUtils.isEmpty(nonMatchingFreeNodes)) {
                String nodeId = nonMatchingFreeNodes.get(0);
                // to remove node from free
                reassignedNodes.add(nodeId);
                LOGGER.debug("Scaling down unused node {}.", nodeId);
                clusterManager.scaleDown(nodeId);
            } else {
                LOGGER.debug("Exceeded maximum cluster size {}.", nodeList.getItems().size() + nodeUpTaskInProgress.size());
                LOGGER.debug("Leaving pending run {}.", runId);
                return;
            }
        }
        int nodeUpTasksSize = nodeUpTaskInProgress.size();
        int maxNodeUpThreads = preferenceManager.getPreference(SystemPreferences.CLUSTER_NODEUP_MAX_THREADS);
        if (nodeUpTasksSize >= maxNodeUpThreads) {
            LOGGER.debug("Exceeded maximum node up tasks queue size {}.", nodeUpTasksSize);
            return;
        }
        scheduledRuns.add(runId);
        createNodeForRun(tasks, runId, requiredInstance);
    } catch (GitClientException | CmdExecutionException | IllegalArgumentException e) {
        LOGGER.error("Failed to create node for run {}.", runId);
        LOGGER.error("Failed to get pipeline configuration: " + e.getMessage(), e);
    }
}
Also used : GitClientException(com.epam.pipeline.exception.git.GitClientException) Date(java.util.Date) PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) LoggerFactory(org.slf4j.LoggerFactory) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) Autowired(org.springframework.beans.factory.annotation.Autowired) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) ParallelExecutorService(com.epam.pipeline.manager.parallel.ParallelExecutorService) Service(org.springframework.stereotype.Service) Duration(java.time.Duration) Map(java.util.Map) ConditionalOnProperty(org.springframework.boot.autoconfigure.condition.ConditionalOnProperty) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) Node(io.fabric8.kubernetes.api.model.Node) PodCondition(io.fabric8.kubernetes.api.model.PodCondition) KubernetesClientException(io.fabric8.kubernetes.client.KubernetesClientException) NodeCondition(io.fabric8.kubernetes.api.model.NodeCondition) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Logger(org.slf4j.Logger) PipelineRunParameter(com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) TaskStatus(com.epam.pipeline.entity.pipeline.TaskStatus) AbstractSchedulingManager(com.epam.pipeline.manager.scheduling.AbstractSchedulingManager) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Pod(io.fabric8.kubernetes.api.model.Pod) Set(java.util.Set) Instant(java.time.Instant) HasMetadata(io.fabric8.kubernetes.api.model.HasMetadata) Collectors(java.util.stream.Collectors) NodeList(io.fabric8.kubernetes.api.model.NodeList) Objects(java.util.Objects) Config(io.fabric8.kubernetes.client.Config) List(java.util.List) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException) PodList(io.fabric8.kubernetes.api.model.PodList) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) NumberUtils(org.apache.commons.lang3.math.NumberUtils) PostConstruct(javax.annotation.PostConstruct) Optional(java.util.Optional) Collections(java.util.Collections) NodeList(io.fabric8.kubernetes.api.model.NodeList) GitClientException(com.epam.pipeline.exception.git.GitClientException) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) Date(java.util.Date) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException)

Example 5 with RunInstance

use of com.epam.pipeline.entity.pipeline.RunInstance in project cloud-pipeline by epam.

the class ClusterManagerImpl method configurationToInstance.

@Override
public RunInstance configurationToInstance(PipelineConfiguration configuration) {
    RunInstance instance = new RunInstance();
    if (configuration.getInstanceType() == null) {
        instance.setNodeType(preferenceManager.getPreference(SystemPreferences.CLUSTER_INSTANCE_TYPE));
    } else {
        instance.setNodeType(configuration.getInstanceType());
    }
    if (configuration.getInstanceDisk() == null) {
        instance.setNodeDisk(preferenceManager.getPreference(SystemPreferences.CLUSTER_INSTANCE_HDD));
    } else {
        instance.setNodeDisk(Integer.parseInt(configuration.getInstanceDisk()));
    }
    instance.setEffectiveNodeDisk(instance.getNodeDisk());
    if (configuration.getInstanceImage() == null) {
        instance.setNodeImage(preferenceManager.getPreference(SystemPreferences.CLUSTER_INSTANCE_IMAGE));
    } else {
        instance.setNodeImage(configuration.getInstanceImage());
    }
    instance.setAwsRegionId(Optional.ofNullable(configuration.getAwsRegionId()).map(regionId -> awsRegionManager.load(regionId)).orElse(awsRegionManager.loadDefaultRegion()).getAwsRegionName());
    return instance;
}
Also used : PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) LoggerFactory(org.slf4j.LoggerFactory) LocalDateTime(java.time.LocalDateTime) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) Autowired(org.springframework.beans.factory.annotation.Autowired) HashMap(java.util.HashMap) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) UserContext(com.epam.pipeline.security.UserContext) Service(org.springframework.stereotype.Service) Duration(java.time.Duration) Map(java.util.Map) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Logger(org.slf4j.Logger) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance) AwsRegion(com.epam.pipeline.entity.region.AwsRegion) Collectors(java.util.stream.Collectors) DateTimeParseException(java.time.format.DateTimeParseException) List(java.util.List) CmdExecutionException(com.epam.pipeline.exception.CmdExecutionException) UserManager(com.epam.pipeline.manager.user.UserManager) Environment(org.springframework.core.env.Environment) DateTimeFormatter(java.time.format.DateTimeFormatter) PostConstruct(javax.annotation.PostConstruct) CmdExecutor(com.epam.pipeline.manager.CmdExecutor) Clock(java.time.Clock) Optional(java.util.Optional) AuthManager(com.epam.pipeline.manager.security.AuthManager) AwsRegionManager(com.epam.pipeline.manager.region.AwsRegionManager) RunInstance(com.epam.pipeline.entity.pipeline.RunInstance)

Aggregations

RunInstance (com.epam.pipeline.entity.pipeline.RunInstance)20 PipelineRun (com.epam.pipeline.entity.pipeline.PipelineRun)14 PipelineConfiguration (com.epam.pipeline.entity.configuration.PipelineConfiguration)6 Date (java.util.Date)6 PipelineRunParameter (com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter)5 CmdExecutionException (com.epam.pipeline.exception.CmdExecutionException)5 List (java.util.List)5 GitClientException (com.epam.pipeline.exception.git.GitClientException)4 PipelineRunManager (com.epam.pipeline.manager.pipeline.PipelineRunManager)4 PreferenceManager (com.epam.pipeline.manager.preference.PreferenceManager)4 SystemPreferences (com.epam.pipeline.manager.preference.SystemPreferences)4 Duration (java.time.Duration)4 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)4 Map (java.util.Map)4 Optional (java.util.Optional)4 Collectors (java.util.stream.Collectors)4 PostConstruct (javax.annotation.PostConstruct)4 TaskStatus (com.epam.pipeline.entity.pipeline.TaskStatus)3 ParallelExecutorService (com.epam.pipeline.manager.parallel.ParallelExecutorService)3