Examples with PipelineRun com.epam.pipeline.entity.pipeline.PipelineRun used on opensource projects

Example 56 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class InstanceOfferManager method getPipelineRunEstimatedPrice.

public PipelineRunPrice getPipelineRunEstimatedPrice(Long runId, Long regionId) {
    PipelineRun pipelineRun = pipelineRunManager.loadPipelineRun(runId);
    RunInstance runInstance = pipelineRun.getInstance();
    AwsRegion region = awsRegionManager.loadRegionOrGetDefault(regionId);
    double pricePerHourForInstance = getPricePerHourForInstance(runInstance.getNodeType(), isSpotRequest(runInstance.getSpot()), region.getAwsRegionName());
    double pricePerDisk = getPriceForDisk(runInstance.getNodeDisk(), region.getAwsRegionName());
    double pricePerHour = pricePerDisk + pricePerHourForInstance;
    PipelineRunPrice price = new PipelineRunPrice();
    price.setInstanceDisk(runInstance.getNodeDisk());
    price.setInstanceType(runInstance.getNodeType());
    price.setPricePerHour(pricePerHour);
    if (pipelineRun.getStatus().isFinal()) {
        long duration = pipelineRun.getEndDate().getTime() - pipelineRun.getStartDate().getTime();
        price.setTotalPrice(duration / ONE_HOUR * pricePerHour);
    } else {
        price.setTotalPrice(0);
    }
    return price;
}

Example 57 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class InstanceOfferManager method getInstanceEstimatedPrice.

public InstancePrice getInstanceEstimatedPrice(Long id, String version, String configName, String instanceType, int instanceDisk, Boolean spot, Long regionId) throws GitClientException {
    Boolean useSpot = spot;
    if (StringUtils.isEmpty(instanceType) || instanceDisk <= 0 || useSpot == null) {
        PipelineConfiguration pipelineConfiguration = versionManager.loadParametersFromScript(id, version, configName);
        if (StringUtils.isEmpty(instanceType)) {
            instanceType = pipelineConfiguration.getInstanceType();
        }
        if (instanceDisk <= 0) {
            instanceDisk = Integer.parseInt(pipelineConfiguration.getInstanceDisk());
        }
        if (useSpot == null) {
            useSpot = pipelineConfiguration.getIsSpot();
        }
    }
    Assert.isTrue(isInstanceAllowed(instanceType), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_TYPE_IS_NOT_ALLOWED, instanceType));
    AwsRegion region = awsRegionManager.loadRegionOrGetDefault(regionId);
    double pricePerHourForInstance = getPricePerHourForInstance(instanceType, isSpotRequest(useSpot), region.getAwsRegionName());
    double pricePerDisk = getPriceForDisk(instanceDisk, region.getAwsRegionName());
    double pricePerHour = pricePerDisk + pricePerHourForInstance;
    InstancePrice instancePrice = new InstancePrice(instanceType, instanceDisk, pricePerHour);
    List<PipelineRun> runs = pipelineRunManager.loadAllRunsByPipeline(id, version).stream().filter(run -> run.getStatus().isFinal()).collect(Collectors.toList());
    if (!runs.isEmpty()) {
        long minimumDuration = -1;
        long maximumDuration = -1;
        long totalDurations = 0;
        for (PipelineRun run : runs) {
            long duration = run.getEndDate().getTime() - run.getStartDate().getTime();
            if (minimumDuration == -1 || minimumDuration > duration) {
                minimumDuration = duration;
            }
            if (maximumDuration == -1 || maximumDuration < duration) {
                maximumDuration = duration;
            }
            totalDurations += duration;
        }
        double averageDuration = (double) totalDurations / runs.size();
        instancePrice.setAverageTimePrice(pricePerHour * averageDuration / ONE_HOUR);
        instancePrice.setMinimumTimePrice(pricePerHour * minimumDuration / ONE_HOUR);
        instancePrice.setMaximumTimePrice(pricePerHour * maximumDuration / ONE_HOUR);
    }
    return instancePrice;
}

Example 58 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class ResourceMonitoringManager method monitorResourceUsage.

public void monitorResourceUsage() {
    int idleTimeout = preferenceManager.getPreference(SYSTEM_MAX_IDLE_TIMEOUT_MINUTES);
    Map<String, PipelineRun> running = pipelineRunManager.loadRunningPipelineRuns().stream().filter(r -> DateUtils.nowUTC().isAfter(r.getProlongedAtTime().plus(idleTimeout, ChronoUnit.MINUTES))).collect(Collectors.toMap(PipelineRun::getPodId, r -> r));
    LOGGER.debug("Checking cpu stats for pipelines: " + running.keySet().stream().collect(Collectors.joining(", ")));
    LocalDateTime now = DateUtils.nowUTC();
    Map<String, Double> cpuMetrics = monitoringDao.loadCpuUsageRateMetrics(running.keySet(), now.minusMinutes(idleTimeout), now);
    LOGGER.debug("CPU Metrics received: " + cpuMetrics.entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).collect(Collectors.joining(", ")));
    double idleCpuLevel = preferenceManager.getPreference(SYSTEM_IDLE_CPU_THRESHOLD_PERCENT) / PERCENT;
    int actionTimeout = preferenceManager.getPreference(SYSTEM_IDLE_ACTION_TIMEOUT_MINUTES);
    IdleRunAction action = IdleRunAction.valueOf(preferenceManager.getPreference(SystemPreferences.SYSTEM_IDLE_ACTION));
    List<PipelineRun> runsToUpdate = processRuns(running, cpuMetrics, idleCpuLevel, actionTimeout, action);
    pipelineRunManager.updatePipelineRunsLastNotification(runsToUpdate);
}

Example 59 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class PodMonitor method clearWorkerNodes.

private void clearWorkerNodes(PipelineRun run, KubernetesClient client) {
    List<Pod> workers = getWorkerPods(String.valueOf(run.getId()), client);
    workers.forEach(worker -> {
        String runIdLabel = getRunIdLabel(worker);
        LOGGER.debug("Clearing worker {} node for parent run {}.", runIdLabel, run.getId());
        Long workerId = Long.parseLong(runIdLabel);
        PipelineRun workerRun = pipelineRunManager.loadPipelineRun(workerId);
        getPodLogs(workerRun, client, worker);
        workerRun.setTerminating(false);
        workerRun.setStatus(run.getStatus());
        workerRun.setEndDate(run.getEndDate());
        pipelineRunManager.updatePipelineStatus(workerRun);
        checkAndUpdateEc2State(workerRun, false);
        client.pods().inNamespace(kubeNamespace).withLabel(KubernetesConstants.RUN_ID_LABEL, runIdLabel).delete();
    });
}

Example 60 with PipelineRun

use of com.epam.pipeline.entity.pipeline.PipelineRun in project cloud-pipeline by epam.

the class PipelineRunManager method commitRun.

/**
 * Commits docker image and push it to a docker registry from specified run
 * @param id {@link PipelineRun} id for pipeline run to be committed
 * @param registryId {@link DockerRegistry} id where new image will be pushed
 * @param deleteFiles if true files from pipeline working directory will be cleaned
 * @param stopPipeline if true pipeline will be stopped after commit
 * @param checkSize if true method will check if free disk space is enough for commit operation
 * @return  {@link PipelineRun} to be committed
 */
@Transactional(propagation = Propagation.REQUIRED)
public PipelineRun commitRun(Long id, Long registryId, String newImageName, boolean deleteFiles, boolean stopPipeline, boolean checkSize) {
    if (checkSize) {
        Assert.state(checkFreeSpaceAvailable(id), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_DISK_NOT_ENOUGH));
    }
    PipelineRun pipelineRun = pipelineRunDao.loadPipelineRun(id);
    DockerRegistry dockerRegistry = dockerRegistryManager.load(registryId);
    Assert.notNull(pipelineRun, messageHelper.getMessage(MessageConstants.ERROR_RUN_PIPELINES_NOT_FOUND, id));
    Assert.state(pipelineRun.getStatus() == TaskStatus.RUNNING, messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_RUN_FINISHED, id));
    Assert.notNull(dockerRegistry, messageHelper.getMessage(MessageConstants.ERROR_REGISTRY_NOT_FOUND, registryId));
    String dockerImageFromRun = retrieveImageName(pipelineRun);
    String resolvedImageName = StringUtils.isEmpty(newImageName) ? dockerImageFromRun : newImageName;
    // check that there is no tool with this name in another registry
    toolManager.assertThatToolUniqueAcrossRegistries(resolvedImageName, dockerRegistry.getPath());
    return dockerContainerOperationManager.commitContainer(pipelineRun, dockerRegistry, resolvedImageName, deleteFiles, stopPipeline);
}