Examples with PipelineConfiguration com.epam.pipeline.entity.configuration.PipelineConfiguration used on opensource projects

Example 26 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class RunConfigurationLoaderTest method shouldLoadRunConfigurationTest.

@Test
void shouldLoadRunConfigurationTest() throws EntityNotFoundException {
    Pipeline pipeline = new Pipeline();
    pipeline.setId(1L);
    pipeline.setName(TEST_NAME);
    PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
    pipelineConfiguration.setDockerImage(TEST_PATH);
    RunConfigurationEntry entry = new RunConfigurationEntry();
    entry.setPipelineId(pipeline.getId());
    entry.setPipelineVersion(TEST_VERSION);
    entry.setName(TEST_NAME);
    entry.setConfiguration(pipelineConfiguration);
    RunConfiguration runConfiguration = new RunConfiguration();
    runConfiguration.setId(1L);
    runConfiguration.setName(TEST_NAME);
    runConfiguration.setDescription(TEST_VALUE);
    runConfiguration.setOwner(TEST_NAME);
    runConfiguration.setEntries(Collections.singletonList(entry));
    RunConfigurationDoc expectedConfiguration = RunConfigurationDoc.builder().configuration(runConfiguration).pipelines(Collections.singletonList(pipeline)).build();
    RunConfigurationLoader runConfigurationLoader = new RunConfigurationLoader(apiClient);
    when(apiClient.loadRunConfiguration(anyLong())).thenReturn(runConfiguration);
    when(apiClient.loadPipeline(anyString())).thenReturn(pipeline);
    Optional<EntityContainer<RunConfigurationDoc>> container = runConfigurationLoader.loadEntity(1L);
    EntityContainer<RunConfigurationDoc> configurationDocContainer = container.orElseThrow(AssertionError::new);
    RunConfigurationDoc actualConfiguration = configurationDocContainer.getEntity();
    assertNotNull(actualConfiguration);
    verifyConfiguration(expectedConfiguration.getConfiguration(), actualConfiguration.getConfiguration());
    verifyPipeline(expectedConfiguration.getPipelines(), actualConfiguration.getPipelines());
    verifyPipelineUser(configurationDocContainer.getOwner());
    verifyPermissions(PERMISSIONS_CONTAINER_WITH_OWNER, configurationDocContainer.getPermissions());
    verifyMetadata(EXPECTED_METADATA, new ArrayList<>(configurationDocContainer.getMetadata().values()));
}

Example 27 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class AutoscaleManager method getNewRunInstance.

private RunInstance getNewRunInstance(String runId) throws GitClientException {
    Long longRunId = Long.parseLong(runId);
    PipelineRun run = pipelineRunManager.loadPipelineRun(longRunId);
    RunInstance instance;
    if (run.getInstance() == null || run.getInstance().isEmpty()) {
        PipelineConfiguration configuration = pipelineRunManager.loadRunConfiguration(longRunId);
        instance = clusterManager.configurationToInstance(configuration);
    } else {
        instance = clusterManager.fillInstance(run.getInstance());
    }
    if (instance.getSpot() != null && instance.getSpot() && spotNodeUpAttempts.getOrDefault(longRunId, 0) >= preferenceManager.getPreference(SystemPreferences.CLUSTER_SPOT_MAX_ATTEMPTS)) {
        instance.setSpot(false);
        pipelineRunManager.updateRunInstance(longRunId, instance);
    }
    return instance;
}

Example 28 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class InstanceOfferManager method getInstanceEstimatedPrice.

public InstancePrice getInstanceEstimatedPrice(Long id, String version, String configName, String instanceType, int instanceDisk, Boolean spot, Long regionId) throws GitClientException {
    Boolean useSpot = spot;
    if (StringUtils.isEmpty(instanceType) || instanceDisk <= 0 || useSpot == null) {
        PipelineConfiguration pipelineConfiguration = versionManager.loadParametersFromScript(id, version, configName);
        if (StringUtils.isEmpty(instanceType)) {
            instanceType = pipelineConfiguration.getInstanceType();
        }
        if (instanceDisk <= 0) {
            instanceDisk = Integer.parseInt(pipelineConfiguration.getInstanceDisk());
        }
        if (useSpot == null) {
            useSpot = pipelineConfiguration.getIsSpot();
        }
    }
    Assert.isTrue(isInstanceAllowed(instanceType), messageHelper.getMessage(MessageConstants.ERROR_INSTANCE_TYPE_IS_NOT_ALLOWED, instanceType));
    AwsRegion region = awsRegionManager.loadRegionOrGetDefault(regionId);
    double pricePerHourForInstance = getPricePerHourForInstance(instanceType, isSpotRequest(useSpot), region.getAwsRegionName());
    double pricePerDisk = getPriceForDisk(instanceDisk, region.getAwsRegionName());
    double pricePerHour = pricePerDisk + pricePerHourForInstance;
    InstancePrice instancePrice = new InstancePrice(instanceType, instanceDisk, pricePerHour);
    List<PipelineRun> runs = pipelineRunManager.loadAllRunsByPipeline(id, version).stream().filter(run -> run.getStatus().isFinal()).collect(Collectors.toList());
    if (!runs.isEmpty()) {
        long minimumDuration = -1;
        long maximumDuration = -1;
        long totalDurations = 0;
        for (PipelineRun run : runs) {
            long duration = run.getEndDate().getTime() - run.getStartDate().getTime();
            if (minimumDuration == -1 || minimumDuration > duration) {
                minimumDuration = duration;
            }
            if (maximumDuration == -1 || maximumDuration < duration) {
                maximumDuration = duration;
            }
            totalDurations += duration;
        }
        double averageDuration = (double) totalDurations / runs.size();
        instancePrice.setAverageTimePrice(pricePerHour * averageDuration / ONE_HOUR);
        instancePrice.setMinimumTimePrice(pricePerHour * minimumDuration / ONE_HOUR);
        instancePrice.setMaximumTimePrice(pricePerHour * maximumDuration / ONE_HOUR);
    }
    return instancePrice;
}

Example 29 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class PipelineRunManager method runPod.

/**
 * Creates a new pod with a given run_id, doesn't create a new pipeline run
 * @param runVO
 * @return
 */
// TODO: refactoring
@ToolSecurityPolicyCheck
@Transactional(propagation = Propagation.REQUIRED)
public PipelineRun runPod(PipelineStart runVO) {
    Assert.notNull(runVO.getCmdTemplate(), messageHelper.getMessage(MessageConstants.SETTING_IS_NOT_PROVIDED, "cmd_template"));
    PipelineRun parentRun = loadPipelineRun(runVO.getUseRunId());
    Assert.state(parentRun.getStatus() == TaskStatus.RUNNING, messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_RUN_NOT_RUNNING, runVO.getUseRunId()));
    PipelineConfiguration configuration = configurationManager.getPipelineConfiguration(runVO);
    Tool tool = getToolForRun(configuration);
    configuration.setSecretName(tool.getSecretName());
    List<String> endpoints = tool.getEndpoints();
    PipelineRun run = new PipelineRun();
    run.setInstance(parentRun.getInstance());
    run.setId(runVO.getUseRunId());
    run.setStartDate(DateUtils.now());
    run.setStatus(TaskStatus.RUNNING);
    run.setPipelineName(DEFAULT_PIPELINE_NAME);
    run.setPodId(getRootPodIDFromTool(tool.getImage(), run.getId()));
    run.setDockerImage(configuration.getDockerImage());
    run.setCmdTemplate(determinateCmdTemplateForRun(configuration));
    run.setTimeout(runVO.getTimeout());
    run.setCommitStatus(CommitStatus.NOT_COMMITTED);
    run.setLastChangeCommitTime(DateUtils.now());
    run.setRunSids(runVO.getRunSids());
    run.setOwner(parentRun.getOwner());
    String launchedCommand = pipelineLauncher.launch(run, configuration, endpoints, runVO.getUseRunId().toString(), false, parentRun.getPodId(), null);
    run.setActualCmd(launchedCommand);
    return run;
}

Example 30 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class PipelineConfigurationManager method mergeParameters.

public PipelineConfiguration mergeParameters(PipelineStart runVO, PipelineConfiguration defaultConfig) {
    Map<String, PipeConfValueVO> params = runVO.getParams() == null ? Collections.emptyMap() : runVO.getParams();
    PipelineConfiguration configuration = new PipelineConfiguration();
    configuration.setMainFile(defaultConfig.getMainFile());
    configuration.setMainClass(defaultConfig.getMainClass());
    configuration.setEnvironmentParams(defaultConfig.getEnvironmentParams());
    configuration.setPrettyUrl(runVO.getPrettyUrl());
    Map<String, PipeConfValueVO> runParameters = new LinkedHashMap<>();
    // filter and get parameters from user
    params.entrySet().stream().filter(entry -> entry.getValue().isRequired() || !StringUtils.isEmpty(entry.getValue().getValue())).forEach(entry -> runParameters.put(entry.getKey(), entry.getValue()));
    // fill in default values, only if user's value wasn't provided
    if (defaultConfig.getParameters() != null) {
        defaultConfig.getParameters().entrySet().stream().filter(entry -> entry.getValue().isRequired() || !StringUtils.isEmpty(entry.getValue().getValue())).forEach(entry -> runParameters.putIfAbsent(entry.getKey(), entry.getValue()));
    }
    // set parameters to the final configuration
    configuration.setParameters(runParameters);
    // fill instance settings
    if (runVO.getHddSize() != null) {
        configuration.setInstanceDisk(String.valueOf(runVO.getHddSize()));
    } else {
        configuration.setInstanceDisk(defaultConfig.getInstanceDisk());
    }
    if (runVO.getInstanceType() != null) {
        configuration.setInstanceType(String.valueOf(runVO.getInstanceType()));
    } else {
        configuration.setInstanceType(defaultConfig.getInstanceType());
    }
    if (runVO.getTimeout() != null) {
        configuration.setTimeout(runVO.getTimeout());
    } else {
        configuration.setTimeout(defaultConfig.getTimeout());
    }
    // client always sends actual node-count
    configuration.setNodeCount(runVO.getNodeCount());
    configuration.setCmdTemplate(runVO.getCmdTemplate() == null ? defaultConfig.getCmdTemplate() : runVO.getCmdTemplate());
    Boolean isSpot = runVO.getIsSpot() == null ? defaultConfig.getIsSpot() : runVO.getIsSpot();
    configuration.setIsSpot(isSpot);
    if (isSpot != null && !isSpot) {
        configuration.setNonPause(runVO.isNonPause());
    }
    configuration.setWorkerCmd(runVO.getWorkerCmd() == null ? defaultConfig.getWorkerCmd() : runVO.getWorkerCmd());
    configuration.setDockerImage(chooseDockerImage(runVO, defaultConfig));
    configuration.buildEnvVariables();
    return configuration;
}