Examples with PipelineStart com.epam.pipeline.entity.pipeline.run.PipelineStart used on opensource projects

Example 11 with PipelineStart

use of com.epam.pipeline.entity.pipeline.run.PipelineStart in project cloud-pipeline by epam.

the class PipelineConfigurationManager method mergeParameters.

public PipelineConfiguration mergeParameters(PipelineStart runVO, PipelineConfiguration defaultConfig) {
    Map<String, PipeConfValueVO> params = runVO.getParams() == null ? Collections.emptyMap() : runVO.getParams();
    PipelineConfiguration configuration = new PipelineConfiguration();
    configuration.setMainFile(defaultConfig.getMainFile());
    configuration.setMainClass(defaultConfig.getMainClass());
    configuration.setEnvironmentParams(defaultConfig.getEnvironmentParams());
    configuration.setPrettyUrl(runVO.getPrettyUrl());
    Map<String, PipeConfValueVO> runParameters = new LinkedHashMap<>();
    // filter and get parameters from user
    params.entrySet().stream().filter(entry -> entry.getValue().isRequired() || !StringUtils.isEmpty(entry.getValue().getValue())).forEach(entry -> runParameters.put(entry.getKey(), entry.getValue()));
    // fill in default values, only if user's value wasn't provided
    if (defaultConfig.getParameters() != null) {
        defaultConfig.getParameters().entrySet().stream().filter(entry -> entry.getValue().isRequired() || !StringUtils.isEmpty(entry.getValue().getValue())).forEach(entry -> runParameters.putIfAbsent(entry.getKey(), entry.getValue()));
    }
    // set parameters to the final configuration
    configuration.setParameters(runParameters);
    // fill instance settings
    if (runVO.getHddSize() != null) {
        configuration.setInstanceDisk(String.valueOf(runVO.getHddSize()));
    } else {
        configuration.setInstanceDisk(defaultConfig.getInstanceDisk());
    }
    if (runVO.getInstanceType() != null) {
        configuration.setInstanceType(String.valueOf(runVO.getInstanceType()));
    } else {
        configuration.setInstanceType(defaultConfig.getInstanceType());
    }
    if (runVO.getTimeout() != null) {
        configuration.setTimeout(runVO.getTimeout());
    } else {
        configuration.setTimeout(defaultConfig.getTimeout());
    }
    // client always sends actual node-count
    configuration.setNodeCount(runVO.getNodeCount());
    configuration.setCmdTemplate(runVO.getCmdTemplate() == null ? defaultConfig.getCmdTemplate() : runVO.getCmdTemplate());
    Boolean isSpot = runVO.getIsSpot() == null ? defaultConfig.getIsSpot() : runVO.getIsSpot();
    configuration.setIsSpot(isSpot);
    if (isSpot != null && !isSpot) {
        configuration.setNonPause(runVO.isNonPause());
    }
    configuration.setWorkerCmd(runVO.getWorkerCmd() == null ? defaultConfig.getWorkerCmd() : runVO.getWorkerCmd());
    configuration.setDockerImage(chooseDockerImage(runVO, defaultConfig));
    configuration.buildEnvVariables();
    return configuration;
}

Example 12 with PipelineStart

use of com.epam.pipeline.entity.pipeline.run.PipelineStart in project cloud-pipeline by epam.

the class CloudPlatformRunner method runConfigurationEntry.

private List<PipelineRun> runConfigurationEntry(RunConfigurationEntry entry, PipelineConfiguration configuration, int copies, String clusterId, boolean startNFS, List<Long> entityIds, Long configurationId) {
    PipelineStart startVO = entry.toPipelineStart();
    if (!StringUtils.hasText(clusterId)) {
        log.debug("Launching master entry {}", entry.getName());
        pipelineConfigurationManager.updateMasterConfiguration(configuration, startNFS);
    } else {
        log.debug("Launching worker entry {}", entry.getName());
        pipelineConfigurationManager.updateWorkerConfiguration(clusterId, startVO, configuration, startNFS, true);
    }
    Pipeline pipeline = entry.getPipelineId() != null ? pipelineManager.load(entry.getPipelineId()) : null;
    List<PipelineRun> result = new ArrayList<>();
    log.debug("Launching total {} copies of entry {}", copies, entry.getName());
    for (int i = 0; i < copies; i++) {
        // only first node may be a NFS server
        if (i != 0) {
            configuration.setCmdTemplate(WORKER_CMD_TEMPLATE);
            configuration.getParameters().remove(NFS_CLUSTER_ROLE);
            configuration.buildEnvVariables();
        }
        result.add(pipelineRunManager.launchPipeline(configuration, pipeline, entry.getPipelineVersion(), startVO.getInstanceType(), startVO.getParentNodeId(), startVO.getConfigurationName(), clusterId, null, entityIds, configurationId, startVO.getRunSids()));
    }
    return result;
}

Example 13 with PipelineStart

use of com.epam.pipeline.entity.pipeline.run.PipelineStart in project cloud-pipeline by epam.

the class FirecloudRunner method createFirecloudStart.

private PipelineStart createFirecloudStart(FirecloudConfiguration settings, FirecloudRunConfigurationEntry entry, List<Long> entities, Long projectId, Long configurationId, FireCloudClass rootEntityType) {
    PipelineStart startVO = new PipelineStart();
    startVO.setDockerImage(settings.getFirecloudLauncherImage());
    startVO.setConfigurationName(entry.getConfigName());
    startVO.setInstanceType(settings.getInstanceType());
    startVO.setHddSize(settings.getInstanceDisk());
    startVO.setRunSids(entry.getRunSids());
    // TODO: using only on demand instances??
    startVO.setIsSpot(false);
    startVO.setParams(entry.getParameters());
    String cmd = shouldCreateNewMethodConfig(entry) ? buildFirecloudLaunchCommandWithNewConfig(entry, settings, entities, projectId, configurationId, rootEntityType) : buildBasicFirecloudLaunchCommand(entry, settings, entities, projectId);
    startVO.setCmdTemplate(cmd);
    return startVO;
}

Example 14 with PipelineStart

use of com.epam.pipeline.entity.pipeline.run.PipelineStart in project cloud-pipeline by epam.

the class FirecloudRunner method runFirecloudAnalysis.

private PipelineRun runFirecloudAnalysis(FirecloudConfiguration settings, FirecloudRunConfigurationEntry entry, Long configurationId, List<Long> entities, Long projectId) {
    FireCloudClass rootEntityType = metadataEntityManager.loadClass(entry.getRootEntityId()).getFireCloudClassName();
    PipelineStart startVO = createFirecloudStart(settings, entry, entities, projectId, configurationId, rootEntityType);
    PipelineConfiguration runConfiguration = pipelineConfigurationManager.getPipelineConfiguration(startVO);
    runConfiguration.setExecutionPreferences(FirecloudPreferences.builder().method(entry.getMethodName()).methodSnapshot(entry.getMethodSnapshot()).methodConfiguration(entry.getMethodConfigurationName()).methodConfigurationSnapshot(entry.getMethodSnapshot()).build());
    addCredentials(runConfiguration, settings);
    return pipelineRunManager.launchPipeline(runConfiguration, null, null, startVO.getInstanceType(), startVO.getParentNodeId(), startVO.getConfigurationName(), null, null, entities, configurationId, startVO.getRunSids());
}

Example 15 with PipelineStart

use of com.epam.pipeline.entity.pipeline.run.PipelineStart in project cloud-pipeline by epam.

the class PipelineConfigurationForRunnerTest method getPipelineStartVO.

private static PipelineStart getPipelineStartVO() {
    PipelineStart vo = new PipelineStart();
    vo.setInstanceType(TEST_INSTANCE_TYPE);
    vo.setDockerImage(TEST_IMAGE);
    vo.setHddSize(Integer.parseInt(TEST_HDD_SIZE));
    vo.setCmdTemplate(TEST_CMD_TEMPLATE);
    vo.setTimeout(TEST_TIMEOUT);
    vo.setNodeCount(TEST_NODE_COUNT);
    vo.setIsSpot(true);
    vo.setParams(Collections.singletonMap(TEST_PARAM_1, TEST_PARAM_VALUE));
    vo.setPrettyUrl(TEST_PRETTY_URL);
    return vo;
}