Examples with PipelineConfiguration com.epam.pipeline.entity.configuration.PipelineConfiguration used on opensource projects

Example 6 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class PipelineConfigurationManager method getPipelineConfiguration.

public PipelineConfiguration getPipelineConfiguration(final PipelineStart runVO, final Tool tool) {
    PipelineConfiguration configuration;
    PipelineConfiguration defaultConfiguration = new PipelineConfiguration();
    boolean toolRun = tool != null;
    boolean pipelineRun = runVO.getPipelineId() != null;
    if (toolRun || pipelineRun) {
        defaultConfiguration = getConfigurationForToolRunOrPipelineRun(runVO, tool, toolRun);
    }
    configuration = mergeParameters(runVO, defaultConfiguration);
    if (pipelineRun) {
        configuration.setGitCredentials(gitManager.getGitCredentials(runVO.getPipelineId()));
    }
    if (toolRun) {
        mergeParametersFromTool(configuration, tool);
    }
    List<AbstractDataStorage> dataStorages = dataStorageApiService.getWritableStorages();
    configuration.setBuckets(zipToString(dataStorages, AbstractDataStorage::getPathMask));
    configuration.setNfsMountOptions(zipToString(dataStorages, mountOptionsSupplier));
    configuration.setMountPoints(zipToString(dataStorages, AbstractDataStorage::getMountPoint));
    // client always sends actual node count value
    configuration.setNodeCount(Optional.ofNullable(runVO.getNodeCount()).orElse(0));
    configuration.setAwsRegionId(runVO.getAwsRegionId());
    setEndpointsErasure(configuration);
    return configuration;
}

Example 7 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class CloudPlatformRunner method runConfiguration.

private List<PipelineRun> runConfiguration(Long configurationId, List<RunConfigurationEntry> entries, ResolvedConfiguration resolvedConfigurations) {
    SplitConfig splitConfig = new SplitConfig(entries);
    RunConfigurationEntry mainEntry = splitConfig.getMain();
    List<RunConfigurationEntry> childEntries = splitConfig.getChildEntries();
    boolean isMasterNFSServer = pipelineConfigurationManager.hasNFSParameter(mainEntry.getConfiguration()) || childEntries.stream().noneMatch(entry -> pipelineConfigurationManager.hasNFSParameter(entry.getConfiguration()));
    boolean nfsStarted = isMasterNFSServer;
    PipelineConfiguration mainConfiguration = resolvedConfigurations.getConfiguration(mainEntry.getName());
    List<PipelineConfiguration> childConfigurations = childEntries.stream().map(entry -> resolvedConfigurations.getConfiguration(entry.getName())).collect(Collectors.toList());
    int masterNodeCount = getNodeCount(mainConfiguration.getNodeCount(), 0);
    int totalNodes = childConfigurations.stream().map(PipelineConfiguration::getNodeCount).mapToInt(nodeCount -> getNodeCount(nodeCount, 1)).sum();
    totalNodes += masterNodeCount;
    log.debug("Running total {} nodes", totalNodes + 1);
    mainConfiguration.setNodeCount(totalNodes);
    // create master run
    List<PipelineRun> masterRun = runConfigurationEntry(mainEntry, mainConfiguration, 1, null, isMasterNFSServer, resolvedConfigurations.getAllAssociatedIds(), configurationId);
    List<PipelineRun> launched = new ArrayList<>(masterRun);
    String clusterId = String.valueOf(masterRun.get(0).getId());
    // create master workers
    if (masterNodeCount > 0) {
        mainEntry.getConfiguration().setWorkerCmd(WORKER_CMD_TEMPLATE);
        launched.addAll(runConfigurationEntry(mainEntry, mainConfiguration, masterNodeCount, clusterId, false, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    // create all other workers
    for (int i = 0; i < childConfigurations.size(); i++) {
        PipelineConfiguration childConfig = childConfigurations.get(i);
        boolean startNFS = !nfsStarted && pipelineConfigurationManager.hasNFSParameter(childConfig);
        nfsStarted = nfsStarted || startNFS;
        int copies = getNodeCount(childConfig.getNodeCount(), 1);
        launched.addAll(runConfigurationEntry(childEntries.get(i), childConfig, copies, clusterId, startNFS, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    return launched;
}

Example 8 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class RunConfigurationDaoTest method getTestConfig.

public PipelineConfiguration getTestConfig() {
    PipelineConfiguration configuration = new PipelineConfiguration();
    configuration.setCmdTemplate(TEST_CMD_TEMPLATE);
    configuration.setDockerImage(TEST_DOCKER);
    configuration.setInstanceDisk(TEST_DISK);
    configuration.setInstanceType(TEST_INSTANCE);
    return configuration;
}

Example 9 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class FolderDaoTest method addConfiguration.

private RunConfiguration addConfiguration(Folder folder) {
    RunConfigurationEntry entry = ObjectCreatorUtils.createConfigEntry(TEST_NAME, true, new PipelineConfiguration());
    RunConfiguration configuration = ObjectCreatorUtils.createConfiguration(TEST_NAME, TEST_NAME, folder.getId(), TEST_USER, Collections.singletonList(entry));
    return configurationDao.create(configuration);
}

Example 10 with PipelineConfiguration

use of com.epam.pipeline.entity.configuration.PipelineConfiguration in project cloud-pipeline by epam.

the class PipelineRunDaoTest method createTestPipelineRun.

private PipelineRun createTestPipelineRun(Long pipelineId) {
    PipelineRun run = new PipelineRun();
    run.setPipelineId(pipelineId);
    run.setVersion("abcdefg");
    run.setStartDate(new Date());
    run.setEndDate(run.getStartDate());
    run.setStatus(TaskStatus.RUNNING);
    run.setCommitStatus(CommitStatus.NOT_COMMITTED);
    run.setLastChangeCommitTime(new Date());
    run.setPodId(TEST_POD_ID);
    run.setParams(TEST_PARAMS);
    run.setOwner(USER);
    run.setServiceUrl(TEST_SERVICE_URL);
    Map<SystemParams, String> systemParams = EnvVarsBuilderTest.matchSystemParams();
    PipelineConfiguration configuration = EnvVarsBuilderTest.matchPipeConfig();
    EnvVarsBuilder.buildEnvVars(run, configuration, systemParams, null);
    run.setEnvVars(run.getEnvVars());
    pipelineRunDao.createPipelineRun(run);
    return run;
}