Search in sources :

Example 1 with RunConfigurationEntry

use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.

the class CloudPlatformRunner method runConfiguration.

private List<PipelineRun> runConfiguration(Long configurationId, List<RunConfigurationEntry> entries, ResolvedConfiguration resolvedConfigurations) {
    SplitConfig splitConfig = new SplitConfig(entries);
    RunConfigurationEntry mainEntry = splitConfig.getMain();
    List<RunConfigurationEntry> childEntries = splitConfig.getChildEntries();
    boolean isMasterNFSServer = pipelineConfigurationManager.hasNFSParameter(mainEntry.getConfiguration()) || childEntries.stream().noneMatch(entry -> pipelineConfigurationManager.hasNFSParameter(entry.getConfiguration()));
    boolean nfsStarted = isMasterNFSServer;
    PipelineConfiguration mainConfiguration = resolvedConfigurations.getConfiguration(mainEntry.getName());
    List<PipelineConfiguration> childConfigurations = childEntries.stream().map(entry -> resolvedConfigurations.getConfiguration(entry.getName())).collect(Collectors.toList());
    int masterNodeCount = getNodeCount(mainConfiguration.getNodeCount(), 0);
    int totalNodes = childConfigurations.stream().map(PipelineConfiguration::getNodeCount).mapToInt(nodeCount -> getNodeCount(nodeCount, 1)).sum();
    totalNodes += masterNodeCount;
    log.debug("Running total {} nodes", totalNodes + 1);
    mainConfiguration.setNodeCount(totalNodes);
    // create master run
    List<PipelineRun> masterRun = runConfigurationEntry(mainEntry, mainConfiguration, 1, null, isMasterNFSServer, resolvedConfigurations.getAllAssociatedIds(), configurationId);
    List<PipelineRun> launched = new ArrayList<>(masterRun);
    String clusterId = String.valueOf(masterRun.get(0).getId());
    // create master workers
    if (masterNodeCount > 0) {
        mainEntry.getConfiguration().setWorkerCmd(WORKER_CMD_TEMPLATE);
        launched.addAll(runConfigurationEntry(mainEntry, mainConfiguration, masterNodeCount, clusterId, false, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    // create all other workers
    for (int i = 0; i < childConfigurations.size(); i++) {
        PipelineConfiguration childConfig = childConfigurations.get(i);
        boolean startNFS = !nfsStarted && pipelineConfigurationManager.hasNFSParameter(childConfig);
        nfsStarted = nfsStarted || startNFS;
        int copies = getNodeCount(childConfig.getNodeCount(), 1);
        launched.addAll(runConfigurationEntry(childEntries.get(i), childConfig, copies, clusterId, startNFS, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    return launched;
}
Also used : PipelineManager(com.epam.pipeline.manager.pipeline.PipelineManager) MessageConstants(com.epam.pipeline.common.MessageConstants) ParameterMapper(com.epam.pipeline.manager.pipeline.ParameterMapper) PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) RequiredArgsConstructor(lombok.RequiredArgsConstructor) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) MessageHelper(com.epam.pipeline.common.MessageHelper) Service(org.springframework.stereotype.Service) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) Pipeline(com.epam.pipeline.entity.pipeline.Pipeline) PipelineConfigurationManager(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager) WORKER_CMD_TEMPLATE(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager.WORKER_CMD_TEMPLATE) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Collection(java.util.Collection) RunConfigurationUtils.getNodeCount(com.epam.pipeline.entity.configuration.RunConfigurationUtils.getNodeCount) Collectors(java.util.stream.Collectors) ResolvedConfiguration(com.epam.pipeline.entity.pipeline.ResolvedConfiguration) NFS_CLUSTER_ROLE(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager.NFS_CLUSTER_ROLE) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Data(lombok.Data) PipelineStart(com.epam.pipeline.entity.pipeline.run.PipelineStart) Assert(org.springframework.util.Assert) StringUtils(org.springframework.util.StringUtils) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) ArrayList(java.util.ArrayList) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration)

Example 2 with RunConfigurationEntry

use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.

the class FolderDaoTest method addConfiguration.

private RunConfiguration addConfiguration(Folder folder) {
    RunConfigurationEntry entry = ObjectCreatorUtils.createConfigEntry(TEST_NAME, true, new PipelineConfiguration());
    RunConfiguration configuration = ObjectCreatorUtils.createConfiguration(TEST_NAME, TEST_NAME, folder.getId(), TEST_USER, Collections.singletonList(entry));
    return configurationDao.create(configuration);
}
Also used : RunConfiguration(com.epam.pipeline.entity.configuration.RunConfiguration) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration)

Example 3 with RunConfigurationEntry

use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.

the class RunConfigurationProviderTest method validateShouldUseToolResourceWhileCheckingToolAllowedIfDockerImageIsSpecified.

@Test
public void validateShouldUseToolResourceWhileCheckingToolAllowedIfDockerImageIsSpecified() {
    final PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
    pipelineConfiguration.setInstanceType(ALLOWED_INSTANCE_TYPE);
    pipelineConfiguration.setDockerImage(TOOL_IMAGE);
    final RunConfigurationEntry runConfigurationEntry = new RunConfigurationEntry();
    runConfigurationEntry.setConfiguration(pipelineConfiguration);
    runConfigurationProvider.validateEntry(runConfigurationEntry);
    verify(instanceOfferManager).isToolInstanceAllowed(any(), eq(TOOL_RESOURCE));
}
Also used : PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) Test(org.junit.Test)

Example 4 with RunConfigurationEntry

use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.

the class RunConfigurationProviderTest method validateShouldFailIfConfigurationInstanceTypeIsNotAllowed.

@Test
public void validateShouldFailIfConfigurationInstanceTypeIsNotAllowed() {
    final PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
    pipelineConfiguration.setInstanceType(NOT_ALLOWED_INSTANCE_TYPE);
    final RunConfigurationEntry runConfigurationEntry = new RunConfigurationEntry();
    runConfigurationEntry.setConfiguration(pipelineConfiguration);
    assertThrows(IllegalArgumentException.class, () -> runConfigurationProvider.validateEntry(runConfigurationEntry));
    verify(instanceOfferManager).isToolInstanceAllowed(eq(NOT_ALLOWED_INSTANCE_TYPE), any());
}
Also used : PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) Test(org.junit.Test)

Example 5 with RunConfigurationEntry

use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.

the class RunConfigurationLoaderTest method shouldLoadRunConfigurationTest.

@Test
void shouldLoadRunConfigurationTest() throws EntityNotFoundException {
    Pipeline pipeline = new Pipeline();
    pipeline.setId(1L);
    pipeline.setName(TEST_NAME);
    PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
    pipelineConfiguration.setDockerImage(TEST_PATH);
    RunConfigurationEntry entry = new RunConfigurationEntry();
    entry.setPipelineId(pipeline.getId());
    entry.setPipelineVersion(TEST_VERSION);
    entry.setName(TEST_NAME);
    entry.setConfiguration(pipelineConfiguration);
    RunConfiguration runConfiguration = new RunConfiguration();
    runConfiguration.setId(1L);
    runConfiguration.setName(TEST_NAME);
    runConfiguration.setDescription(TEST_VALUE);
    runConfiguration.setOwner(TEST_NAME);
    runConfiguration.setEntries(Collections.singletonList(entry));
    RunConfigurationDoc expectedConfiguration = RunConfigurationDoc.builder().configuration(runConfiguration).pipelines(Collections.singletonList(pipeline)).build();
    RunConfigurationLoader runConfigurationLoader = new RunConfigurationLoader(apiClient);
    when(apiClient.loadRunConfiguration(anyLong())).thenReturn(runConfiguration);
    when(apiClient.loadPipeline(anyString())).thenReturn(pipeline);
    Optional<EntityContainer<RunConfigurationDoc>> container = runConfigurationLoader.loadEntity(1L);
    EntityContainer<RunConfigurationDoc> configurationDocContainer = container.orElseThrow(AssertionError::new);
    RunConfigurationDoc actualConfiguration = configurationDocContainer.getEntity();
    assertNotNull(actualConfiguration);
    verifyConfiguration(expectedConfiguration.getConfiguration(), actualConfiguration.getConfiguration());
    verifyPipeline(expectedConfiguration.getPipelines(), actualConfiguration.getPipelines());
    verifyPipelineUser(configurationDocContainer.getOwner());
    verifyPermissions(PERMISSIONS_CONTAINER_WITH_OWNER, configurationDocContainer.getPermissions());
    verifyMetadata(EXPECTED_METADATA, new ArrayList<>(configurationDocContainer.getMetadata().values()));
}
Also used : RunConfiguration(com.epam.pipeline.entity.configuration.RunConfiguration) RunConfigurationDoc(com.epam.pipeline.elasticsearchagent.model.RunConfigurationDoc) EntityContainer(com.epam.pipeline.elasticsearchagent.model.EntityContainer) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) Pipeline(com.epam.pipeline.entity.pipeline.Pipeline) LoaderVerificationUtils.verifyPipeline(com.epam.pipeline.elasticsearchagent.LoaderVerificationUtils.verifyPipeline) Test(org.junit.jupiter.api.Test)

Aggregations

RunConfigurationEntry (com.epam.pipeline.entity.configuration.RunConfigurationEntry)13 PipelineConfiguration (com.epam.pipeline.entity.configuration.PipelineConfiguration)10 Test (org.junit.Test)7 RunConfiguration (com.epam.pipeline.entity.configuration.RunConfiguration)6 FirecloudRunConfigurationEntry (com.epam.pipeline.entity.configuration.FirecloudRunConfigurationEntry)4 Pipeline (com.epam.pipeline.entity.pipeline.Pipeline)4 AbstractSpringTest (com.epam.pipeline.AbstractSpringTest)3 AbstractRunConfigurationEntry (com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry)3 Transactional (org.springframework.transaction.annotation.Transactional)3 Folder (com.epam.pipeline.entity.pipeline.Folder)2 Before (org.junit.Before)2 MessageConstants (com.epam.pipeline.common.MessageConstants)1 MessageHelper (com.epam.pipeline.common.MessageHelper)1 AwsRegionVO (com.epam.pipeline.controller.vo.AwsRegionVO)1 DataStorageVO (com.epam.pipeline.controller.vo.DataStorageVO)1 EntityVO (com.epam.pipeline.controller.vo.EntityVO)1 MetadataVO (com.epam.pipeline.controller.vo.MetadataVO)1 PipelineVO (com.epam.pipeline.controller.vo.PipelineVO)1 RunConfigurationVO (com.epam.pipeline.controller.vo.configuration.RunConfigurationVO)1 MetadataEntityVO (com.epam.pipeline.controller.vo.metadata.MetadataEntityVO)1