use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.
the class CloudPlatformRunner method runConfiguration.
private List<PipelineRun> runConfiguration(Long configurationId, List<RunConfigurationEntry> entries, ResolvedConfiguration resolvedConfigurations) {
SplitConfig splitConfig = new SplitConfig(entries);
RunConfigurationEntry mainEntry = splitConfig.getMain();
List<RunConfigurationEntry> childEntries = splitConfig.getChildEntries();
boolean isMasterNFSServer = pipelineConfigurationManager.hasNFSParameter(mainEntry.getConfiguration()) || childEntries.stream().noneMatch(entry -> pipelineConfigurationManager.hasNFSParameter(entry.getConfiguration()));
boolean nfsStarted = isMasterNFSServer;
PipelineConfiguration mainConfiguration = resolvedConfigurations.getConfiguration(mainEntry.getName());
List<PipelineConfiguration> childConfigurations = childEntries.stream().map(entry -> resolvedConfigurations.getConfiguration(entry.getName())).collect(Collectors.toList());
int masterNodeCount = getNodeCount(mainConfiguration.getNodeCount(), 0);
int totalNodes = childConfigurations.stream().map(PipelineConfiguration::getNodeCount).mapToInt(nodeCount -> getNodeCount(nodeCount, 1)).sum();
totalNodes += masterNodeCount;
log.debug("Running total {} nodes", totalNodes + 1);
mainConfiguration.setNodeCount(totalNodes);
// create master run
List<PipelineRun> masterRun = runConfigurationEntry(mainEntry, mainConfiguration, 1, null, isMasterNFSServer, resolvedConfigurations.getAllAssociatedIds(), configurationId);
List<PipelineRun> launched = new ArrayList<>(masterRun);
String clusterId = String.valueOf(masterRun.get(0).getId());
// create master workers
if (masterNodeCount > 0) {
mainEntry.getConfiguration().setWorkerCmd(WORKER_CMD_TEMPLATE);
launched.addAll(runConfigurationEntry(mainEntry, mainConfiguration, masterNodeCount, clusterId, false, resolvedConfigurations.getAllAssociatedIds(), configurationId));
}
// create all other workers
for (int i = 0; i < childConfigurations.size(); i++) {
PipelineConfiguration childConfig = childConfigurations.get(i);
boolean startNFS = !nfsStarted && pipelineConfigurationManager.hasNFSParameter(childConfig);
nfsStarted = nfsStarted || startNFS;
int copies = getNodeCount(childConfig.getNodeCount(), 1);
launched.addAll(runConfigurationEntry(childEntries.get(i), childConfig, copies, clusterId, startNFS, resolvedConfigurations.getAllAssociatedIds(), configurationId));
}
return launched;
}
use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.
the class FolderDaoTest method addConfiguration.
private RunConfiguration addConfiguration(Folder folder) {
RunConfigurationEntry entry = ObjectCreatorUtils.createConfigEntry(TEST_NAME, true, new PipelineConfiguration());
RunConfiguration configuration = ObjectCreatorUtils.createConfiguration(TEST_NAME, TEST_NAME, folder.getId(), TEST_USER, Collections.singletonList(entry));
return configurationDao.create(configuration);
}
use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.
the class RunConfigurationProviderTest method validateShouldUseToolResourceWhileCheckingToolAllowedIfDockerImageIsSpecified.
@Test
public void validateShouldUseToolResourceWhileCheckingToolAllowedIfDockerImageIsSpecified() {
final PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
pipelineConfiguration.setInstanceType(ALLOWED_INSTANCE_TYPE);
pipelineConfiguration.setDockerImage(TOOL_IMAGE);
final RunConfigurationEntry runConfigurationEntry = new RunConfigurationEntry();
runConfigurationEntry.setConfiguration(pipelineConfiguration);
runConfigurationProvider.validateEntry(runConfigurationEntry);
verify(instanceOfferManager).isToolInstanceAllowed(any(), eq(TOOL_RESOURCE));
}
use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.
the class RunConfigurationProviderTest method validateShouldFailIfConfigurationInstanceTypeIsNotAllowed.
@Test
public void validateShouldFailIfConfigurationInstanceTypeIsNotAllowed() {
final PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
pipelineConfiguration.setInstanceType(NOT_ALLOWED_INSTANCE_TYPE);
final RunConfigurationEntry runConfigurationEntry = new RunConfigurationEntry();
runConfigurationEntry.setConfiguration(pipelineConfiguration);
assertThrows(IllegalArgumentException.class, () -> runConfigurationProvider.validateEntry(runConfigurationEntry));
verify(instanceOfferManager).isToolInstanceAllowed(eq(NOT_ALLOWED_INSTANCE_TYPE), any());
}
use of com.epam.pipeline.entity.configuration.RunConfigurationEntry in project cloud-pipeline by epam.
the class RunConfigurationLoaderTest method shouldLoadRunConfigurationTest.
@Test
void shouldLoadRunConfigurationTest() throws EntityNotFoundException {
Pipeline pipeline = new Pipeline();
pipeline.setId(1L);
pipeline.setName(TEST_NAME);
PipelineConfiguration pipelineConfiguration = new PipelineConfiguration();
pipelineConfiguration.setDockerImage(TEST_PATH);
RunConfigurationEntry entry = new RunConfigurationEntry();
entry.setPipelineId(pipeline.getId());
entry.setPipelineVersion(TEST_VERSION);
entry.setName(TEST_NAME);
entry.setConfiguration(pipelineConfiguration);
RunConfiguration runConfiguration = new RunConfiguration();
runConfiguration.setId(1L);
runConfiguration.setName(TEST_NAME);
runConfiguration.setDescription(TEST_VALUE);
runConfiguration.setOwner(TEST_NAME);
runConfiguration.setEntries(Collections.singletonList(entry));
RunConfigurationDoc expectedConfiguration = RunConfigurationDoc.builder().configuration(runConfiguration).pipelines(Collections.singletonList(pipeline)).build();
RunConfigurationLoader runConfigurationLoader = new RunConfigurationLoader(apiClient);
when(apiClient.loadRunConfiguration(anyLong())).thenReturn(runConfiguration);
when(apiClient.loadPipeline(anyString())).thenReturn(pipeline);
Optional<EntityContainer<RunConfigurationDoc>> container = runConfigurationLoader.loadEntity(1L);
EntityContainer<RunConfigurationDoc> configurationDocContainer = container.orElseThrow(AssertionError::new);
RunConfigurationDoc actualConfiguration = configurationDocContainer.getEntity();
assertNotNull(actualConfiguration);
verifyConfiguration(expectedConfiguration.getConfiguration(), actualConfiguration.getConfiguration());
verifyPipeline(expectedConfiguration.getPipelines(), actualConfiguration.getPipelines());
verifyPipelineUser(configurationDocContainer.getOwner());
verifyPermissions(PERMISSIONS_CONTAINER_WITH_OWNER, configurationDocContainer.getPermissions());
verifyMetadata(EXPECTED_METADATA, new ArrayList<>(configurationDocContainer.getMetadata().values()));
}
Aggregations