Search in sources :

Example 1 with ResolvedConfiguration

use of com.epam.pipeline.entity.pipeline.ResolvedConfiguration in project cloud-pipeline by epam.

the class ParameterMapper method resolveParameters.

/**
 * Gets configuration for a list of {@link AbstractRunConfigurationEntry} resolving
 * template parameters from
 * @param entity        to use for parameters template mapping
 * @param entries       to resolve
 * @param projectData   metadata of associated project {@link com.epam.pipeline.entity.pipeline.Folder}
 * @return configuration for all input entries
 */
public ResolvedConfiguration resolveParameters(MetadataEntity entity, List<? extends AbstractRunConfigurationEntry> entries, Map<String, PipeConfValue> projectData) {
    if (CollectionUtils.isEmpty(entries)) {
        return new ResolvedConfiguration(entity, Collections.emptyMap());
    }
    if (entity == null) {
        return new ResolvedConfiguration(null, entries.stream().collect(Collectors.toMap(AbstractRunConfigurationEntry::getName, this::getEntryConfiguration)));
    }
    Map<MetadataKey, MetadataEntity> entityReferences = loadReferences(entity);
    Map<String, PipelineConfiguration> resolved = new HashMap<>();
    entries.forEach(entry -> {
        checkClassIdMatch(entity, entry.getRootEntityId());
        PipelineConfiguration configuration = getEntryConfiguration(entry);
        if (MapUtils.isNotEmpty(configuration.getParameters())) {
            configuration.setParameters(mapParameters(entity, projectData, configuration.getParameters(), entityReferences));
        }
        resolved.put(entry.getName(), configuration);
    });
    return new ResolvedConfiguration(entity, resolved);
}
Also used : ResolvedConfiguration(com.epam.pipeline.entity.pipeline.ResolvedConfiguration) MetadataEntity(com.epam.pipeline.entity.metadata.MetadataEntity) HashMap(java.util.HashMap) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration)

Example 2 with ResolvedConfiguration

use of com.epam.pipeline.entity.pipeline.ResolvedConfiguration in project cloud-pipeline by epam.

the class ParameterMapper method resolveConfigurations.

public List<ResolvedConfiguration> resolveConfigurations(AnalysisConfiguration<? extends AbstractRunConfigurationEntry> configuration) {
    FolderWithMetadata project = folderManager.getProject(configuration.getConfigurationId(), AclClass.CONFIGURATION);
    Map<String, PipeConfValue> projectData = project == null ? new HashMap<>() : project.getData();
    List<? extends AbstractRunConfigurationEntry> entries = configuration.getEntries();
    if (CollectionUtils.isEmpty(configuration.getEntitiesIds())) {
        return Collections.singletonList(resolveParameters(entries, projectData));
    }
    // In case of array references one entity may be expanded to
    // list of references entities, e.g. SampleSet is expanded
    // to list of Sample entities
    // TODO: The only reason to store it as map - is to add association to run
    // TODO: to initial entity, from which link comes. Find better solution.
    Map<Long, List<MetadataEntity>> targetEntities = fetchAndExpandInputEntities(configuration);
    // resolve all parameter references in configurations
    Map<Long, ResolvedConfiguration> resolvedConfigurations = targetEntities.values().stream().flatMap(Collection::stream).collect(Collectors.toMap(BaseEntity::getId, entity -> resolveParameters(entity, entries, projectData)));
    return targetEntities.entrySet().stream().map(idToEntities -> idToEntities.getValue().stream().map(entity -> {
        ResolvedConfiguration currentConfiguration = resolvedConfigurations.get(entity.getId());
        currentConfiguration.getAssociatedEntityIds().add(idToEntities.getKey());
        return currentConfiguration;
    }).collect(Collectors.toList())).flatMap(Collection::stream).collect(Collectors.toList());
}
Also used : EntityTypeField(com.epam.pipeline.manager.metadata.parser.EntityTypeField) Setter(lombok.Setter) Getter(lombok.Getter) MessageConstants(com.epam.pipeline.common.MessageConstants) RequiredArgsConstructor(lombok.RequiredArgsConstructor) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) PipeConfValueVO(com.epam.pipeline.entity.configuration.PipeConfValueVO) StringUtils(org.apache.commons.lang3.StringUtils) Function(java.util.function.Function) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PipeConfValue(com.epam.pipeline.entity.metadata.PipeConfValue) MessageHelper(com.epam.pipeline.common.MessageHelper) AnalysisConfiguration(com.epam.pipeline.manager.pipeline.runner.AnalysisConfiguration) Service(org.springframework.stereotype.Service) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) FolderWithMetadata(com.epam.pipeline.entity.metadata.FolderWithMetadata) Map(java.util.Map) TypeReference(com.fasterxml.jackson.core.type.TypeReference) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) MapUtils(org.apache.commons.collections4.MapUtils) BaseEntity(com.epam.pipeline.entity.BaseEntity) Logger(org.slf4j.Logger) Collection(java.util.Collection) JsonMapper(com.epam.pipeline.config.JsonMapper) EqualsAndHashCode(lombok.EqualsAndHashCode) Collectors(java.util.stream.Collectors) ResolvedConfiguration(com.epam.pipeline.entity.pipeline.ResolvedConfiguration) Objects(java.util.Objects) List(java.util.List) MetadataEntity(com.epam.pipeline.entity.metadata.MetadataEntity) MetadataEntityManager(com.epam.pipeline.manager.metadata.MetadataEntityManager) AclClass(com.epam.pipeline.entity.security.acl.AclClass) PipelineStart(com.epam.pipeline.entity.pipeline.run.PipelineStart) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) Assert(org.springframework.util.Assert) ResolvedConfiguration(com.epam.pipeline.entity.pipeline.ResolvedConfiguration) PipeConfValue(com.epam.pipeline.entity.metadata.PipeConfValue) ArrayList(java.util.ArrayList) List(java.util.List) FolderWithMetadata(com.epam.pipeline.entity.metadata.FolderWithMetadata)

Example 3 with ResolvedConfiguration

use of com.epam.pipeline.entity.pipeline.ResolvedConfiguration in project cloud-pipeline by epam.

the class CloudPlatformRunner method runConfiguration.

private List<PipelineRun> runConfiguration(Long configurationId, List<RunConfigurationEntry> entries, ResolvedConfiguration resolvedConfigurations) {
    SplitConfig splitConfig = new SplitConfig(entries);
    RunConfigurationEntry mainEntry = splitConfig.getMain();
    List<RunConfigurationEntry> childEntries = splitConfig.getChildEntries();
    boolean isMasterNFSServer = pipelineConfigurationManager.hasNFSParameter(mainEntry.getConfiguration()) || childEntries.stream().noneMatch(entry -> pipelineConfigurationManager.hasNFSParameter(entry.getConfiguration()));
    boolean nfsStarted = isMasterNFSServer;
    PipelineConfiguration mainConfiguration = resolvedConfigurations.getConfiguration(mainEntry.getName());
    List<PipelineConfiguration> childConfigurations = childEntries.stream().map(entry -> resolvedConfigurations.getConfiguration(entry.getName())).collect(Collectors.toList());
    int masterNodeCount = getNodeCount(mainConfiguration.getNodeCount(), 0);
    int totalNodes = childConfigurations.stream().map(PipelineConfiguration::getNodeCount).mapToInt(nodeCount -> getNodeCount(nodeCount, 1)).sum();
    totalNodes += masterNodeCount;
    log.debug("Running total {} nodes", totalNodes + 1);
    mainConfiguration.setNodeCount(totalNodes);
    // create master run
    List<PipelineRun> masterRun = runConfigurationEntry(mainEntry, mainConfiguration, 1, null, isMasterNFSServer, resolvedConfigurations.getAllAssociatedIds(), configurationId);
    List<PipelineRun> launched = new ArrayList<>(masterRun);
    String clusterId = String.valueOf(masterRun.get(0).getId());
    // create master workers
    if (masterNodeCount > 0) {
        mainEntry.getConfiguration().setWorkerCmd(WORKER_CMD_TEMPLATE);
        launched.addAll(runConfigurationEntry(mainEntry, mainConfiguration, masterNodeCount, clusterId, false, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    // create all other workers
    for (int i = 0; i < childConfigurations.size(); i++) {
        PipelineConfiguration childConfig = childConfigurations.get(i);
        boolean startNFS = !nfsStarted && pipelineConfigurationManager.hasNFSParameter(childConfig);
        nfsStarted = nfsStarted || startNFS;
        int copies = getNodeCount(childConfig.getNodeCount(), 1);
        launched.addAll(runConfigurationEntry(childEntries.get(i), childConfig, copies, clusterId, startNFS, resolvedConfigurations.getAllAssociatedIds(), configurationId));
    }
    return launched;
}
Also used : PipelineManager(com.epam.pipeline.manager.pipeline.PipelineManager) MessageConstants(com.epam.pipeline.common.MessageConstants) ParameterMapper(com.epam.pipeline.manager.pipeline.ParameterMapper) PipelineRunManager(com.epam.pipeline.manager.pipeline.PipelineRunManager) RequiredArgsConstructor(lombok.RequiredArgsConstructor) SystemPreferences(com.epam.pipeline.manager.preference.SystemPreferences) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) MessageHelper(com.epam.pipeline.common.MessageHelper) Service(org.springframework.stereotype.Service) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration) Pipeline(com.epam.pipeline.entity.pipeline.Pipeline) PipelineConfigurationManager(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager) WORKER_CMD_TEMPLATE(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager.WORKER_CMD_TEMPLATE) PreferenceManager(com.epam.pipeline.manager.preference.PreferenceManager) Collection(java.util.Collection) RunConfigurationUtils.getNodeCount(com.epam.pipeline.entity.configuration.RunConfigurationUtils.getNodeCount) Collectors(java.util.stream.Collectors) ResolvedConfiguration(com.epam.pipeline.entity.pipeline.ResolvedConfiguration) NFS_CLUSTER_ROLE(com.epam.pipeline.manager.pipeline.PipelineConfigurationManager.NFS_CLUSTER_ROLE) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) Data(lombok.Data) PipelineStart(com.epam.pipeline.entity.pipeline.run.PipelineStart) Assert(org.springframework.util.Assert) StringUtils(org.springframework.util.StringUtils) PipelineRun(com.epam.pipeline.entity.pipeline.PipelineRun) ArrayList(java.util.ArrayList) AbstractRunConfigurationEntry(com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry) RunConfigurationEntry(com.epam.pipeline.entity.configuration.RunConfigurationEntry) PipelineConfiguration(com.epam.pipeline.entity.configuration.PipelineConfiguration)

Aggregations

AbstractRunConfigurationEntry (com.epam.pipeline.entity.configuration.AbstractRunConfigurationEntry)3 PipelineConfiguration (com.epam.pipeline.entity.configuration.PipelineConfiguration)3 ResolvedConfiguration (com.epam.pipeline.entity.pipeline.ResolvedConfiguration)3 MessageConstants (com.epam.pipeline.common.MessageConstants)2 MessageHelper (com.epam.pipeline.common.MessageHelper)2 MetadataEntity (com.epam.pipeline.entity.metadata.MetadataEntity)2 PipelineStart (com.epam.pipeline.entity.pipeline.run.PipelineStart)2 ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2 RequiredArgsConstructor (lombok.RequiredArgsConstructor)2 CollectionUtils (org.apache.commons.collections4.CollectionUtils)2 Service (org.springframework.stereotype.Service)2 Assert (org.springframework.util.Assert)2 JsonMapper (com.epam.pipeline.config.JsonMapper)1 BaseEntity (com.epam.pipeline.entity.BaseEntity)1 PipeConfValueVO (com.epam.pipeline.entity.configuration.PipeConfValueVO)1 RunConfigurationEntry (com.epam.pipeline.entity.configuration.RunConfigurationEntry)1