Examples with PipelineConfiguration com.amazon.dataprepper.parser.model.PipelineConfiguration used on opensource projects

Example 1 with PipelineConfiguration

use of com.amazon.dataprepper.parser.model.PipelineConfiguration in project data-prepper by opensearch-project.

the class PipelineConfigurationValidator method validateSourceMapping.

/**
 * This method validates if pipeline's source is correctly configured to reflect the sink of its parent i.e.
 * if p2 is defined as sink for p1, source of p2 should be defined as p1.
 *
 * @param sourcePipeline           name of the expected source pipeline
 * @param currentPipeline          name of the current pipeline that is being validated
 * @param pipelineConfigurationMap pipeline name to pipeline configuration map
 */
private static void validateSourceMapping(final String sourcePipeline, final String currentPipeline, final Map<String, PipelineConfiguration> pipelineConfigurationMap) {
    if (!pipelineConfigurationMap.containsKey(currentPipeline)) {
        throw new RuntimeException(format("Invalid configuration, no pipeline is defined with name %s", currentPipeline));
    }
    final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(currentPipeline);
    final PluginSetting sourcePluginSettings = pipelineConfiguration.getSourcePluginSetting();
    if (!isPipelineAttributeExists(sourcePluginSettings, sourcePipeline)) {
        LOG.error("Invalid configuration, expected source {} for pipeline {} is missing", sourcePipeline, currentPipeline);
        throw new RuntimeException(format("Invalid configuration, expected source %s for pipeline %s is missing", sourcePipeline, currentPipeline));
    }
}

Example 2 with PipelineConfiguration

use of com.amazon.dataprepper.parser.model.PipelineConfiguration in project data-prepper by opensearch-project.

the class PipelineConfigurationValidator method visitAndValidate.

private static void visitAndValidate(final String pipeline, final Map<String, PipelineConfiguration> pipelineConfigurationMap, final Set<String> touchedPipelineSet, final Set<String> visitedAndProcessedPipelineSet, final List<String> sortedPipelineNames) {
    // if it is already marked, it means it results in a cycle
    if (touchedPipelineSet.contains(pipeline)) {
        LOG.error("Configuration results in a cycle - check pipeline: {}", pipeline);
        throw new RuntimeException(format("Provided configuration results in a loop, check pipeline: %s", pipeline));
    }
    // if its not already visited, recursively check
    if (!visitedAndProcessedPipelineSet.contains(pipeline)) {
        final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(pipeline);
        touchedPipelineSet.add(pipeline);
        // if validation is successful, then there is definitely sink
        final List<PluginSetting> connectedPipelinesSettings = pipelineConfiguration.getSinkPluginSettings();
        // Recursively check connected pipelines
        for (PluginSetting pluginSetting : connectedPipelinesSettings) {
            // Further process only if the sink is of pipeline type
            if (pluginSetting.getName().equals(PIPELINE_TYPE)) {
                final String connectedPipelineName = (String) pluginSetting.getAttributeFromSettings(PIPELINE_ATTRIBUTE_NAME);
                validatePipelineAttributeName(connectedPipelineName, pipeline);
                validateSourceMapping(pipeline, connectedPipelineName, pipelineConfigurationMap);
                visitAndValidate(connectedPipelineName, pipelineConfigurationMap, touchedPipelineSet, visitedAndProcessedPipelineSet, sortedPipelineNames);
            }
        }
        visitedAndProcessedPipelineSet.add(pipeline);
        touchedPipelineSet.remove(pipeline);
        sortedPipelineNames.add(pipeline);
    }
}

Example 3 with PipelineConfiguration

use of com.amazon.dataprepper.parser.model.PipelineConfiguration in project data-prepper by opensearch-project.

the class PipelineConfigurationValidator method validateForOrphans.

/**
 * Validates for orphan pipeline configurations causing ambiguous execution model.
 * TODO: Should this be removed? (unused code)
 *
 * @param sortedPipelines          pipeline names sorted in reverse order
 * @param pipelineConfigurationMap Map of pipeline name and configuration
 */
private static void validateForOrphans(final List<String> sortedPipelines, final Map<String, PipelineConfiguration> pipelineConfigurationMap) {
    final Set<String> expectedPipelineSet = new HashSet<>();
    // Add root pipeline name to expected set
    expectedPipelineSet.add(sortedPipelines.get(0));
    for (String currentPipelineName : sortedPipelines) {
        if (!expectedPipelineSet.contains(currentPipelineName)) {
            throw new RuntimeException("Invalid configuration, cannot proceed with ambiguous configuration");
        }
        final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(currentPipelineName);
        final List<PluginSetting> pluginSettings = pipelineConfiguration.getSinkPluginSettings();
        for (PluginSetting pluginSetting : pluginSettings) {
            if (PIPELINE_TYPE.equals(pluginSetting.getName()) && pluginSetting.getAttributeFromSettings(PIPELINE_ATTRIBUTE_NAME) != null) {
                // Add next set of pipeline names to expected set
                expectedPipelineSet.add((String) pluginSetting.getAttributeFromSettings(PIPELINE_ATTRIBUTE_NAME));
            }
        }
    }
}

Example 4 with PipelineConfiguration

use of com.amazon.dataprepper.parser.model.PipelineConfiguration in project data-prepper by opensearch-project.

the class PipelineParser method removeConnectedPipelines.

/**
 * This removes all built connected pipelines of given pipeline from pipelineMap.
 * TODO Update this to be more elegant and trigger destroy of plugins
 */
private void removeConnectedPipelines(final String failedPipeline, final Map<String, PipelineConfiguration> pipelineConfigurationMap, final Map<String, Pipeline> pipelineMap) {
    final PipelineConfiguration failedPipelineConfiguration = pipelineConfigurationMap.remove(failedPipeline);
    // remove source connected pipelines
    final Optional<String> sourcePipelineOptional = getPipelineNameIfPipelineType(failedPipelineConfiguration.getSourcePluginSetting());
    sourcePipelineOptional.ifPresent(sourcePipeline -> processRemoveIfRequired(sourcePipeline, pipelineConfigurationMap, pipelineMap));
    // remove sink connected pipelines
    final List<PluginSetting> sinkPluginSettings = failedPipelineConfiguration.getSinkPluginSettings();
    sinkPluginSettings.forEach(sinkPluginSetting -> {
        getPipelineNameIfPipelineType(sinkPluginSetting).ifPresent(sinkPipeline -> processRemoveIfRequired(sinkPipeline, pipelineConfigurationMap, pipelineMap));
    });
}

Example 5 with PipelineConfiguration

use of com.amazon.dataprepper.parser.model.PipelineConfiguration in project data-prepper by opensearch-project.

the class PipelineParser method buildPipelineFromConfiguration.

private void buildPipelineFromConfiguration(final String pipelineName, final Map<String, PipelineConfiguration> pipelineConfigurationMap, final Map<String, Pipeline> pipelineMap) {
    final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(pipelineName);
    LOG.info("Building pipeline [{}] from provided configuration", pipelineName);
    try {
        final PluginSetting sourceSetting = pipelineConfiguration.getSourcePluginSetting();
        final Optional<Source> pipelineSource = getSourceIfPipelineType(pipelineName, sourceSetting, pipelineMap, pipelineConfigurationMap);
        final Source source = pipelineSource.orElseGet(() -> pluginFactory.loadPlugin(Source.class, sourceSetting));
        LOG.info("Building buffer for the pipeline [{}]", pipelineName);
        final Buffer buffer = pluginFactory.loadPlugin(Buffer.class, pipelineConfiguration.getBufferPluginSetting());
        LOG.info("Building processors for the pipeline [{}]", pipelineName);
        final int processorThreads = pipelineConfiguration.getWorkers();
        final List<List<Processor>> processorSets = pipelineConfiguration.getProcessorPluginSettings().stream().map(this::newProcessor).collect(Collectors.toList());
        final int readBatchDelay = pipelineConfiguration.getReadBatchDelay();
        LOG.info("Building sinks for the pipeline [{}]", pipelineName);
        final List<Sink> sinks = pipelineConfiguration.getSinkPluginSettings().stream().map(this::buildSinkOrConnector).collect(Collectors.toList());
        final Pipeline pipeline = new Pipeline(pipelineName, source, buffer, processorSets, sinks, processorThreads, readBatchDelay);
        pipelineMap.put(pipelineName, pipeline);
    } catch (Exception ex) {
        // If pipeline construction errors out, we will skip that pipeline and proceed
        LOG.error("Construction of pipeline components failed, skipping building of pipeline [{}] and its connected " + "pipelines", pipelineName, ex);
        processRemoveIfRequired(pipelineName, pipelineConfigurationMap, pipelineMap);
    }
}