Examples with Source com.amazon.dataprepper.model.source.Source used on opensource projects

Example 1 with Source

use of com.amazon.dataprepper.model.source.Source in project data-prepper by opensearch-project.

the class SourceFactoryTests method testSourceClassByName.

/**
 * Tests if SourceFactory is able to retrieve default Source plugins by name
 */
@Test
public void testSourceClassByName() {
    final PluginSetting stdInSourceConfiguration = new PluginSetting("stdin", new HashMap<>());
    stdInSourceConfiguration.setPipelineName(TEST_PIPELINE);
    final Source actualSource = SourceFactory.newSource(stdInSourceConfiguration);
    final Source expectedSource = new StdInSource(500, TEST_PIPELINE);
    assertThat(actualSource, notNullValue());
    assertThat(actualSource.getClass().getSimpleName(), is(equalTo(expectedSource.getClass().getSimpleName())));
}

Example 2 with Source

use of com.amazon.dataprepper.model.source.Source in project data-prepper by opensearch-project.

the class PluginFactoryTests method testNoMandatoryConstructor.

@Test
public void testNoMandatoryConstructor() {
    final PluginSetting testPluginSettings = new PluginSetting("junit-test", new HashMap<>());
    final Class<Source> clazz = PluginRepository.getSourceClass(testPluginSettings.getName());
    assertNotNull(clazz);
    try {
        PluginFactory.newPlugin(testPluginSettings, clazz);
    } catch (PluginException e) {
        assertTrue("Incorrect exception or exception message was thrown", e.getMessage().startsWith("Data Prepper plugin requires a constructor with PluginSetting parameter; Plugin " + "ConstructorLessComponent with name junit-test is missing such constructor."));
    }
}

Example 3 with Source

use of com.amazon.dataprepper.model.source.Source in project data-prepper by opensearch-project.

the class PipelineParser method buildPipelineFromConfiguration.

private void buildPipelineFromConfiguration(final String pipelineName, final Map<String, PipelineConfiguration> pipelineConfigurationMap, final Map<String, Pipeline> pipelineMap) {
    final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(pipelineName);
    LOG.info("Building pipeline [{}] from provided configuration", pipelineName);
    try {
        final PluginSetting sourceSetting = pipelineConfiguration.getSourcePluginSetting();
        final Optional<Source> pipelineSource = getSourceIfPipelineType(pipelineName, sourceSetting, pipelineMap, pipelineConfigurationMap);
        final Source source = pipelineSource.orElseGet(() -> pluginFactory.loadPlugin(Source.class, sourceSetting));
        LOG.info("Building buffer for the pipeline [{}]", pipelineName);
        final Buffer buffer = pluginFactory.loadPlugin(Buffer.class, pipelineConfiguration.getBufferPluginSetting());
        LOG.info("Building processors for the pipeline [{}]", pipelineName);
        final int processorThreads = pipelineConfiguration.getWorkers();
        final List<List<Processor>> processorSets = pipelineConfiguration.getProcessorPluginSettings().stream().map(this::newProcessor).collect(Collectors.toList());
        final int readBatchDelay = pipelineConfiguration.getReadBatchDelay();
        LOG.info("Building sinks for the pipeline [{}]", pipelineName);
        final List<Sink> sinks = pipelineConfiguration.getSinkPluginSettings().stream().map(this::buildSinkOrConnector).collect(Collectors.toList());
        final Pipeline pipeline = new Pipeline(pipelineName, source, buffer, processorSets, sinks, processorThreads, readBatchDelay);
        pipelineMap.put(pipelineName, pipeline);
    } catch (Exception ex) {
        // If pipeline construction errors out, we will skip that pipeline and proceed
        LOG.error("Construction of pipeline components failed, skipping building of pipeline [{}] and its connected " + "pipelines", pipelineName, ex);
        processRemoveIfRequired(pipelineName, pipelineConfigurationMap, pipelineMap);
    }
}

Example 4 with Source

use of com.amazon.dataprepper.model.source.Source in project data-prepper by opensearch-project.

the class PluginRepository method findPlugins.

private static void findPlugins() {
    final Reflections reflections = new Reflections(DEFAULT_PLUGINS_CLASSPATH);
    final Set<Class<?>> annotatedClasses = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class);
    for (final Class<?> annotatedClass : annotatedClasses) {
        final DataPrepperPlugin dataPrepperPluginAnnotation = annotatedClass.getAnnotation(DataPrepperPlugin.class);
        final String pluginName = dataPrepperPluginAnnotation.name();
        final PluginType pluginType = extractPluginType(dataPrepperPluginAnnotation);
        switch(pluginType) {
            case SOURCE:
                SOURCES.put(pluginName, (Class<Source>) annotatedClass);
                break;
            case BUFFER:
                BUFFERS.put(pluginName, (Class<Buffer>) annotatedClass);
                break;
            case PREPPER:
                PROCESSORS.put(pluginName, (Class<Processor>) annotatedClass);
                break;
            case SINK:
                SINKS.put(pluginName, (Class<Sink>) annotatedClass);
                break;
        }
    }
}