Examples with Processor com.amazon.dataprepper.model.processor.Processor used on opensource projects

Example 1 with Processor

use of com.amazon.dataprepper.model.processor.Processor in project data-prepper by opensearch-project.

the class PrepperFactoryTests method testNewSingletonPrepperClassByNameThatExists.

/**
 * Tests if PrepperFactory is able to retrieve default Source plugins by name
 */
@Test
public void testNewSingletonPrepperClassByNameThatExists() {
    PrepperFactory.dangerousMethod_setPluginFunction((s, c) -> new NoOpPrepper());
    final PluginSetting noOpPrepperConfiguration = new PluginSetting("no-op", new HashMap<>());
    noOpPrepperConfiguration.setPipelineName(TEST_PIPELINE);
    final List<Processor> actualPrepperSets = PrepperFactory.newPreppers(noOpPrepperConfiguration);
    assertEquals(1, actualPrepperSets.size());
    final Processor actualPrepper = actualPrepperSets.get(0);
    final Processor expectedPrepper = new NoOpPrepper();
    assertThat(actualPrepper, notNullValue());
    assertThat(actualPrepper.getClass().getSimpleName(), is(equalTo(expectedPrepper.getClass().getSimpleName())));
}

Example 2 with Processor

use of com.amazon.dataprepper.model.processor.Processor in project data-prepper by opensearch-project.

the class PrepperFactoryTests method testNewMultiInstancePrepperClassByNameThatExists.

@Test
public void testNewMultiInstancePrepperClassByNameThatExists() {
    final PluginSetting testPrepperConfiguration = new PluginSetting("test_prepper", new HashMap<>());
    testPrepperConfiguration.setProcessWorkers(2);
    testPrepperConfiguration.setPipelineName(TEST_PIPELINE);
    final List<Processor> actualPrepperSets = PrepperFactory.newPreppers(testPrepperConfiguration);
    assertEquals(2, actualPrepperSets.size());
    final Processor expectedPrepper = new TestPrepper(testPrepperConfiguration);
    assertThat(actualPrepperSets.get(0), notNullValue());
    assertThat(actualPrepperSets.get(0).getClass().getSimpleName(), is(equalTo(expectedPrepper.getClass().getSimpleName())));
    assertThat(actualPrepperSets.get(1), notNullValue());
    assertThat(actualPrepperSets.get(1).getClass().getSimpleName(), is(equalTo(expectedPrepper.getClass().getSimpleName())));
}

Example 3 with Processor

use of com.amazon.dataprepper.model.processor.Processor in project data-prepper by opensearch-project.

the class ProcessWorker method run.

@Override
public void run() {
    try {
        do {
            final Map.Entry<Collection, CheckpointState> readResult = readBuffer.read(pipeline.getReadBatchTimeoutInMillis());
            Collection records = readResult.getKey();
            final CheckpointState checkpointState = readResult.getValue();
            // TODO Hacky way to avoid logging continuously - Will be removed as part of metrics implementation
            if (records.isEmpty()) {
                if (!isEmptyRecordsLogged) {
                    LOG.info(" {} Worker: No records received from buffer", pipeline.getName());
                    isEmptyRecordsLogged = true;
                }
            } else {
                LOG.info(" {} Worker: Processing {} records from buffer", pipeline.getName(), records.size());
            }
            // Should Empty list from buffer should be sent to the processors? For now sending as the Stateful processors expects it.
            for (final Processor processor : processors) {
                records = processor.execute(records);
            }
            if (!records.isEmpty()) {
                postToSink(records);
            }
            // Checkpoint the current batch read from the buffer after being processed by processors and sinks.
            readBuffer.checkpoint(checkpointState);
        } while (!shouldStop());
    } catch (final Exception e) {
        LOG.error("Encountered exception during pipeline {} processing", pipeline.getName(), e);
    }
}

Example 4 with Processor

use of com.amazon.dataprepper.model.processor.Processor in project data-prepper by opensearch-project.

the class PluginRepository method findPlugins.

private static void findPlugins() {
    final Reflections reflections = new Reflections(DEFAULT_PLUGINS_CLASSPATH);
    final Set<Class<?>> annotatedClasses = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class);
    for (final Class<?> annotatedClass : annotatedClasses) {
        final DataPrepperPlugin dataPrepperPluginAnnotation = annotatedClass.getAnnotation(DataPrepperPlugin.class);
        final String pluginName = dataPrepperPluginAnnotation.name();
        final PluginType pluginType = extractPluginType(dataPrepperPluginAnnotation);
        switch(pluginType) {
            case SOURCE:
                SOURCES.put(pluginName, (Class<Source>) annotatedClass);
                break;
            case BUFFER:
                BUFFERS.put(pluginName, (Class<Buffer>) annotatedClass);
                break;
            case PREPPER:
                PROCESSORS.put(pluginName, (Class<Processor>) annotatedClass);
                break;
            case SINK:
                SINKS.put(pluginName, (Class<Sink>) annotatedClass);
                break;
        }
    }
}