use of com.amazon.dataprepper.model.sink.Sink in project data-prepper by opensearch-project.
the class SinkFactoryTests method testNewSinkClassByNameThatExists.
/**
* Tests if SinkFactory is able to retrieve default Source plugins by name
*/
@Test
public void testNewSinkClassByNameThatExists() {
final PluginSetting stdOutSinkConfiguration = new PluginSetting("stdout", new HashMap<>());
final Sink actualSink = SinkFactory.newSink(stdOutSinkConfiguration);
final Sink expectedSink = new StdOutSink();
assertThat(actualSink, notNullValue());
assertThat(actualSink.getClass().getSimpleName(), is(equalTo(expectedSink.getClass().getSimpleName())));
}
use of com.amazon.dataprepper.model.sink.Sink in project data-prepper by opensearch-project.
the class OpenSearchSink method initialize.
public void initialize() throws IOException {
LOG.info("Initializing OpenSearch sink");
restHighLevelClient = openSearchSinkConfig.getConnectionConfiguration().createClient();
indexManager = indexManagerFactory.getIndexManager(indexType, restHighLevelClient, openSearchSinkConfig);
final String dlqFile = openSearchSinkConfig.getRetryConfiguration().getDlqFile();
if (dlqFile != null) {
dlqWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
indexManager.setupIndex();
OpenSearchTransport transport = new RestClientTransport(restHighLevelClient.getLowLevelClient(), new PreSerializedJsonpMapper());
openSearchClient = new OpenSearchClient(transport);
bulkRequestSupplier = () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder().index(indexManager.getIndexAlias()));
bulkRetryStrategy = new BulkRetryStrategy(bulkRequest -> openSearchClient.bulk(bulkRequest.getRequest()), this::logFailure, pluginMetrics, bulkRequestSupplier);
LOG.info("Initialized OpenSearch sink");
objectMapper = new ObjectMapper();
}
use of com.amazon.dataprepper.model.sink.Sink in project data-prepper by opensearch-project.
the class PipelineParser method buildPipelineFromConfiguration.
private void buildPipelineFromConfiguration(final String pipelineName, final Map<String, PipelineConfiguration> pipelineConfigurationMap, final Map<String, Pipeline> pipelineMap) {
final PipelineConfiguration pipelineConfiguration = pipelineConfigurationMap.get(pipelineName);
LOG.info("Building pipeline [{}] from provided configuration", pipelineName);
try {
final PluginSetting sourceSetting = pipelineConfiguration.getSourcePluginSetting();
final Optional<Source> pipelineSource = getSourceIfPipelineType(pipelineName, sourceSetting, pipelineMap, pipelineConfigurationMap);
final Source source = pipelineSource.orElseGet(() -> pluginFactory.loadPlugin(Source.class, sourceSetting));
LOG.info("Building buffer for the pipeline [{}]", pipelineName);
final Buffer buffer = pluginFactory.loadPlugin(Buffer.class, pipelineConfiguration.getBufferPluginSetting());
LOG.info("Building processors for the pipeline [{}]", pipelineName);
final int processorThreads = pipelineConfiguration.getWorkers();
final List<List<Processor>> processorSets = pipelineConfiguration.getProcessorPluginSettings().stream().map(this::newProcessor).collect(Collectors.toList());
final int readBatchDelay = pipelineConfiguration.getReadBatchDelay();
LOG.info("Building sinks for the pipeline [{}]", pipelineName);
final List<Sink> sinks = pipelineConfiguration.getSinkPluginSettings().stream().map(this::buildSinkOrConnector).collect(Collectors.toList());
final Pipeline pipeline = new Pipeline(pipelineName, source, buffer, processorSets, sinks, processorThreads, readBatchDelay);
pipelineMap.put(pipelineName, pipeline);
} catch (Exception ex) {
// If pipeline construction errors out, we will skip that pipeline and proceed
LOG.error("Construction of pipeline components failed, skipping building of pipeline [{}] and its connected " + "pipelines", pipelineName, ex);
processRemoveIfRequired(pipelineName, pipelineConfigurationMap, pipelineMap);
}
}
use of com.amazon.dataprepper.model.sink.Sink in project data-prepper by opensearch-project.
the class PluginRepository method findPlugins.
private static void findPlugins() {
final Reflections reflections = new Reflections(DEFAULT_PLUGINS_CLASSPATH);
final Set<Class<?>> annotatedClasses = reflections.getTypesAnnotatedWith(DataPrepperPlugin.class);
for (final Class<?> annotatedClass : annotatedClasses) {
final DataPrepperPlugin dataPrepperPluginAnnotation = annotatedClass.getAnnotation(DataPrepperPlugin.class);
final String pluginName = dataPrepperPluginAnnotation.name();
final PluginType pluginType = extractPluginType(dataPrepperPluginAnnotation);
switch(pluginType) {
case SOURCE:
SOURCES.put(pluginName, (Class<Source>) annotatedClass);
break;
case BUFFER:
BUFFERS.put(pluginName, (Class<Buffer>) annotatedClass);
break;
case PREPPER:
PROCESSORS.put(pluginName, (Class<Processor>) annotatedClass);
break;
case SINK:
SINKS.put(pluginName, (Class<Sink>) annotatedClass);
break;
}
}
}
Aggregations