Examples with PluginSetting com.amazon.dataprepper.model.configuration.PluginSetting used on opensource projects

Example 91 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method generatePluginSettingByMetadata.

private PluginSetting generatePluginSettingByMetadata(final Map<String, Object> configurationMetadata) {
    final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, configurationMetadata);
    pluginSetting.setPipelineName(PIPELINE_NAME);
    return pluginSetting;
}

Example 92 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputServiceMapDefault.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
public void testOutputServiceMapDefault(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    final String testDoc = readDocFromFile(DEFAULT_SERVICE_MAP_FILE);
    final ObjectMapper mapper = new ObjectMapper();
    @SuppressWarnings("unchecked") final Map<String, Object> expData = mapper.readValue(testDoc, Map.class);
    final List<Record<Object>> testRecords = Collections.singletonList(stringToRecord.apply(testDoc));
    final PluginSetting pluginSetting = generatePluginSetting(false, true, null, null);
    OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(expIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(retSources.get(0), equalTo(expData));
    MatcherAssert.assertThat(getDocumentCount(expIndexAlias, "_id", (String) expData.get("hashId")), equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // verify metrics
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    MatcherAssert.assertThat(bulkRequestLatencies.size(), equalTo(3));
    // COUNT
    MatcherAssert.assertThat(bulkRequestLatencies.get(0).getValue(), closeTo(1.0, 0));
    /**
     * Metrics: Bulk Request Size in Bytes
     */
    final List<Measurement> bulkRequestSizeBytesMetrics = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_SIZE_BYTES).toString());
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.size(), equalTo(3));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(0).getValue(), closeTo(1.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(1).getValue(), closeTo(265.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(265.0, 0));
    // Check restart for index already exists
    sink = new OpenSearchSink(pluginSetting);
    sink.shutdown();
}

Example 93 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testEventOutput.

@Test
public void testEventOutput() throws IOException, InterruptedException {
    final Event testEvent = JacksonEvent.builder().withData("{\"log\": \"foobar\"}").withEventType("event").build();
    final List<Record<Object>> testRecords = Collections.singletonList(new Record<>(testEvent));
    final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(expIndexAlias);
    final Map<String, Object> expectedContent = new HashMap<>();
    expectedContent.put("log", "foobar");
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(retSources.containsAll(Arrays.asList(expectedContent)), equalTo(true));
    MatcherAssert.assertThat(getDocumentCount(expIndexAlias, "log", "foobar"), equalTo(Integer.valueOf(1)));
    sink.shutdown();
}

Example 94 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputCustomIndex.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
public void testOutputCustomIndex(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    final String testIndexAlias = "test-alias";
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
    final String testIdField = "someId";
    final String testId = "foo";
    final List<Record<Object>> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId)));
    final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile);
    pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(testIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(getDocumentCount(testIndexAlias, "_id", testId), equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // verify metrics
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    MatcherAssert.assertThat(bulkRequestLatencies.size(), equalTo(3));
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
}

Example 95 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class GrokPrepperConfigTests method testValidConfig.

@Test
public void testValidConfig() {
    final PluginSetting validPluginSetting = completePluginSettingForGrokPrepper(false, true, TEST_MATCH, false, TEST_KEYS_TO_OVERWRITE, TEST_PATTERNS_DIRECTORIES, TEST_PATTERNS_FILES_GLOB, TEST_PATTERN_DEFINITIONS, TEST_TIMEOUT_MILLIS, TEST_TARGET_KEY);
    final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(validPluginSetting);
    assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(false));
    assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(true));
    assertThat(grokPrepperConfig.getMatch(), equalTo(TEST_MATCH));
    assertThat(grokPrepperConfig.getkeysToOverwrite(), equalTo(TEST_KEYS_TO_OVERWRITE));
    assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(TEST_PATTERN_DEFINITIONS));
    assertThat(grokPrepperConfig.getPatternsDirectories(), equalTo(TEST_PATTERNS_DIRECTORIES));
    assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(TEST_PATTERNS_FILES_GLOB));
    assertThat(grokPrepperConfig.getTargetKey(), equalTo(TEST_TARGET_KEY));
    assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(false));
    assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(TEST_TIMEOUT_MILLIS));
}