Examples with PluginSetting com.amazon.dataprepper.model.configuration.PluginSetting used on opensource projects

Example 11 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testInstantiateSinkCustomIndex_NoRollOver.

@Test
public void testInstantiateSinkCustomIndex_NoRollOver() throws IOException {
    final String testIndexAlias = "test-alias";
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
    final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile);
    OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    final Request request = new Request(HttpMethod.HEAD, testIndexAlias);
    final Response response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    sink.shutdown();
    // Check restart for index already exists
    sink = new OpenSearchSink(pluginSetting);
    sink.shutdown();
}

Example 12 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputRawSpanDefault.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
public void testOutputRawSpanDefault(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    final String testDoc1 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1);
    final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_2);
    final ObjectMapper mapper = new ObjectMapper();
    @SuppressWarnings("unchecked") final Map<String, Object> expData1 = mapper.readValue(testDoc1, Map.class);
    @SuppressWarnings("unchecked") final Map<String, Object> expData2 = mapper.readValue(testDoc2, Map.class);
    final List<Record<Object>> testRecords = Arrays.asList(stringToRecord.apply(testDoc1), stringToRecord.apply(testDoc2));
    final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(expIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(2));
    MatcherAssert.assertThat(retSources, hasItems(expData1, expData2));
    MatcherAssert.assertThat(getDocumentCount(expIndexAlias, "_id", (String) expData1.get("spanId")), equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // Verify metrics
    final List<Measurement> bulkRequestErrors = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_ERRORS).toString());
    MatcherAssert.assertThat(bulkRequestErrors.size(), equalTo(1));
    Assert.assertEquals(0.0, bulkRequestErrors.get(0).getValue(), 0);
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    MatcherAssert.assertThat(bulkRequestLatencies.size(), equalTo(3));
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
    // TOTAL_TIME
    Assert.assertTrue(bulkRequestLatencies.get(1).getValue() > 0.0);
    // MAX
    Assert.assertTrue(bulkRequestLatencies.get(2).getValue() > 0.0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    MatcherAssert.assertThat(documentsSuccessMeasurements.size(), equalTo(1));
    MatcherAssert.assertThat(documentsSuccessMeasurements.get(0).getValue(), closeTo(2.0, 0));
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    MatcherAssert.assertThat(documentsSuccessFirstAttemptMeasurements.size(), equalTo(1));
    MatcherAssert.assertThat(documentsSuccessFirstAttemptMeasurements.get(0).getValue(), closeTo(2.0, 0));
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    MatcherAssert.assertThat(documentErrorsMeasurements.size(), equalTo(1));
    MatcherAssert.assertThat(documentErrorsMeasurements.get(0).getValue(), closeTo(0.0, 0));
    /**
     * Metrics: Bulk Request Size in Bytes
     */
    final List<Measurement> bulkRequestSizeBytesMetrics = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_SIZE_BYTES).toString());
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.size(), equalTo(3));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(0).getValue(), closeTo(1.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(1).getValue(), closeTo(2058.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(2058.0, 0));
}

Example 13 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testInstantiateSinkServiceMapDefault.

@Test
public void testInstantiateSinkServiceMapDefault() throws IOException {
    final PluginSetting pluginSetting = generatePluginSetting(false, true, null, null);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_SERVICE_MAP);
    final Request request = new Request(HttpMethod.HEAD, indexAlias);
    final Response response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    final Map<String, Object> mappings = getIndexMappings(indexAlias);
    MatcherAssert.assertThat(mappings, notNullValue());
    MatcherAssert.assertThat((boolean) mappings.get("date_detection"), equalTo(false));
    sink.shutdown();
    if (isOSBundle()) {
        // Check managed index
        MatcherAssert.assertThat(getIndexPolicyId(indexAlias), nullValue());
    }
}

Example 14 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class HTTPSourceTest method getBuffer.

private BlockingBuffer<Record<Log>> getBuffer() {
    final HashMap<String, Object> integerHashMap = new HashMap<>();
    integerHashMap.put("buffer_size", 1);
    integerHashMap.put("batch_size", 1);
    final PluginSetting pluginSetting = new PluginSetting("blocking_buffer", integerHashMap) {

        {
            setPipelineName(TEST_PIPELINE_NAME);
        }
    };
    return new BlockingBuffer<>(pluginSetting);
}

Example 15 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class GrokPrepperConfigTests method testDefault.

@Test
public void testDefault() {
    final GrokPrepperConfig grokPrepperConfig = GrokPrepperConfig.buildConfig(new PluginSetting(PLUGIN_NAME, null));
    assertThat(grokPrepperConfig.isBreakOnMatch(), equalTo(DEFAULT_BREAK_ON_MATCH));
    assertThat(grokPrepperConfig.isKeepEmptyCaptures(), equalTo(DEFAULT_KEEP_EMPTY_CAPTURES));
    assertThat(grokPrepperConfig.getMatch(), equalTo(Collections.emptyMap()));
    assertThat(grokPrepperConfig.getkeysToOverwrite(), equalTo(Collections.emptyList()));
    assertThat(grokPrepperConfig.getPatternDefinitions(), equalTo(Collections.emptyMap()));
    assertThat(grokPrepperConfig.getPatternsDirectories(), equalTo(Collections.emptyList()));
    assertThat(grokPrepperConfig.getPatternsFilesGlob(), equalTo(DEFAULT_PATTERNS_FILES_GLOB));
    assertThat(grokPrepperConfig.getTargetKey(), equalTo(DEFAULT_TARGET_KEY));
    assertThat(grokPrepperConfig.isNamedCapturesOnly(), equalTo(DEFAULT_NAMED_CAPTURES_ONLY));
    assertThat(grokPrepperConfig.getTimeoutMillis(), equalTo(DEFAULT_TIMEOUT_MILLIS));
}