Examples with PluginSetting com.amazon.dataprepper.model.configuration.PluginSetting used on opensource projects

Example 6 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testInstantiateSinkDoesNotOverwriteNewerIndexTemplates.

@Test
public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates() throws IOException {
    final String testIndexAlias = "test-alias";
    final String expectedIndexTemplateName = testIndexAlias + "-index-template";
    final String testTemplateFileV1 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE).getFile();
    final String testTemplateFileV2 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V2_FILE).getFile();
    // Create sink with template version 1
    PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1);
    OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    Request getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName);
    Response getTemplateResponse = client.performRequest(getTemplateRequest);
    MatcherAssert.assertThat(getTemplateResponse.getStatusLine().getStatusCode(), equalTo(SC_OK));
    String responseBody = EntityUtils.toString(getTemplateResponse.getEntity());
    @SuppressWarnings("unchecked") final Integer firstResponseVersion = (Integer) ((Map<String, Object>) createContentParser(XContentType.JSON.xContent(), responseBody).map().get(expectedIndexTemplateName)).get("version");
    MatcherAssert.assertThat(firstResponseVersion, equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // Create sink with template version 2
    pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV2);
    sink = new OpenSearchSink(pluginSetting);
    getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName);
    getTemplateResponse = client.performRequest(getTemplateRequest);
    MatcherAssert.assertThat(getTemplateResponse.getStatusLine().getStatusCode(), equalTo(SC_OK));
    responseBody = EntityUtils.toString(getTemplateResponse.getEntity());
    @SuppressWarnings("unchecked") final Integer secondResponseVersion = (Integer) ((Map<String, Object>) createContentParser(XContentType.JSON.xContent(), responseBody).map().get(expectedIndexTemplateName)).get("version");
    MatcherAssert.assertThat(secondResponseVersion, equalTo(Integer.valueOf(2)));
    sink.shutdown();
    // Create sink with template version 1 again
    pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1);
    sink = new OpenSearchSink(pluginSetting);
    getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName);
    getTemplateResponse = client.performRequest(getTemplateRequest);
    MatcherAssert.assertThat(getTemplateResponse.getStatusLine().getStatusCode(), equalTo(SC_OK));
    responseBody = EntityUtils.toString(getTemplateResponse.getEntity());
    @SuppressWarnings("unchecked") final Integer thirdResponseVersion = (Integer) ((Map<String, Object>) createContentParser(XContentType.JSON.xContent(), responseBody).map().get(expectedIndexTemplateName)).get("version");
    // Assert version 2 was not overwritten by version 1
    MatcherAssert.assertThat(thirdResponseVersion, equalTo(Integer.valueOf(2)));
    sink.shutdown();
}

Example 7 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputRawSpanWithDLQ.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
public void testOutputRawSpanWithDLQ(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    // TODO: write test case
    final String testDoc1 = readDocFromFile("raw-span-error.json");
    final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1);
    final ObjectMapper mapper = new ObjectMapper();
    @SuppressWarnings("unchecked") final Map<String, Object> expData = mapper.readValue(testDoc2, Map.class);
    final List<Record<Object>> testRecords = Arrays.asList(stringToRecord.apply(testDoc1), stringToRecord.apply(testDoc2));
    final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null);
    // generate temporary directory for dlq file
    final File tempDirectory = Files.createTempDirectory("").toFile();
    // add dlq file path into setting
    final String expDLQFile = tempDirectory.getAbsolutePath() + "/test-dlq.txt";
    pluginSetting.getSettings().put(RetryConfiguration.DLQ_FILE, expDLQFile);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    sink.shutdown();
    final StringBuilder dlqContent = new StringBuilder();
    Files.lines(Paths.get(expDLQFile)).forEach(dlqContent::append);
    final String nonPrettyJsonString = mapper.writeValueAsString(mapper.readValue(testDoc1, JsonNode.class));
    MatcherAssert.assertThat(dlqContent.toString(), containsString(nonPrettyJsonString));
    final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(expIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(retSources.get(0), equalTo(expData));
    // clean up temporary directory
    FileUtils.deleteQuietly(tempDirectory);
    // verify metrics
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    MatcherAssert.assertThat(documentsSuccessMeasurements.size(), equalTo(1));
    MatcherAssert.assertThat(documentsSuccessMeasurements.get(0).getValue(), closeTo(1.0, 0));
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    MatcherAssert.assertThat(documentErrorsMeasurements.size(), equalTo(1));
    MatcherAssert.assertThat(documentErrorsMeasurements.get(0).getValue(), closeTo(1.0, 0));
    /**
     * Metrics: Bulk Request Size in Bytes
     */
    final List<Measurement> bulkRequestSizeBytesMetrics = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_SIZE_BYTES).toString());
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.size(), equalTo(3));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(0).getValue(), closeTo(1.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(1).getValue(), closeTo(2072.0, 0));
    MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(2072.0, 0));
}

Example 8 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testInstantiateSinkCustomIndex_WithIsmPolicy.

@Test
public void testInstantiateSinkCustomIndex_WithIsmPolicy() throws IOException {
    final String indexAlias = "sink-custom-index-ism-test-alias";
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
    final Map<String, Object> metadata = initializeConfigurationMetadata(false, false, indexAlias, testTemplateFile);
    metadata.put(IndexConfiguration.ISM_POLICY_FILE, TEST_CUSTOM_INDEX_POLICY_FILE);
    final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata);
    OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    Request request = new Request(HttpMethod.HEAD, indexAlias);
    Response response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    final String index = String.format("%s-000001", indexAlias);
    final Map<String, Object> mappings = getIndexMappings(index);
    MatcherAssert.assertThat(mappings, notNullValue());
    MatcherAssert.assertThat((boolean) mappings.get("date_detection"), equalTo(false));
    sink.shutdown();
    final String expectedIndexPolicyName = indexAlias + "-policy";
    if (isOSBundle()) {
        // Check managed index
        await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {
            MatcherAssert.assertThat(getIndexPolicyId(index), equalTo(expectedIndexPolicyName));
        });
    }
    // roll over initial index
    request = new Request(HttpMethod.POST, String.format("%s/_rollover", indexAlias));
    request.setJsonEntity("{ \"conditions\" : { } }\n");
    response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    // Instantiate sink again
    sink = new OpenSearchSink(pluginSetting);
    // Make sure no new write index *-000001 is created under alias
    final String rolloverIndexName = String.format("%s-000002", indexAlias);
    request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias");
    response = client.performRequest(request);
    MatcherAssert.assertThat(checkIsWriteIndex(EntityUtils.toString(response.getEntity()), indexAlias, rolloverIndexName), equalTo(true));
    sink.shutdown();
    if (isOSBundle()) {
        // Check managed index
        MatcherAssert.assertThat(getIndexPolicyId(rolloverIndexName), equalTo(expectedIndexPolicyName));
    }
}

Example 9 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testInstantiateSinkRawSpanDefault.

@Test
public void testInstantiateSinkRawSpanDefault() throws IOException {
    final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null);
    OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    final String indexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW);
    Request request = new Request(HttpMethod.HEAD, indexAlias);
    Response response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    final String index = String.format("%s-000001", indexAlias);
    final Map<String, Object> mappings = getIndexMappings(index);
    MatcherAssert.assertThat(mappings, notNullValue());
    MatcherAssert.assertThat((boolean) mappings.get("date_detection"), equalTo(false));
    sink.shutdown();
    if (isOSBundle()) {
        // Check managed index
        await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {
            MatcherAssert.assertThat(getIndexPolicyId(index), equalTo(IndexConstants.RAW_ISM_POLICY));
        });
    }
    // roll over initial index
    request = new Request(HttpMethod.POST, String.format("%s/_rollover", indexAlias));
    request.setJsonEntity("{ \"conditions\" : { } }\n");
    response = client.performRequest(request);
    MatcherAssert.assertThat(response.getStatusLine().getStatusCode(), equalTo(SC_OK));
    // Instantiate sink again
    sink = new OpenSearchSink(pluginSetting);
    // Make sure no new write index *-000001 is created under alias
    final String rolloverIndexName = String.format("%s-000002", indexAlias);
    request = new Request(HttpMethod.GET, rolloverIndexName + "/_alias");
    response = client.performRequest(request);
    MatcherAssert.assertThat(checkIsWriteIndex(EntityUtils.toString(response.getEntity()), indexAlias, rolloverIndexName), equalTo(true));
    sink.shutdown();
    if (isOSBundle()) {
        // Check managed index
        MatcherAssert.assertThat(getIndexPolicyId(rolloverIndexName), equalTo(IndexConstants.RAW_ISM_POLICY));
    }
}

Example 10 with PluginSetting

use of com.amazon.dataprepper.model.configuration.PluginSetting in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputManagementDisabled.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
@Timeout(value = 1, unit = TimeUnit.MINUTES)
public void testOutputManagementDisabled(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    final String testIndexAlias = "test-" + UUID.randomUUID();
    final String roleName = UUID.randomUUID().toString();
    final String username = UUID.randomUUID().toString();
    final String password = UUID.randomUUID().toString();
    final OpenSearchSecurityAccessor securityAccessor = new OpenSearchSecurityAccessor(client);
    securityAccessor.createBulkWritingRole(roleName, testIndexAlias + "*");
    securityAccessor.createUser(username, password, roleName);
    final String testIdField = "someId";
    final String testId = "foo";
    final List<Record<Object>> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId)));
    final Map<String, Object> metadata = initializeConfigurationMetadata(false, false, testIndexAlias, null);
    metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue());
    metadata.put(ConnectionConfiguration.USERNAME, username);
    metadata.put(ConnectionConfiguration.PASSWORD, password);
    metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField);
    final PluginSetting pluginSetting = generatePluginSettingByMetadata(metadata);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource("management-disabled-index-template.json")).getFile();
    createIndexTemplate(testIndexAlias, testIndexAlias + "*", testTemplateFile);
    createIndex(testIndexAlias);
    sink.output(testRecords);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(testIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(getDocumentCount(testIndexAlias, "_id", testId), equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // verify metrics
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    MatcherAssert.assertThat(bulkRequestLatencies.size(), equalTo(3));
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
}