Examples with JavaClientAccumulatingBulkRequest com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest used on opensource projects

Example 1 with JavaClientAccumulatingBulkRequest

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteSuccessOnFirstAttempt.

@Test
public void testExecuteSuccessOnFirstAttempt() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.successOnFirstAttempt = true;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    // verify metrics
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    assertEquals(1, documentsSuccessFirstAttemptMeasurements.size());
    assertEquals(4.0, documentsSuccessFirstAttemptMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(4.0, documentsSuccessMeasurements.get(0).getValue(), 0);
}

Example 2 with JavaClientAccumulatingBulkRequest

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest in project data-prepper by opensearch-project.

the class OpenSearchSink method initialize.

public void initialize() throws IOException {
    LOG.info("Initializing OpenSearch sink");
    restHighLevelClient = openSearchSinkConfig.getConnectionConfiguration().createClient();
    indexManager = indexManagerFactory.getIndexManager(indexType, restHighLevelClient, openSearchSinkConfig);
    final String dlqFile = openSearchSinkConfig.getRetryConfiguration().getDlqFile();
    if (dlqFile != null) {
        dlqWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
    }
    indexManager.setupIndex();
    OpenSearchTransport transport = new RestClientTransport(restHighLevelClient.getLowLevelClient(), new PreSerializedJsonpMapper());
    openSearchClient = new OpenSearchClient(transport);
    bulkRequestSupplier = () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder().index(indexManager.getIndexAlias()));
    bulkRetryStrategy = new BulkRetryStrategy(bulkRequest -> openSearchClient.bulk(bulkRequest.getRequest()), this::logFailure, pluginMetrics, bulkRequestSupplier);
    LOG.info("Initialized OpenSearch sink");
    objectMapper = new ObjectMapper();
}

Example 3 with JavaClientAccumulatingBulkRequest

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteNonRetryableResponse.

@Test
public void testExecuteNonRetryableResponse() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.retryable = false;
    client.nonRetryableException = false;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerExceptionArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer, times(3)).accept(loggerWriteRequestArgCaptor.capture(), isA(RuntimeException.class));
    final List<BulkOperation> allLoggerWriteRequests = loggerWriteRequestArgCaptor.getAllValues();
    for (int i = 0; i < allLoggerWriteRequests.size(); i++) {
        final BulkOperation actualFailedWrite = allLoggerWriteRequests.get(i);
        MatcherAssert.assertThat(actualFailedWrite.index().index(), equalTo(testIndex));
        String expectedIndexName = Integer.toString(i + 2);
        MatcherAssert.assertThat(actualFailedWrite.index().id(), equalTo(expectedIndexName));
    }
    // verify metrics
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(1.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(3.0, documentErrorsMeasurements.get(0).getValue(), 0);
}

Example 4 with JavaClientAccumulatingBulkRequest

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteRetryable.

@Test
public void testExecuteRetryable() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(3, client.attempt);
    assertEquals(2, client.finalResponse.items().size());
    assertFalse(client.finalResponse.errors());
    assertEquals("3", client.finalRequest.operations().get(0).index().id());
    assertEquals("4", client.finalRequest.operations().get(1).index().id());
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerThrowableArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer).accept(loggerWriteRequestArgCaptor.capture(), loggerThrowableArgCaptor.capture());
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue(), notNullValue());
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue().index().index(), equalTo(testIndex));
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue().index().id(), equalTo("2"));
    MatcherAssert.assertThat(loggerThrowableArgCaptor.getValue(), notNullValue());
    // verify metrics
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    assertEquals(1, documentsSuccessFirstAttemptMeasurements.size());
    assertEquals(1.0, documentsSuccessFirstAttemptMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(3.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(1.0, documentErrorsMeasurements.get(0).getValue(), 0);
}

Example 5 with JavaClientAccumulatingBulkRequest

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.JavaClientAccumulatingBulkRequest in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteNonRetryableException.

@Test
public void testExecuteNonRetryableException() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.retryable = false;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerExceptionArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer, times(4)).accept(loggerWriteRequestArgCaptor.capture(), isA(IllegalArgumentException.class));
    final List<BulkOperation> allLoggerWriteRequests = loggerWriteRequestArgCaptor.getAllValues();
    for (int i = 0; i < allLoggerWriteRequests.size(); i++) {
        final BulkOperation actualFailedWrite = allLoggerWriteRequests.get(i);
        MatcherAssert.assertThat(actualFailedWrite.index().index(), equalTo(testIndex));
        String expectedIndexName = Integer.toString(i + 1);
        MatcherAssert.assertThat(actualFailedWrite.index().id(), equalTo(expectedIndexName));
    }
    // verify metrics
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(0.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(4.0, documentErrorsMeasurements.get(0).getValue(), 0);
}