Examples with SerializedJson com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson used on opensource projects

Example 1 with SerializedJson

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteSuccessOnFirstAttempt.

@Test
public void testExecuteSuccessOnFirstAttempt() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.successOnFirstAttempt = true;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    // verify metrics
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    assertEquals(1, documentsSuccessFirstAttemptMeasurements.size());
    assertEquals(4.0, documentsSuccessFirstAttemptMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(4.0, documentsSuccessMeasurements.get(0).getValue(), 0);
}

Example 2 with SerializedJson

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson in project data-prepper by opensearch-project.

the class OpenSearchSink method doOutput.

@Override
public void doOutput(final Collection<Record<Object>> records) {
    if (records.isEmpty()) {
        return;
    }
    AccumulatingBulkRequest<BulkOperation, BulkRequest> bulkRequest = bulkRequestSupplier.get();
    for (final Record<Object> record : records) {
        final SerializedJson document = getDocument(record.getData());
        final IndexOperation.Builder<Object> indexOperationBuilder = new IndexOperation.Builder<>().index(indexManager.getIndexAlias()).document(document);
        final Map documentAsMap;
        try {
            documentAsMap = objectMapper.readValue(document.getSerializedJson(), Map.class);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        if (documentAsMap != null) {
            final String docId = (String) documentAsMap.get(documentIdField);
            if (docId != null) {
                indexOperationBuilder.id(docId);
            }
        }
        final BulkOperation indexBulkOperation = new BulkOperation.Builder().index(indexOperationBuilder.build()).build();
        final long estimatedBytesBeforeAdd = bulkRequest.estimateSizeInBytesWithDocument(indexBulkOperation);
        if (bulkSize >= 0 && estimatedBytesBeforeAdd >= bulkSize && bulkRequest.getOperationsCount() > 0) {
            flushBatch(bulkRequest);
            bulkRequest = bulkRequestSupplier.get();
        }
        bulkRequest.addOperation(indexBulkOperation);
    }
    // Flush the remaining requests
    if (bulkRequest.getOperationsCount() > 0) {
        flushBatch(bulkRequest);
    }
}

Example 3 with SerializedJson

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteNonRetryableResponse.

@Test
public void testExecuteNonRetryableResponse() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.retryable = false;
    client.nonRetryableException = false;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerExceptionArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer, times(3)).accept(loggerWriteRequestArgCaptor.capture(), isA(RuntimeException.class));
    final List<BulkOperation> allLoggerWriteRequests = loggerWriteRequestArgCaptor.getAllValues();
    for (int i = 0; i < allLoggerWriteRequests.size(); i++) {
        final BulkOperation actualFailedWrite = allLoggerWriteRequests.get(i);
        MatcherAssert.assertThat(actualFailedWrite.index().index(), equalTo(testIndex));
        String expectedIndexName = Integer.toString(i + 2);
        MatcherAssert.assertThat(actualFailedWrite.index().id(), equalTo(expectedIndexName));
    }
    // verify metrics
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(1.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(3.0, documentErrorsMeasurements.get(0).getValue(), 0);
}

Example 4 with SerializedJson

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteRetryable.

@Test
public void testExecuteRetryable() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(3, client.attempt);
    assertEquals(2, client.finalResponse.items().size());
    assertFalse(client.finalResponse.errors());
    assertEquals("3", client.finalRequest.operations().get(0).index().id());
    assertEquals("4", client.finalRequest.operations().get(1).index().id());
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerThrowableArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer).accept(loggerWriteRequestArgCaptor.capture(), loggerThrowableArgCaptor.capture());
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue(), notNullValue());
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue().index().index(), equalTo(testIndex));
    MatcherAssert.assertThat(loggerWriteRequestArgCaptor.getValue().index().id(), equalTo("2"));
    MatcherAssert.assertThat(loggerThrowableArgCaptor.getValue(), notNullValue());
    // verify metrics
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    assertEquals(1, documentsSuccessFirstAttemptMeasurements.size());
    assertEquals(1.0, documentsSuccessFirstAttemptMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(3.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(1.0, documentErrorsMeasurements.get(0).getValue(), 0);
}

Example 5 with SerializedJson

use of com.amazon.dataprepper.plugins.sink.opensearch.bulk.SerializedJson in project data-prepper by opensearch-project.

the class BulkRetryStrategyTests method testExecuteNonRetryableException.

@Test
public void testExecuteNonRetryableException() throws Exception {
    final String testIndex = "bar";
    final FakeClient client = new FakeClient(testIndex);
    client.retryable = false;
    final BulkRetryStrategy bulkRetryStrategy = new BulkRetryStrategy(client::bulk, logFailureConsumer, PLUGIN_METRICS, () -> new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder()));
    final IndexOperation<SerializedJson> indexOperation1 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("1").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation2 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("2").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation3 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("3").document(arbitraryDocument()).build();
    final IndexOperation<SerializedJson> indexOperation4 = new IndexOperation.Builder<SerializedJson>().index(testIndex).id("4").document(arbitraryDocument()).build();
    final AccumulatingBulkRequest accumulatingBulkRequest = new JavaClientAccumulatingBulkRequest(new BulkRequest.Builder());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation1).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation2).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation3).build());
    accumulatingBulkRequest.addOperation(new BulkOperation.Builder().index(indexOperation4).build());
    bulkRetryStrategy.execute(accumulatingBulkRequest);
    assertEquals(1, client.attempt);
    ArgumentCaptor<BulkOperation> loggerWriteRequestArgCaptor = ArgumentCaptor.forClass(BulkOperation.class);
    ArgumentCaptor<Throwable> loggerExceptionArgCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(logFailureConsumer, times(4)).accept(loggerWriteRequestArgCaptor.capture(), isA(IllegalArgumentException.class));
    final List<BulkOperation> allLoggerWriteRequests = loggerWriteRequestArgCaptor.getAllValues();
    for (int i = 0; i < allLoggerWriteRequests.size(); i++) {
        final BulkOperation actualFailedWrite = allLoggerWriteRequests.get(i);
        MatcherAssert.assertThat(actualFailedWrite.index().index(), equalTo(testIndex));
        String expectedIndexName = Integer.toString(i + 1);
        MatcherAssert.assertThat(actualFailedWrite.index().id(), equalTo(expectedIndexName));
    }
    // verify metrics
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(0.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(4.0, documentErrorsMeasurements.get(0).getValue(), 0);
}