Examples with CheckpointState com.amazon.dataprepper.model.CheckpointState used on opensource projects

Example 1 with CheckpointState

use of com.amazon.dataprepper.model.CheckpointState in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse200.

@Test
public void testHTTPJsonResponse200() {
    // Prepare
    final String testData = "[{\"log\": \"somelog\"}]";
    final int testPayloadSize = testData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Then
    Assertions.assertFalse(testBuffer.isEmpty());
    final Map.Entry<Collection<Record<Log>>, CheckpointState> result = testBuffer.read(100);
    List<Record<Log>> records = new ArrayList<>(result.getKey());
    Assertions.assertEquals(1, records.size());
    final Record<Log> record = records.get(0);
    Assertions.assertEquals("somelog", record.getData().get("log", String.class));
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement requestProcessDurationCount = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestProcessDurationCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    Assertions.assertTrue(requestProcessDurationMax.getValue() > 0);
    final Measurement payloadSizeMax = MetricsTestUtil.getMeasurementFromList(payloadSizeSummaryMeasurements, Statistic.MAX);
    Assertions.assertEquals(testPayloadSize, payloadSizeMax.getValue());
}

Example 2 with CheckpointState

use of com.amazon.dataprepper.model.CheckpointState in project data-prepper by opensearch-project.

the class AbstractBufferTest method testCheckpointMetrics.

@Test
public void testCheckpointMetrics() throws Exception {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        testRecords.add(new Record<>(UUID.randomUUID().toString()));
    }
    abstractBuffer.writeAll(testRecords, 1000);
    final Map.Entry<Collection<Record<String>>, CheckpointState> readResult = abstractBuffer.read(1000);
    final List<Measurement> checkpointTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.CHECKPOINT_TIME_ELAPSED).toString());
    Assert.assertEquals(0.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertEquals(0.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0);
    // When
    abstractBuffer.checkpoint(readResult.getValue());
    // Then
    Assert.assertEquals(0, abstractBuffer.getRecordsInFlight());
    final List<Measurement> recordsInFlightMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_INFLIGHT).toString());
    final List<Measurement> recordsProcessedMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(MetricNames.RECORDS_PROCESSED).toString());
    final Measurement recordsInFlightMeasurement = recordsInFlightMeasurements.get(0);
    final Measurement recordsProcessedMeasurement = recordsProcessedMeasurements.get(0);
    Assert.assertEquals(0.0, recordsInFlightMeasurement.getValue(), 0);
    Assert.assertEquals(5.0, recordsProcessedMeasurement.getValue(), 0);
    Assert.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.0, 0.001));
}

Example 3 with CheckpointState

use of com.amazon.dataprepper.model.CheckpointState in project data-prepper by opensearch-project.

the class ProcessWorker method run.

@Override
public void run() {
    try {
        do {
            final Map.Entry<Collection, CheckpointState> readResult = readBuffer.read(pipeline.getReadBatchTimeoutInMillis());
            Collection records = readResult.getKey();
            final CheckpointState checkpointState = readResult.getValue();
            // TODO Hacky way to avoid logging continuously - Will be removed as part of metrics implementation
            if (records.isEmpty()) {
                if (!isEmptyRecordsLogged) {
                    LOG.info(" {} Worker: No records received from buffer", pipeline.getName());
                    isEmptyRecordsLogged = true;
                }
            } else {
                LOG.info(" {} Worker: Processing {} records from buffer", pipeline.getName(), records.size());
            }
            // Should Empty list from buffer should be sent to the processors? For now sending as the Stateful processors expects it.
            for (final Processor processor : processors) {
                records = processor.execute(records);
            }
            if (!records.isEmpty()) {
                postToSink(records);
            }
            // Checkpoint the current batch read from the buffer after being processed by processors and sinks.
            readBuffer.checkpoint(checkpointState);
        } while (!shouldStop());
    } catch (final Exception e) {
        LOG.error("Encountered exception during pipeline {} processing", pipeline.getName(), e);
    }
}

Example 4 with CheckpointState

use of com.amazon.dataprepper.model.CheckpointState in project data-prepper by opensearch-project.

the class TestBuffer method read.

@Override
public Map.Entry<Collection<Record<Event>>, CheckpointState> read(int timeoutInMillis) {
    final List<Record<Event>> records = new ArrayList<>();
    int index = 0;
    Record<Event> record;
    while (index < batchSize && (record = buffer.poll()) != null) {
        records.add(record);
        index++;
    }
    final CheckpointState checkpointState = new CheckpointState(records.size());
    return new AbstractMap.SimpleEntry<>(records, checkpointState);
}

Example 5 with CheckpointState

use of com.amazon.dataprepper.model.CheckpointState in project data-prepper by opensearch-project.

the class StdInSourceTests method testStdInSourceSuccessfulWriteToBuffer.

@Test
public void testStdInSourceSuccessfulWriteToBuffer() {
    final Queue<Record<Event>> bufferQueue = new LinkedList<>();
    final TestBuffer buffer = new TestBuffer(bufferQueue, 1);
    final StdInSource stdInSource = new StdInSource(TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME);
    assertThat(buffer.size(), is(equalTo(0)));
    stdInSource.start(buffer);
    assertThat(buffer.size(), is(equalTo(1)));
    final Map.Entry<Collection<Record<Event>>, CheckpointState> readResult = buffer.read(TEST_WRITE_TIMEOUT);
    final Collection<Record<Event>> recordsFromBuffer = readResult.getKey();
    assertThat(recordsFromBuffer.size(), is(equalTo(1)));
    recordsFromBuffer.forEach(actualRecord -> assertThat(actualRecord.getData().get("message", String.class), is(equalTo(READ_CONTENT))));
}