Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 1 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractBufferTest method testReadMetrics.

@Test
public void testReadMetrics() throws Exception {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        testRecords.add(new Record<>(UUID.randomUUID().toString()));
    }
    abstractBuffer.writeAll(testRecords, 1000);
    // When
    abstractBuffer.read(1000);
    // Then
    final List<Measurement> recordsReadMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_READ).toString());
    final List<Measurement> recordsInFlightMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_INFLIGHT).toString());
    final List<Measurement> recordsProcessedMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(MetricNames.RECORDS_PROCESSED).toString());
    final List<Measurement> readTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.READ_TIME_ELAPSED).toString());
    Assert.assertEquals(1, recordsReadMeasurements.size());
    Assert.assertEquals(5.0, recordsReadMeasurements.get(0).getValue(), 0);
    Assert.assertEquals(1, recordsInFlightMeasurements.size());
    Assert.assertEquals(5, abstractBuffer.getRecordsInFlight());
    final Measurement recordsInFlightMeasurement = recordsInFlightMeasurements.get(0);
    Assert.assertEquals(5.0, recordsInFlightMeasurement.getValue(), 0);
    Assert.assertEquals(1, recordsProcessedMeasurements.size());
    final Measurement recordsProcessedMeasurement = recordsProcessedMeasurements.get(0);
    Assert.assertEquals(0.0, recordsProcessedMeasurement.getValue(), 0);
    Assert.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(readTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(readTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.05, 0.25));
}

Example 2 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractBufferTest method testWriteAllTimeoutMetric.

@Test
public void testWriteAllTimeoutMetric() throws TimeoutException {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferTimeoutImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = Arrays.asList(new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString()));
    // When/Then
    Assert.assertThrows(TimeoutException.class, () -> abstractBuffer.writeAll(testRecords, 1000));
    final List<Measurement> timeoutMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.WRITE_TIMEOUTS).toString());
    Assert.assertEquals(1, timeoutMeasurements.size());
    Assert.assertEquals(1.0, timeoutMeasurements.get(0).getValue(), 0);
}

Example 3 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputRawSpanDefault.

public void testOutputRawSpanDefault() throws IOException, InterruptedException {
    final String testDoc1 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1);
    final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_2);
    final ObjectMapper mapper = new ObjectMapper();
    @SuppressWarnings("unchecked") final Map<String, Object> expData1 = mapper.readValue(testDoc1, Map.class);
    @SuppressWarnings("unchecked") final Map<String, Object> expData2 = mapper.readValue(testDoc2, Map.class);
    final List<Record<Object>> testRecords = Arrays.asList(new Record<>(testDoc1), new Record<>(testDoc2));
    final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final String expIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(expIndexAlias);
    assertEquals(2, retSources.size());
    assertTrue(retSources.containsAll(Arrays.asList(expData1, expData2)));
    assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String) expData1.get("spanId")));
    sink.shutdown();
    // Verify metrics
    final List<Measurement> bulkRequestErrors = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_ERRORS).toString());
    assertEquals(1, bulkRequestErrors.size());
    Assert.assertEquals(0.0, bulkRequestErrors.get(0).getValue(), 0);
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    assertEquals(3, bulkRequestLatencies.size());
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
    // TOTAL_TIME
    Assert.assertTrue(bulkRequestLatencies.get(1).getValue() > 0.0);
    // MAX
    Assert.assertTrue(bulkRequestLatencies.get(2).getValue() > 0.0);
    final List<Measurement> documentsSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS).toString());
    assertEquals(1, documentsSuccessMeasurements.size());
    assertEquals(2.0, documentsSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentsSuccessFirstAttemptMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENTS_SUCCESS_FIRST_ATTEMPT).toString());
    assertEquals(1, documentsSuccessFirstAttemptMeasurements.size());
    assertEquals(2.0, documentsSuccessFirstAttemptMeasurements.get(0).getValue(), 0);
    final List<Measurement> documentErrorsMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(BulkRetryStrategy.DOCUMENT_ERRORS).toString());
    assertEquals(1, documentErrorsMeasurements.size());
    assertEquals(0.0, documentErrorsMeasurements.get(0).getValue(), 0);
    /**
     * Metrics: Bulk Request Size in Bytes
     */
    final List<Measurement> bulkRequestSizeBytesMetrics = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_SIZE_BYTES).toString());
    assertEquals(3, bulkRequestSizeBytesMetrics.size());
    assertEquals(1.0, bulkRequestSizeBytesMetrics.get(0).getValue(), 0);
    assertEquals(2188.0, bulkRequestSizeBytesMetrics.get(1).getValue(), 0);
    assertEquals(2188.0, bulkRequestSizeBytesMetrics.get(2).getValue(), 0);
}

Example 4 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputCustomIndex.

public void testOutputCustomIndex() throws IOException, InterruptedException {
    final String testIndexAlias = "test-alias";
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
    final String testIdField = "someId";
    final String testId = "foo";
    final List<Record<Object>> testRecords = Collections.singletonList(generateCustomRecord(testIdField, testId));
    final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile);
    pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(testIndexAlias);
    assertEquals(1, retSources.size());
    assertEquals(Integer.valueOf(1), getDocumentCount(testIndexAlias, "_id", testId));
    sink.shutdown();
    // verify metrics
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    assertEquals(3, bulkRequestLatencies.size());
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
}

Example 5 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse200.

@Test
public void testHTTPJsonResponse200() {
    // Prepare
    final String testData = "[{\"log\": \"somelog\"}]";
    final int testPayloadSize = testData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Then
    Assertions.assertFalse(testBuffer.isEmpty());
    final Map.Entry<Collection<Record<Log>>, CheckpointState> result = testBuffer.read(100);
    List<Record<Log>> records = new ArrayList<>(result.getKey());
    Assertions.assertEquals(1, records.size());
    final Record<Log> record = records.get(0);
    Assertions.assertEquals("somelog", record.getData().get("log", String.class));
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement requestProcessDurationCount = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestProcessDurationCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    Assertions.assertTrue(requestProcessDurationMax.getValue() > 0);
    final Measurement payloadSizeMax = MetricsTestUtil.getMeasurementFromList(payloadSizeSummaryMeasurements, Statistic.MAX);
    Assertions.assertEquals(testPayloadSize, payloadSizeMax.getValue());
}