Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 51 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractBufferTest method testWriteAllTimeoutMetric.

@Test
public void testWriteAllTimeoutMetric() throws TimeoutException {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferTimeoutImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = Arrays.asList(new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString()));
    // When/Then
    Assert.assertThrows(TimeoutException.class, () -> abstractBuffer.writeAll(testRecords, 1000));
    final List<Measurement> timeoutMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.WRITE_TIMEOUTS).toString());
    Assert.assertEquals(1, timeoutMeasurements.size());
    Assert.assertEquals(1.0, timeoutMeasurements.get(0).getValue(), 0);
}

Example 52 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractBufferTest method testReadMetrics.

@Test
public void testReadMetrics() throws Exception {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        testRecords.add(new Record<>(UUID.randomUUID().toString()));
    }
    abstractBuffer.writeAll(testRecords, 1000);
    // When
    abstractBuffer.read(1000);
    // Then
    final List<Measurement> recordsReadMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_READ).toString());
    final List<Measurement> recordsInFlightMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_INFLIGHT).toString());
    final List<Measurement> recordsProcessedMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(MetricNames.RECORDS_PROCESSED).toString());
    final List<Measurement> readTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.READ_TIME_ELAPSED).toString());
    Assert.assertEquals(1, recordsReadMeasurements.size());
    Assert.assertEquals(5.0, recordsReadMeasurements.get(0).getValue(), 0);
    Assert.assertEquals(1, recordsInFlightMeasurements.size());
    Assert.assertEquals(5, abstractBuffer.getRecordsInFlight());
    final Measurement recordsInFlightMeasurement = recordsInFlightMeasurements.get(0);
    Assert.assertEquals(5.0, recordsInFlightMeasurement.getValue(), 0);
    Assert.assertEquals(1, recordsProcessedMeasurements.size());
    final Measurement recordsProcessedMeasurement = recordsProcessedMeasurements.get(0);
    Assert.assertEquals(0.0, recordsProcessedMeasurement.getValue(), 0);
    Assert.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(readTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(readTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.05, 0.25));
}

Example 53 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractProcessorTest method testMetricsWithPluginSettingsConstructor.

@Test
public void testMetricsWithPluginSettingsConstructor() {
    final String processorName = "testProcessor";
    final String pipelineName = "testPipeline";
    MetricsTestUtil.initMetrics();
    PluginSetting pluginSetting = new PluginSetting(processorName, Collections.emptyMap());
    pluginSetting.setPipelineName(pipelineName);
    AbstractProcessor<Record<String>, Record<String>> processor = new ProcessorImpl(pluginSetting);
    processor.execute(Arrays.asList(new Record<>("Value1"), new Record<>("Value2"), new Record<>("Value3")));
    final List<Measurement> recordsInMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(pipelineName).add(processorName).add(MetricNames.RECORDS_IN).toString());
    final List<Measurement> recordsOutMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(pipelineName).add(processorName).add(MetricNames.RECORDS_OUT).toString());
    final List<Measurement> elapsedTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(pipelineName).add(processorName).add(MetricNames.TIME_ELAPSED).toString());
    Assertions.assertEquals(1, recordsInMeasurements.size());
    Assertions.assertEquals(3.0, recordsInMeasurements.get(0).getValue(), 0);
    Assertions.assertEquals(1, recordsOutMeasurements.size());
    Assertions.assertEquals(6.0, recordsOutMeasurements.get(0).getValue(), 0);
    Assertions.assertEquals(3, elapsedTimeMeasurements.size());
    Assertions.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(elapsedTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assertions.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(elapsedTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.1, 0.2));
}

Example 54 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractSinkTest method testMetrics.

@Test
public void testMetrics() {
    final String sinkName = "testSink";
    final String pipelineName = "pipelineName";
    MetricsTestUtil.initMetrics();
    PluginSetting pluginSetting = new PluginSetting(sinkName, Collections.emptyMap());
    pluginSetting.setPipelineName(pipelineName);
    AbstractSink<Record<String>> abstractSink = new AbstractSinkImpl(pluginSetting);
    abstractSink.output(Arrays.asList(new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString()), new Record<>(UUID.randomUUID().toString())));
    final List<Measurement> recordsInMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(pipelineName).add(sinkName).add(MetricNames.RECORDS_IN).toString());
    final List<Measurement> elapsedTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(pipelineName).add(sinkName).add(MetricNames.TIME_ELAPSED).toString());
    Assert.assertEquals(1, recordsInMeasurements.size());
    Assert.assertEquals(5.0, recordsInMeasurements.get(0).getValue(), 0);
    Assert.assertEquals(3, elapsedTimeMeasurements.size());
    Assert.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(elapsedTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(elapsedTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.5, 0.6));
}

Example 55 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpForwardedRequestOnlyWithEventRecordData.

@Test
public void testSingleRemoteIpForwardedRequestOnlyWithEventRecordData() throws Exception {
    final List<String> testIps = generateTestIps(2);
    final Channel channel = mock(Channel.class);
    final String peerIp = testIps.get(1);
    final String fullPeerIp = String.format("%s:21890", peerIp);
    when(channel.authority()).thenReturn(fullPeerIp);
    when(peerClientPool.getClient(peerIp)).thenReturn(client);
    when(client.getChannel()).thenReturn(channel);
    final Map<String, List<ExportTraceServiceRequest>> requestsByIp = testIps.stream().collect(Collectors.toMap(ip -> ip, ip -> new ArrayList<>()));
    doAnswer(invocation -> {
        final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0);
        requestsByIp.get(peerIp).add(exportTraceServiceRequest);
        return null;
    }).when(client).export(any(ExportTraceServiceRequest.class));
    MetricsTestUtil.initMetrics();
    final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
    final List<Record<Object>> exportedRecords = testPeerForwarder.doExecute(TEST_SPANS_B.stream().map(span -> new Record<Object>(span)).collect(Collectors.toList()));
    Assert.assertEquals(1, requestsByIp.get(peerIp).size());
    final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0);
    final List<ResourceSpans> forwardedResourceSpans = forwardedRequest.getResourceSpansList();
    assertEquals(3, forwardedResourceSpans.size());
    forwardedResourceSpans.forEach(rs -> {
        assertEquals(TEST_SERVICE_B, extractServiceName(rs));
        assertEquals(1, rs.getInstrumentationLibrarySpansCount());
        final InstrumentationLibrarySpans ils = rs.getInstrumentationLibrarySpans(0);
        assertEquals(1, ils.getSpansCount());
        final io.opentelemetry.proto.trace.v1.Span sp = ils.getSpans(0);
        assertEquals(TEST_TRACE_ID_2, Hex.encodeHexString(sp.getTraceId().toByteArray()));
    });
    Assert.assertEquals(0, exportedRecords.size());
    // Verify metrics
    final List<Measurement> forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.ERRORS).toString());
    Assert.assertEquals(1, forwardRequestErrorMeasurements.size());
    Assert.assertEquals(0.0, forwardRequestErrorMeasurements.get(0).getValue(), 0);
    final List<Measurement> forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.REQUESTS).toString());
    Assert.assertEquals(1, forwardRequestSuccessMeasurements.size());
    Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.LATENCY).toString());
    Assert.assertEquals(3, forwardRequestLatencyMeasurements.size());
    // COUNT
    Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0);
    // TOTAL_TIME
    assertTrue(forwardRequestLatencyMeasurements.get(1).getValue() > 0.0);
    // MAX
    assertTrue(forwardRequestLatencyMeasurements.get(2).getValue() > 0.0);
}