Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 46 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceGroupPrepperTests method testTraceGroupFillFailDueToFailedRequest.

@Test
public void testTraceGroupFillFailDueToFailedRequest() throws IOException {
    // Arrange
    Record<String> testRecord = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1);
    List<Record<String>> testRecords = Collections.singletonList(testRecord);
    when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))).thenThrow(new OpenSearchException("Failure due to search request"));
    // Act
    List<Record<String>> recordsOut = (List<Record<String>>) otelTraceGroupPrepper.doExecute(testRecords);
    // Assert
    assertEquals(1, recordsOut.size());
    Record<String> recordOut = recordsOut.get(0);
    assertEquals(testRecord, recordOut);
    checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_IN_MISSING_TRACE_GROUP, 1.0);
    checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_FIXED_TRACE_GROUP, 0.0);
    checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_MISSING_TRACE_GROUP, 1.0);
}

Example 47 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class MetricsPluginHistogramTest method test.

@Test
public void test() throws JsonProcessingException {
    final double bound_0 = 10.0;
    final double bound_1 = 100.0;
    final double bound_2 = 1000.0;
    HistogramDataPoint dp = HistogramDataPoint.newBuilder().addBucketCounts(0).addBucketCounts(5).addBucketCounts(17).addBucketCounts(33).addExplicitBounds(bound_0).addExplicitBounds(bound_1).addExplicitBounds(bound_2).setCount(4).setSum(1d / 3d).build();
    Histogram histogram = Histogram.newBuilder().addDataPoints(dp).build();
    List<Record<? extends Metric>> processedRecords = (List<Record<? extends Metric>>) rawProcessor.doExecute(Collections.singletonList(new Record<>(fillServiceRequest(histogram))));
    Record<? extends Metric> record = processedRecords.get(0);
    ObjectMapper objectMapper = new ObjectMapper();
    Map<Object, Object> map = objectMapper.readValue(record.getData().toJsonString(), Map.class);
    DefaultBucket bucket_0 = new DefaultBucket((double) -Float.MAX_VALUE, bound_0, 0L);
    DefaultBucket bucket_1 = new DefaultBucket(bound_0, bound_1, 5L);
    DefaultBucket bucket_2 = new DefaultBucket(bound_1, bound_2, 17L);
    DefaultBucket bucket_3 = new DefaultBucket(bound_2, (double) Float.MAX_VALUE, 33L);
    assertHistogramProcessing(map, Arrays.asList(bucket_0, bucket_1, bucket_2, bucket_3));
}

Example 48 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceGroupProcessor method doExecute.

@Override
public Collection<Record<Span>> doExecute(final Collection<Record<Span>> rawSpanRecords) {
    final List<Record<Span>> recordsOut = new LinkedList<>();
    final Set<Record<Span>> recordsMissingTraceGroupInfo = new HashSet<>();
    final Set<String> traceIdsToLookUp = new HashSet<>();
    for (Record<Span> record : rawSpanRecords) {
        final Span span = record.getData();
        final String traceGroup = span.getTraceGroup();
        final String traceId = span.getTraceId();
        if (Strings.isNullOrEmpty(traceGroup)) {
            traceIdsToLookUp.add(traceId);
            recordsMissingTraceGroupInfo.add(record);
            recordsInMissingTraceGroupCounter.increment();
        } else {
            recordsOut.add(record);
        }
    }
    final Map<String, TraceGroup> traceIdToTraceGroup = searchTraceGroupByTraceIds(traceIdsToLookUp);
    for (final Record<Span> record : recordsMissingTraceGroupInfo) {
        final Span span = record.getData();
        final String traceId = span.getTraceId();
        final TraceGroup traceGroup = traceIdToTraceGroup.get(traceId);
        if (traceGroup != null) {
            try {
                fillInTraceGroupInfo(span, traceGroup);
                recordsOut.add(record);
                recordsOutFixedTraceGroupCounter.increment();
            } catch (Exception e) {
                recordsOut.add(record);
                recordsOutMissingTraceGroupCounter.increment();
                LOG.error("Failed to process the span: [{}]", record.getData(), e);
            }
        } else {
            recordsOut.add(record);
            recordsOutMissingTraceGroupCounter.increment();
            final String spanId = span.getSpanId();
            LOG.warn("Failed to find traceGroup for spanId: {} due to traceGroup missing for traceId: {}", spanId, traceId);
        }
    }
    return recordsOut;
}

Example 49 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceGroupProcessorTests method testTraceGroupFillFailDueToFailedRequest.

@Test
public void testTraceGroupFillFailDueToFailedRequest() throws IOException {
    // Arrange
    Record<Span> testRecord = buildSpanRecordFromJsonFile(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1);
    List<Record<Span>> testRecords = Collections.singletonList(testRecord);
    when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))).thenThrow(new OpenSearchException("Failure due to search request"));
    // Act
    List<Record<Span>> recordsOut = (List<Record<Span>>) otelTraceGroupProcessor.doExecute(testRecords);
    // Assert
    assertEquals(1, recordsOut.size());
    Record<Span> recordOut = recordsOut.get(0);
    assertEquals(testRecord, recordOut);
    checkMeasurementValue(OTelTraceGroupProcessor.RECORDS_IN_MISSING_TRACE_GROUP, 1.0);
    checkMeasurementValue(OTelTraceGroupProcessor.RECORDS_OUT_FIXED_TRACE_GROUP, 0.0);
    checkMeasurementValue(OTelTraceGroupProcessor.RECORDS_OUT_MISSING_TRACE_GROUP, 1.0);
}

Example 50 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceRawPrepper method convertRawSpansToJsonRecords.

private List<Record<String>> convertRawSpansToJsonRecords(final List<RawSpan> rawSpans) {
    final List<Record<String>> records = new LinkedList<>();
    for (RawSpan rawSpan : rawSpans) {
        String rawSpanJson;
        try {
            rawSpanJson = rawSpan.toJson();
        } catch (JsonProcessingException e) {
            LOG.error("Unable to process invalid Span {}:", rawSpan, e);
            spanErrorsCounter.increment();
            totalProcessingErrorsCounter.increment();
            continue;
        }
        records.add(new Record<>(rawSpanJson));
    }
    return records;
}