Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 96 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceGroupProcessorTests method buildSpanRecordFromJsonFile.

private Record<Span> buildSpanRecordFromJsonFile(final String jsonFileName) throws IOException {
    JacksonSpan.Builder spanBuilder = JacksonSpan.builder();
    try (final InputStream inputStream = Objects.requireNonNull(OTelTraceGroupProcessorTests.class.getClassLoader().getResourceAsStream(jsonFileName))) {
        final Map<String, Object> spanMap = OBJECT_MAPPER.readValue(inputStream, new TypeReference<Map<String, Object>>() {
        });
        final String traceId = (String) spanMap.get("traceId");
        final String spanId = (String) spanMap.get("spanId");
        final String parentSpanId = (String) spanMap.get("parentSpanId");
        final String traceState = (String) spanMap.get("traceState");
        final String name = (String) spanMap.get("name");
        final String kind = (String) spanMap.get("kind");
        final Long durationInNanos = ((Number) spanMap.get("durationInNanos")).longValue();
        final String startTime = (String) spanMap.get("startTime");
        final String endTime = (String) spanMap.get("endTime");
        final String traceGroup = (String) spanMap.get(TraceGroup.TRACE_GROUP_NAME_FIELD);
        spanBuilder = spanBuilder.withTraceId(traceId).withSpanId(spanId).withParentSpanId(parentSpanId).withTraceState(traceState).withName(name).withKind(kind).withDurationInNanos(durationInNanos).withStartTime(startTime).withEndTime(endTime).withTraceGroup(traceGroup);
        DefaultTraceGroupFields.Builder traceGroupFieldsBuilder = DefaultTraceGroupFields.builder();
        if (traceGroup != null) {
            final Integer traceGroupFieldsStatusCode = ((Number) spanMap.get(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD)).intValue();
            final String traceGroupFieldsEndTime = (String) spanMap.get(TraceGroup.TRACE_GROUP_END_TIME_FIELD);
            final Long traceGroupFieldsDurationInNanos = ((Number) spanMap.get(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD)).longValue();
            traceGroupFieldsBuilder = traceGroupFieldsBuilder.withStatusCode(traceGroupFieldsStatusCode).withEndTime(traceGroupFieldsEndTime).withDurationInNanos(traceGroupFieldsDurationInNanos);
        }
        spanBuilder = spanBuilder.withTraceGroup(traceGroup).withTraceGroupFields(traceGroupFieldsBuilder.build());
    }
    return new Record<>(spanBuilder.build());
}

Example 97 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OTelTraceRawProcessorTest method setup.

@Before
public void setup() {
    TEST_TRACE_GROUP_1_ROOT_SPAN = buildSpanFromJsonFile(TEST_TRACE_GROUP_1_ROOT_SPAN_JSON_FILE);
    TEST_TRACE_GROUP_1_CHILD_SPAN_1 = buildSpanFromJsonFile(TEST_TRACE_GROUP_1_CHILD_SPAN_1_JSON_FILE);
    TEST_TRACE_GROUP_1_CHILD_SPAN_2 = buildSpanFromJsonFile(TEST_TRACE_GROUP_1_CHILD_SPAN_2_JSON_FILE);
    TEST_TRACE_GROUP_2_ROOT_SPAN = buildSpanFromJsonFile(TEST_TRACE_GROUP_2_ROOT_SPAN_JSON_FILE);
    TEST_TRACE_GROUP_2_CHILD_SPAN_1 = buildSpanFromJsonFile(TEST_TRACE_GROUP_2_CHILD_SPAN_1_JSON_FILE);
    TEST_TRACE_GROUP_2_CHILD_SPAN_2 = buildSpanFromJsonFile(TEST_TRACE_GROUP_2_CHILD_SPAN_2_JSON_FILE);
    TEST_ONE_FULL_TRACE_GROUP_RECORDS = Stream.of(TEST_TRACE_GROUP_1_ROOT_SPAN, TEST_TRACE_GROUP_1_CHILD_SPAN_1, TEST_TRACE_GROUP_1_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    TEST_ONE_TRACE_GROUP_MISSING_ROOT_RECORDS = Stream.of(TEST_TRACE_GROUP_2_CHILD_SPAN_1, TEST_TRACE_GROUP_2_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    TEST_TWO_FULL_TRACE_GROUP_RECORDS = Stream.of(TEST_TRACE_GROUP_1_ROOT_SPAN, TEST_TRACE_GROUP_1_CHILD_SPAN_1, TEST_TRACE_GROUP_1_CHILD_SPAN_2, TEST_TRACE_GROUP_2_ROOT_SPAN, TEST_TRACE_GROUP_2_CHILD_SPAN_1, TEST_TRACE_GROUP_2_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    TEST_TWO_TRACE_GROUP_INTERLEAVED_PART_1_RECORDS = Stream.of(TEST_TRACE_GROUP_1_ROOT_SPAN, TEST_TRACE_GROUP_2_CHILD_SPAN_1, TEST_TRACE_GROUP_2_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    TEST_TWO_TRACE_GROUP_INTERLEAVED_PART_2_RECORDS = Stream.of(TEST_TRACE_GROUP_2_ROOT_SPAN, TEST_TRACE_GROUP_1_CHILD_SPAN_1, TEST_TRACE_GROUP_1_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    TEST_TWO_TRACE_GROUP_MISSING_ROOT_RECORDS = Stream.of(TEST_TRACE_GROUP_1_CHILD_SPAN_1, TEST_TRACE_GROUP_1_CHILD_SPAN_2, TEST_TRACE_GROUP_2_CHILD_SPAN_1, TEST_TRACE_GROUP_2_CHILD_SPAN_2).map(Record::new).collect(Collectors.toList());
    MetricsTestUtil.initMetrics();
    pluginSetting = new PluginSetting("OTelTrace", new HashMap<String, Object>() {

        {
            put(OtelTraceRawProcessorConfig.TRACE_FLUSH_INTERVAL, TEST_TRACE_FLUSH_INTERVAL);
        }
    });
    pluginSetting.setPipelineName("pipelineOTelTrace");
    pluginSetting.setProcessWorkers(TEST_CONCURRENCY_SCALE);
    oTelTraceRawProcessor = new OTelTraceRawProcessor(pluginSetting);
    executorService = Executors.newFixedThreadPool(TEST_CONCURRENCY_SCALE);
}

Example 98 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarder method doExecute.

@Override
public List<Record<ExportTraceServiceRequest>> doExecute(final Collection<Record<ExportTraceServiceRequest>> records) {
    final Map<String, List<ResourceSpans>> groupedRS = new HashMap<>();
    // Group ResourceSpans by consistent hashing of traceId
    for (final Record<ExportTraceServiceRequest> record : records) {
        for (final ResourceSpans rs : record.getData().getResourceSpansList()) {
            final List<Map.Entry<String, ResourceSpans>> rsBatch = PeerForwarderUtils.splitByTrace(rs);
            for (final Map.Entry<String, ResourceSpans> entry : rsBatch) {
                final String traceId = entry.getKey();
                final ResourceSpans newRS = entry.getValue();
                final String dataPrepperIp = hashRing.getServerIp(traceId).orElse(StaticPeerListProvider.LOCAL_ENDPOINT);
                groupedRS.computeIfAbsent(dataPrepperIp, x -> new ArrayList<>()).add(newRS);
            }
        }
    }
    final List<Record<ExportTraceServiceRequest>> recordsToProcessLocally = new ArrayList<>();
    final List<CompletableFuture<Record>> forwardedRequestFutures = new ArrayList<>();
    for (final Map.Entry<String, List<ResourceSpans>> entry : groupedRS.entrySet()) {
        final TraceServiceGrpc.TraceServiceBlockingStub client = getClient(entry.getKey());
        // Create ExportTraceRequest for storing single batch of spans
        ExportTraceServiceRequest.Builder currRequestBuilder = ExportTraceServiceRequest.newBuilder();
        int currSpansCount = 0;
        for (final ResourceSpans rs : entry.getValue()) {
            final int rsSize = PeerForwarderUtils.getResourceSpansSize(rs);
            if (currSpansCount >= maxNumSpansPerRequest) {
                final ExportTraceServiceRequest currRequest = currRequestBuilder.build();
                if (isLocalClient(client)) {
                    recordsToProcessLocally.add(new Record<>(currRequest));
                } else {
                    forwardedRequestFutures.add(processRequest(client, currRequest));
                }
                currRequestBuilder = ExportTraceServiceRequest.newBuilder();
                currSpansCount = 0;
            }
            currRequestBuilder.addResourceSpans(rs);
            currSpansCount += rsSize;
        }
        // Dealing with the last batch request
        if (currSpansCount > 0) {
            final ExportTraceServiceRequest currRequest = currRequestBuilder.build();
            if (client == null) {
                recordsToProcessLocally.add(new Record<>(currRequest));
            } else {
                forwardedRequestFutures.add(processRequest(client, currRequest));
            }
        }
    }
    for (final CompletableFuture<Record> future : forwardedRequestFutures) {
        try {
            final Record record = future.get();
            if (record != null) {
                recordsToProcessLocally.add(record);
            }
        } catch (InterruptedException | ExecutionException e) {
            LOG.error("Problem with asynchronous peer forwarding", e);
        }
    }
    return recordsToProcessLocally;
}

Example 99 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpBothLocalAndForwardedRequest.

@Test
public void testSingleRemoteIpBothLocalAndForwardedRequest() {
    final List<String> testIps = generateTestIps(2);
    final Channel channel = mock(Channel.class);
    final String peerIp = testIps.get(1);
    when(channel.authority()).thenReturn(String.format("%s:21890", peerIp));
    when(peerClientPool.getClient(peerIp)).thenReturn(client);
    when(client.getChannel()).thenReturn(channel);
    final Map<String, List<ExportTraceServiceRequest>> requestsByIp = testIps.stream().collect(Collectors.toMap(ip -> ip, ip -> new ArrayList<>()));
    doAnswer(invocation -> {
        final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0);
        requestsByIp.get(peerIp).add(exportTraceServiceRequest);
        return null;
    }).when(client).export(any(ExportTraceServiceRequest.class));
    MetricsTestUtil.initMetrics();
    final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
    final List<Record<ExportTraceServiceRequest>> exportedRecords = testPeerForwarder.doExecute(Arrays.asList(new Record<>(REQUEST_1), new Record<>(REQUEST_2)));
    final List<ResourceSpans> expectedLocalResourceSpans = Arrays.asList(generateResourceSpans(SPAN_1, SPAN_2), generateResourceSpans(SPAN_3));
    final List<ResourceSpans> expectedForwardedResourceSpans = Arrays.asList(generateResourceSpans(SPAN_5, SPAN_6), generateResourceSpans(SPAN_4));
    Assert.assertEquals(1, exportedRecords.size());
    final ExportTraceServiceRequest localRequest = exportedRecords.get(0).getData();
    final List<ResourceSpans> localResourceSpans = localRequest.getResourceSpansList();
    assertTrue(localResourceSpans.containsAll(expectedLocalResourceSpans));
    assertTrue(expectedLocalResourceSpans.containsAll(localResourceSpans));
    Assert.assertEquals(1, requestsByIp.get(peerIp).size());
    final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0);
    final List<ResourceSpans> forwardedResourceSpans = forwardedRequest.getResourceSpansList();
    assertTrue(forwardedResourceSpans.containsAll(expectedForwardedResourceSpans));
    assertTrue(expectedForwardedResourceSpans.containsAll(forwardedResourceSpans));
}

Example 100 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpForwardedRequestOnly.

@Test
public void testSingleRemoteIpForwardedRequestOnly() throws Exception {
    final List<String> testIps = generateTestIps(2);
    final Channel channel = mock(Channel.class);
    final String peerIp = testIps.get(1);
    final String fullPeerIp = String.format("%s:21890", peerIp);
    when(channel.authority()).thenReturn(fullPeerIp);
    when(peerClientPool.getClient(peerIp)).thenReturn(client);
    when(client.getChannel()).thenReturn(channel);
    final Map<String, List<ExportTraceServiceRequest>> requestsByIp = testIps.stream().collect(Collectors.toMap(ip -> ip, ip -> new ArrayList<>()));
    doAnswer(invocation -> {
        final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0);
        requestsByIp.get(peerIp).add(exportTraceServiceRequest);
        return null;
    }).when(client).export(any(ExportTraceServiceRequest.class));
    MetricsTestUtil.initMetrics();
    final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
    final List<Record<ExportTraceServiceRequest>> exportedRecords = testPeerForwarder.doExecute(Collections.singletonList(new Record<>(REQUEST_4)));
    final List<ResourceSpans> expectedForwardedResourceSpans = Collections.singletonList(generateResourceSpans(SPAN_4, SPAN_5, SPAN_6));
    Assert.assertEquals(1, requestsByIp.get(peerIp).size());
    final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0);
    final List<ResourceSpans> forwardedResourceSpans = forwardedRequest.getResourceSpansList();
    assertTrue(forwardedResourceSpans.containsAll(expectedForwardedResourceSpans));
    assertTrue(expectedForwardedResourceSpans.containsAll(forwardedResourceSpans));
    Assert.assertEquals(0, exportedRecords.size());
    // Verify metrics
    final List<Measurement> forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.ERRORS).toString());
    Assert.assertEquals(1, forwardRequestErrorMeasurements.size());
    Assert.assertEquals(0.0, forwardRequestErrorMeasurements.get(0).getValue(), 0);
    final List<Measurement> forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.REQUESTS).toString());
    Assert.assertEquals(1, forwardRequestSuccessMeasurements.size());
    Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0);
    final List<Measurement> forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder").add(PeerForwarder.LATENCY).toString());
    Assert.assertEquals(3, forwardRequestLatencyMeasurements.size());
    // COUNT
    Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0);
    // TOTAL_TIME
    assertTrue(forwardRequestLatencyMeasurements.get(1).getValue() > 0.0);
    // MAX
    assertTrue(forwardRequestLatencyMeasurements.get(2).getValue() > 0.0);
}