Examples with Span com.amazon.dataprepper.model.trace.Span used on opensource projects

Example 11 with Span

use of com.amazon.dataprepper.model.trace.Span in project data-prepper by opensearch-project.

the class ServiceMapStatefulPrepperTest method testTraceGroupsWithEventRecordData.

@Test
public void testTraceGroupsWithEventRecordData() throws Exception {
    final Clock clock = Mockito.mock(Clock.class);
    Mockito.when(clock.millis()).thenReturn(1L);
    Mockito.when(clock.instant()).thenReturn(Instant.now());
    ExecutorService threadpool = Executors.newCachedThreadPool();
    final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH);
    final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING);
    final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING);
    final byte[] rootSpanId1Bytes = ServiceMapTestUtils.getRandomBytes(8);
    final byte[] rootSpanId2Bytes = ServiceMapTestUtils.getRandomBytes(8);
    final byte[] traceId1Bytes = ServiceMapTestUtils.getRandomBytes(16);
    final byte[] traceId2Bytes = ServiceMapTestUtils.getRandomBytes(16);
    final String rootSpanId1 = Hex.encodeHexString(rootSpanId1Bytes);
    final String rootSpanId2 = Hex.encodeHexString(rootSpanId2Bytes);
    final String traceId1 = Hex.encodeHexString(traceId1Bytes);
    final String traceId2 = Hex.encodeHexString(traceId2Bytes);
    final String traceGroup1 = "reset_password";
    final String traceGroup2 = "checkout";
    final Span frontendSpans1 = ServiceMapTestUtils.getSpan(FRONTEND_SERVICE, traceGroup1, rootSpanId1, "", traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT);
    final Span authenticationSpansServer = ServiceMapTestUtils.getSpan(AUTHENTICATION_SERVICE, "reset", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), frontendSpans1.getSpanId(), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    final Span authenticationSpansClient = ServiceMapTestUtils.getSpan(AUTHENTICATION_SERVICE, "reset", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), authenticationSpansServer.getSpanId(), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT);
    final Span passwordDbSpans = ServiceMapTestUtils.getSpan(PASSWORD_DATABASE, "update", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), authenticationSpansClient.getSpanId(), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    final Span frontendSpans2 = ServiceMapTestUtils.getSpan(FRONTEND_SERVICE, traceGroup2, rootSpanId2, "", traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT);
    final Span checkoutSpansServer = ServiceMapTestUtils.getSpan(CHECKOUT_SERVICE, "checkout", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), rootSpanId2, traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    final Span checkoutSpansClient = ServiceMapTestUtils.getSpan(CHECKOUT_SERVICE, "checkout", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), checkoutSpansServer.getSpanId(), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT);
    final Span cartSpans = ServiceMapTestUtils.getSpan(CART_SERVICE, "get_items", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), checkoutSpansClient.getSpanId(), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    final Span paymentSpans = ServiceMapTestUtils.getSpan(PAYMENT_SERVICE, "charge", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), checkoutSpansClient.getSpanId(), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    // Expected relationships
    final ServiceMapRelationship frontendAuth = ServiceMapRelationship.newDestinationRelationship(FRONTEND_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), AUTHENTICATION_SERVICE, "reset", traceGroup1);
    final ServiceMapRelationship authPassword = ServiceMapRelationship.newDestinationRelationship(AUTHENTICATION_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), PASSWORD_DATABASE, "update", traceGroup1);
    final ServiceMapRelationship frontendCheckout = ServiceMapRelationship.newDestinationRelationship(FRONTEND_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), CHECKOUT_SERVICE, "checkout", traceGroup2);
    final ServiceMapRelationship checkoutCart = ServiceMapRelationship.newDestinationRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), CART_SERVICE, "get_items", traceGroup2);
    final ServiceMapRelationship checkoutPayment = ServiceMapRelationship.newDestinationRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), PAYMENT_SERVICE, "charge", traceGroup2);
    final ServiceMapRelationship checkoutTarget = ServiceMapRelationship.newTargetRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), CHECKOUT_SERVICE, "checkout", traceGroup2);
    final ServiceMapRelationship authTarget = ServiceMapRelationship.newTargetRelationship(AUTHENTICATION_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), AUTHENTICATION_SERVICE, "reset", traceGroup1);
    final ServiceMapRelationship passwordTarget = ServiceMapRelationship.newTargetRelationship(PASSWORD_DATABASE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), PASSWORD_DATABASE, "update", traceGroup1);
    final ServiceMapRelationship paymentTarget = ServiceMapRelationship.newTargetRelationship(PAYMENT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), PAYMENT_SERVICE, "charge", traceGroup2);
    final ServiceMapRelationship cartTarget = ServiceMapRelationship.newTargetRelationship(CART_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), CART_SERVICE, "get_items", traceGroup2);
    final Set<ServiceMapRelationship> relationshipsFound = new HashSet<>();
    // First batch
    Mockito.when(clock.millis()).thenReturn(110L);
    Future<Set<ServiceMapRelationship>> r1 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList(new Record<>(frontendSpans1), new Record<>(checkoutSpansServer)));
    Future<Set<ServiceMapRelationship>> r2 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList(new Record<>(frontendSpans2), new Record<>(checkoutSpansClient)));
    relationshipsFound.addAll(r1.get());
    relationshipsFound.addAll(r2.get());
    // Shouldn't find any relationships
    Assert.assertEquals(0, relationshipsFound.size());
    // Second batch
    Mockito.when(clock.millis()).thenReturn(220L);
    Future<Set<ServiceMapRelationship>> r3 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList(new Record<>(authenticationSpansServer), new Record<>(authenticationSpansClient), new Record<>(cartSpans)));
    Future<Set<ServiceMapRelationship>> r4 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList(new Record<>(passwordDbSpans), new Record<>(paymentSpans)));
    relationshipsFound.addAll(r3.get());
    relationshipsFound.addAll(r4.get());
    // Should find the frontend->checkout relationship indicated in the first batch
    Assert.assertEquals(2, relationshipsFound.size());
    assertTrue(relationshipsFound.containsAll(Arrays.asList(frontendCheckout, checkoutTarget)));
    // Third batch
    Mockito.when(clock.millis()).thenReturn(340L);
    Future<Set<ServiceMapRelationship>> r5 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList());
    Future<Set<ServiceMapRelationship>> r6 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList());
    relationshipsFound.addAll(r5.get());
    relationshipsFound.addAll(r6.get());
    // Should find the rest of the relationships
    Assert.assertEquals(10, relationshipsFound.size());
    assertTrue(relationshipsFound.containsAll(Arrays.asList(frontendAuth, authTarget, authPassword, passwordTarget, checkoutCart, cartTarget, checkoutPayment, paymentTarget)));
    // Extra validation
    final List<ServiceMapSourceDest> expectedSourceDests = Arrays.asList(new ServiceMapSourceDest(FRONTEND_SERVICE, AUTHENTICATION_SERVICE), new ServiceMapSourceDest(AUTHENTICATION_SERVICE, PASSWORD_DATABASE), new ServiceMapSourceDest(FRONTEND_SERVICE, CHECKOUT_SERVICE), new ServiceMapSourceDest(CHECKOUT_SERVICE, CART_SERVICE), new ServiceMapSourceDest(CHECKOUT_SERVICE, PAYMENT_SERVICE));
    assertTrue(evaluateEdges(relationshipsFound).containsAll(expectedSourceDests));
    // Verify gauges
    final List<Measurement> spansDbSizeMeasurement = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper").add(ServiceMapStatefulPrepper.SPANS_DB_SIZE).toString());
    Assert.assertEquals(1, spansDbSizeMeasurement.size());
    final List<Measurement> traceGroupDbSizeMeasurement = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper").add(ServiceMapStatefulPrepper.TRACE_GROUP_DB_SIZE).toString());
    Assert.assertEquals(1, traceGroupDbSizeMeasurement.size());
    // Make sure that future relationships that are equivalent are caught by cache
    final byte[] rootSpanId3Bytes = ServiceMapTestUtils.getRandomBytes(8);
    final byte[] traceId3Bytes = ServiceMapTestUtils.getRandomBytes(16);
    final String rootSpanId3 = Hex.encodeHexString(rootSpanId3Bytes);
    final String traceId3 = Hex.encodeHexString(traceId3Bytes);
    final Span frontendSpans3 = ServiceMapTestUtils.getSpan(FRONTEND_SERVICE, traceGroup1, rootSpanId3, rootSpanId3, traceId3, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT);
    final Span authenticationSpansServer2 = ServiceMapTestUtils.getSpan(AUTHENTICATION_SERVICE, "reset", Hex.encodeHexString(ServiceMapTestUtils.getRandomBytes(8)), frontendSpans3.getSpanId(), traceId3, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER);
    when(clock.millis()).thenReturn(450L);
    Future<Set<ServiceMapRelationship>> r7 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Collections.singletonList(new Record<>(frontendSpans3)));
    Future<Set<ServiceMapRelationship>> r8 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Collections.singletonList(new Record<>(authenticationSpansServer2)));
    assertTrue(r7.get().isEmpty());
    assertTrue(r8.get().isEmpty());
    when(clock.millis()).thenReturn(560L);
    Future<Set<ServiceMapRelationship>> r9 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList());
    Future<Set<ServiceMapRelationship>> r10 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList());
    assertTrue(r9.get().isEmpty());
    assertTrue(r10.get().isEmpty());
    serviceMapStateful1.shutdown();
    serviceMapStateful2.shutdown();
}

Example 12 with Span

use of com.amazon.dataprepper.model.trace.Span in project data-prepper by opensearch-project.

the class PeerForwarder method executeSpans.

private List<Span> executeSpans(final List<Span> spans) {
    final Map<String, List<Span>> spansByTraceId = PeerForwarderUtils.splitByTrace(spans);
    // Group ResourceSpans by consistent hashing of traceId
    final Map<String, List<Span>> spansByEndPoint = new HashMap<>();
    for (final Map.Entry<String, List<Span>> entry : spansByTraceId.entrySet()) {
        final String traceId = entry.getKey();
        final String dataPrepperIp = hashRing.getServerIp(traceId).orElse(StaticPeerListProvider.LOCAL_ENDPOINT);
        spansByEndPoint.computeIfAbsent(dataPrepperIp, x -> new ArrayList<>()).addAll(entry.getValue());
    }
    final List<Span> spansToProcessLocally = new ArrayList<>();
    final Map<CompletableFuture<ExportTraceServiceRequest>, List<Span>> forwardedRequestFuturesToSpans = new HashMap<>();
    for (final Map.Entry<String, List<Span>> entry : spansByEndPoint.entrySet()) {
        final TraceServiceGrpc.TraceServiceBlockingStub client = getClient(entry.getKey());
        if (isLocalClient(client)) {
            spansToProcessLocally.addAll(entry.getValue());
            continue;
        }
        // Create ExportTraceRequest for storing single batch of spans
        ExportTraceServiceRequest.Builder currRequestBuilder = ExportTraceServiceRequest.newBuilder();
        List<Span> currBatchSpans = new ArrayList<>();
        for (final Span span : entry.getValue()) {
            try {
                final ResourceSpans rs = oTelProtoEncoder.convertToResourceSpans(span);
                currRequestBuilder.addResourceSpans(rs);
                currBatchSpans.add(span);
            } catch (UnsupportedEncodingException | DecoderException e) {
                LOG.error("failed to encode span with spanId: {} into opentelemetry-protobuf, span will be processed locally.", span.getSpanId(), e);
                spansToProcessLocally.add(span);
            }
            if (currBatchSpans.size() >= maxNumSpansPerRequest) {
                final ExportTraceServiceRequest currRequest = currRequestBuilder.build();
                forwardedRequestFuturesToSpans.put(processRequest(client, currRequest), currBatchSpans);
                currRequestBuilder = ExportTraceServiceRequest.newBuilder();
                currBatchSpans = new ArrayList<>();
            }
        }
        // Dealing with the last batch request
        if (currBatchSpans.size() > 0) {
            final ExportTraceServiceRequest currRequest = currRequestBuilder.build();
            forwardedRequestFuturesToSpans.put(processRequest(client, currRequest), currBatchSpans);
        }
    }
    for (final Map.Entry<CompletableFuture<ExportTraceServiceRequest>, List<Span>> entry : forwardedRequestFuturesToSpans.entrySet()) {
        try {
            final CompletableFuture<ExportTraceServiceRequest> future = entry.getKey();
            final ExportTraceServiceRequest request = future.get();
            if (request != null) {
                final List<Span> spansFailedToForward = entry.getValue();
                spansToProcessLocally.addAll(spansFailedToForward);
            }
        } catch (InterruptedException | ExecutionException e) {
            LOG.error("Problem with asynchronous peer forwarding, current batch of spans will be processed locally.", e);
            final List<Span> spansFailedToForward = entry.getValue();
            spansToProcessLocally.addAll(spansFailedToForward);
        }
    }
    return spansToProcessLocally;
}

Example 13 with Span

use of com.amazon.dataprepper.model.trace.Span in project data-prepper by opensearch-project.

the class OTelTraceRawProcessorTest method getMissingTraceGroupFieldsSpanCount.

private int getMissingTraceGroupFieldsSpanCount(List<Record<Span>> records) {
    int count = 0;
    for (Record<Span> record : records) {
        final Span span = record.getData();
        final String traceGroup = span.getTraceGroup();
        final TraceGroupFields traceGroupFields = span.getTraceGroupFields();
        if (Stream.of(traceGroup, traceGroupFields.getEndTime(), traceGroupFields.getDurationInNanos(), traceGroupFields.getStatusCode()).allMatch(Objects::isNull)) {
            count += 1;
        }
    }
    return count;
}

Example 14 with Span

use of com.amazon.dataprepper.model.trace.Span in project data-prepper by opensearch-project.

the class OTelTraceRawProcessor method processRootSpan.

/**
 * Retrieves all child spans from memory and returns them as a set with the root span.
 * Also adds an entry to the traceID cache so that later child spans can be tagged,
 * in the case where a child span is processed AFTER the root span.
 *
 * @param parentSpan
 * @return List containing root span, along with any child spans that have already been processed.
 */
private List<Span> processRootSpan(final Span parentSpan) {
    final TraceGroup traceGroup = new TraceGroup.TraceGroupBuilder().setFromSpan(parentSpan).build();
    final String parentSpanTraceId = parentSpan.getTraceId();
    traceIdTraceGroupCache.put(parentSpanTraceId, traceGroup);
    final List<Span> recordsToFlush = new LinkedList<>();
    recordsToFlush.add(parentSpan);
    final SpanSet spanSet = traceIdSpanSetMap.get(parentSpanTraceId);
    if (spanSet != null) {
        for (final Span span : spanSet.getSpans()) {
            fillInTraceGroupInfo(span, traceGroup);
            recordsToFlush.add(span);
        }
        traceIdSpanSetMap.remove(parentSpanTraceId);
    }
    return recordsToFlush;
}

Example 15 with Span

use of com.amazon.dataprepper.model.trace.Span in project data-prepper by opensearch-project.

the class OTelTraceRawProcessor method doExecute.

/**
 * execute the processor logic which could potentially modify the incoming record. The level to which the record has
 * been modified depends on the implementation
 *
 * @param records Input records that will be modified/processed
 * @return Record  modified output records
 */
@Override
public Collection<Record<Span>> doExecute(Collection<Record<Span>> records) {
    final List<Span> processedSpans = new LinkedList<>();
    for (Record<Span> record : records) {
        final Span span = record.getData();
        processSpan(span, processedSpans);
    }
    processedSpans.addAll(getTracesToFlushByGarbageCollection());
    return processedSpans.stream().map(Record::new).collect(Collectors.toList());
}