Examples with RawSpan org.hypertrace.core.datamodel.RawSpan used on opensource projects

Example 6 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class RawSpansProcessor method transform.

public KeyValue<String, StructuredTrace> transform(TraceIdentity key, RawSpan value) {
    Instant start = Instant.now();
    long currentTimeMs = System.currentTimeMillis();
    TraceState traceState = traceStateStore.get(key);
    boolean firstEntry = (traceState == null);
    if (shouldDropSpan(key, traceState)) {
        return null;
    }
    String tenantId = key.getTenantId();
    ByteBuffer traceId = value.getTraceId();
    ByteBuffer spanId = value.getEvent().getEventId();
    spanStore.put(new SpanIdentity(tenantId, traceId, spanId), value);
    /*
     the trace emit ts is essentially currentTs + groupingWindowTimeoutMs
     i.e. if there is no span added in the next 'groupingWindowTimeoutMs' interval
     then the trace can be finalized and emitted
    */
    long traceEmitTs = currentTimeMs + groupingWindowTimeoutMs;
    if (logger.isDebugEnabled()) {
        logger.debug("Updating trigger_ts=[{}] for for tenant_id=[{}], trace_id=[{}]", Instant.ofEpochMilli(traceEmitTs), key.getTenantId(), HexUtils.getHex(traceId));
    }
    if (firstEntry) {
        traceState = fastNewBuilder(TraceState.Builder.class).setTraceStartTimestamp(currentTimeMs).setTraceEndTimestamp(currentTimeMs).setEmitTs(traceEmitTs).setTenantId(tenantId).setTraceId(traceId).setSpanIds(List.of(spanId)).build();
        schedulePunctuator(key);
    } else {
        traceState.getSpanIds().add(spanId);
        traceState.setTraceEndTimestamp(currentTimeMs);
        traceState.setEmitTs(traceEmitTs);
    }
    traceStateStore.put(key, traceState);
    tenantToSpansGroupingTimer.computeIfAbsent(value.getCustomerId(), k -> PlatformMetricsRegistry.registerTimer(PROCESSING_LATENCY_TIMER, Map.of("tenantId", k))).record(Duration.between(start, Instant.now()).toMillis(), TimeUnit.MILLISECONDS);
    // the punctuator will emit the trace
    return null;
}

Example 7 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class TraceEmitPunctuator method punctuate.

/**
 * @param timestamp correspond to current system time
 */
@Override
public void punctuate(long timestamp) {
    Instant startTime = Instant.now();
    // always cancel the punctuator else it will get re-scheduled automatically
    cancellable.cancel();
    TraceState traceState = traceStateStore.get(key);
    if (null == traceState || null == traceState.getSpanIds() || traceState.getSpanIds().isEmpty()) {
        /*
       todo - debug why this happens .
       Typically seen when punctuators are created via {@link RawSpansGroupingTransformer.restorePunctuators}
      */
        logger.warn("TraceState for tenant_id=[{}], trace_id=[{}] is missing.", key.getTenantId(), HexUtils.getHex(key.getTraceId()));
        return;
    }
    long emitTs = traceState.getEmitTs();
    if (emitTs <= timestamp) {
        // we can emit this trace so just delete the entry for this 'key'
        // Implies that no new spans for the trace have arrived within the last
        // 'groupingWindowTimeoutMs' interval
        // so the trace can be finalized and emitted
        traceStateStore.delete(key);
        ByteBuffer traceId = traceState.getTraceId();
        String tenantId = traceState.getTenantId();
        List<RawSpan> rawSpanList = new ArrayList<>();
        Set<ByteBuffer> spanIds = new HashSet<>(traceState.getSpanIds());
        spanIds.forEach(v -> {
            SpanIdentity spanIdentity = new SpanIdentity(tenantId, traceId, v);
            RawSpan rawSpan = spanStore.delete(spanIdentity);
            // ideally this shouldn't happen
            if (rawSpan != null) {
                rawSpanList.add(rawSpan);
            }
        });
        if (traceState.getSpanIds().size() != spanIds.size()) {
            tenantToTraceWithDuplicateSpansCounter.computeIfAbsent(tenantId, k -> PlatformMetricsRegistry.registerCounter(TRACE_WITH_DUPLICATE_SPANS, Map.of("tenantId", k))).increment();
            if (logger.isDebugEnabled()) {
                logger.debug("Duplicate spanIds: [{}], unique spanIds count: [{}] for tenant: [{}] trace: [{}]", traceState.getSpanIds().size(), spanIds.size(), tenantId, HexUtils.getHex(traceId));
            }
        }
        recordSpansPerTrace(rawSpanList.size(), List.of(Tag.of("tenant_id", tenantId)));
        Timestamps timestamps = trackEndToEndLatencyTimestamps(timestamp, traceState.getTraceStartTimestamp());
        StructuredTrace trace = StructuredTraceBuilder.buildStructuredTraceFromRawSpans(rawSpanList, traceId, tenantId, timestamps);
        if (logger.isDebugEnabled()) {
            logger.debug("Emit tenant_id=[{}], trace_id=[{}], spans_count=[{}]", tenantId, HexUtils.getHex(traceId), rawSpanList.size());
        }
        // report entries in spanStore
        if (spanStoreCountRateLimiter.tryAcquire()) {
            tenantToSpanStoreCountCounter.computeIfAbsent(tenantId, k -> PlatformMetricsRegistry.registerCounter(SPAN_STORE_COUNT, Map.of("tenantId", k))).increment(spanStore.approximateNumEntries() * 1.0);
        }
        // report count of spanIds per trace
        tenantToSpanPerTraceCounter.computeIfAbsent(tenantId, k -> PlatformMetricsRegistry.registerCounter(SPANS_PER_TRACE, Map.of("tenantId", k))).increment(spanIds.size() * 1.0);
        // report trace emitted count
        tenantToTraceEmittedCounter.computeIfAbsent(tenantId, k -> PlatformMetricsRegistry.registerCounter(TRACES_EMITTER_COUNTER, Map.of("tenantId", k))).increment();
        // report punctuate latency
        tenantToPunctuateLatencyTimer.computeIfAbsent(tenantId, k -> PlatformMetricsRegistry.registerTimer(PUNCTUATE_LATENCY_TIMER, Map.of("tenantId", k))).record(Duration.between(startTime, Instant.now()).toMillis(), TimeUnit.MILLISECONDS);
        context.forward(null, trace, outputTopicProducer);
    } else {
        // so the session inactivity window is extended from the last timestamp
        if (logger.isDebugEnabled()) {
            logger.debug("Re-scheduling emit trigger for tenant_id=[{}], trace_id=[{}] to [{}]", key.getTenantId(), HexUtils.getHex(key.getTraceId()), Instant.ofEpochMilli(emitTs + groupingWindowTimeoutMs));
        }
        long newEmitTs = emitTs + groupingWindowTimeoutMs;
        // if current timestamp is ahead of newEmitTs then just add a grace of 100ms and fire it
        long duration = Math.max(100, newEmitTs - timestamp);
        cancellable = context.schedule(Duration.ofMillis(duration), PunctuationType.WALL_CLOCK_TIME, this);
    }
}

Example 8 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class RawSpansGrouperTest method whenRawSpansAreReceivedWithInactivityExpectTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "raw-spans-grouper")
public void whenRawSpansAreReceivedWithInactivityExpectTraceToBeOutput(@TempDir Path tempDir) {
    File file = tempDir.resolve("state").toFile();
    RawSpansGrouper underTest = new RawSpansGrouper(ConfigClientFactory.getClient());
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/raw-spans-grouper/application.conf"));
    Map<String, Object> baseProps = underTest.getBaseStreamsConfig();
    Map<String, Object> streamsProps = underTest.getStreamsConfig(config);
    baseProps.forEach(streamsProps::put);
    Map<String, Object> mergedProps = streamsProps;
    mergedProps.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, SpecificAvroSerde.class);
    mergedProps.put(RawSpanGrouperConstants.RAW_SPANS_GROUPER_JOB_CONFIG, config);
    mergedProps.put(StreamsConfig.STATE_DIR_CONFIG, file.getAbsolutePath());
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    Serde defaultValueSerde = new StreamsConfig(mergedProps).defaultValueSerde();
    Serde<TraceIdentity> traceIdentitySerde = new StreamsConfig(mergedProps).defaultKeySerde();
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<TraceIdentity, RawSpan> inputTopic = td.createInputTopic(config.getString(RawSpanGrouperConstants.INPUT_TOPIC_CONFIG_KEY), traceIdentitySerde.serializer(), defaultValueSerde.serializer());
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(RawSpanGrouperConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), defaultValueSerde.deserializer());
    String tenantId = "tenant1";
    // create spans for trace-1 of tenant1
    RawSpan span1 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-1".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-1", "tenant1")).build();
    RawSpan span2 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-1".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-2", "tenant1")).build();
    RawSpan span3 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-1".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-3", "tenant1")).build();
    // create spans for trace-2 of tenant1
    RawSpan span4 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-2".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-4", "tenant1")).build();
    RawSpan span5 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-2".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-5", "tenant1")).build();
    // create spans for trace-3 of tenant1
    RawSpan span6 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-6", "tenant1")).build();
    RawSpan span7 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-7", "tenant1")).build();
    RawSpan span8 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-8", "tenant1")).build();
    RawSpan span9 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-9", "tenant1")).build();
    RawSpan span10 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-10", "tenant1")).build();
    RawSpan span11 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-3".getBytes())).setCustomerId("tenant1").setEvent(createEvent("event-11", "tenant1")).build();
    // create 8 spans for tenant-2 for trace-4
    String tenant2 = "tenant2";
    RawSpan span12 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-12", tenant2)).build();
    RawSpan span13 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-13", tenant2)).build();
    RawSpan span14 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-14", tenant2)).build();
    RawSpan span15 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-15", tenant2)).build();
    RawSpan span16 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-16", tenant2)).build();
    RawSpan span17 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-17", tenant2)).build();
    RawSpan span18 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-18", tenant2)).build();
    RawSpan span19 = RawSpan.newBuilder().setTraceId(ByteBuffer.wrap("trace-4".getBytes())).setCustomerId(tenant2).setEvent(createEvent("event-19", tenant2)).build();
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-1"), span1);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-2"), span4);
    td.advanceWallClockTime(Duration.ofSeconds(1));
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-1"), span2);
    // select a value < 30s (groupingWindowTimeoutInMs)
    // this shouldn't trigger a punctuate call
    td.advanceWallClockTime(Duration.ofMillis(200));
    assertTrue(outputTopic.isEmpty());
    // the next advance should trigger a punctuate call and emit a trace with 2 spans
    td.advanceWallClockTime(Duration.ofSeconds(32));
    // trace1 should have 2 span span1, span2
    StructuredTrace trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(2, trace.getEventList().size());
    Set<String> traceEventIds = trace.getEventList().stream().map(id -> new String(id.getEventId().array())).collect(Collectors.toSet());
    assertTrue(traceEventIds.contains("event-1"));
    assertTrue(traceEventIds.contains("event-2"));
    // trace2 should have 1 span span3
    trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(1, trace.getEventList().size());
    assertEquals("event-4", new String(trace.getEventList().get(0).getEventId().array()));
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-1"), span3);
    td.advanceWallClockTime(Duration.ofSeconds(45));
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-2"), span5);
    // the next advance should trigger a punctuate call and emit a trace with 2 spans
    td.advanceWallClockTime(Duration.ofSeconds(35));
    // trace1 should have 1 span i.e. span3
    trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(1, trace.getEventList().size());
    assertEquals("event-3", new String(trace.getEventList().get(0).getEventId().array()));
    // trace2 should have 1 span i.e. span4
    trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(1, trace.getEventList().size());
    assertEquals("event-5", new String(trace.getEventList().get(0).getEventId().array()));
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span6);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span7);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span8);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span9);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span10);
    inputTopic.pipeInput(createTraceIdentity(tenantId, "trace-3"), span11);
    td.advanceWallClockTime(Duration.ofSeconds(35));
    // trace should be truncated with 5 spans
    trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(5, trace.getEventList().size());
    // input 8 spans of trace-4 for tenant2, as there is global upper limit apply, it will emit only
    // 6
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span12);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span13);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span14);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span15);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span16);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span17);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span18);
    inputTopic.pipeInput(createTraceIdentity(tenant2, "trace-4"), span19);
    td.advanceWallClockTime(Duration.ofSeconds(35));
    trace = (StructuredTrace) outputTopic.readValue();
    assertEquals(6, trace.getEventList().size());
}

Example 9 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class EndpointEnricherTest method getBigTrace.

protected StructuredTrace getBigTrace() {
    /*
     * The graph looks like
     *             0 (entry)
     *             |
     *             1 (intermediate)
     * ------------|-------------------
     *             2 (exit)
     */
    // 0th raw span
    Map<String, AttributeValue> entrySpan0Map = new HashMap<>();
    entrySpan0Map.put(EnrichedSpanConstants.getValue(Http.HTTP_REQUEST_URL), AttributeValue.newBuilder().setValue("http://someservice.ai/users/1/checkout").build());
    Map<String, AttributeValue> enrichedEntrySpan0Map = new HashMap<>();
    enrichedEntrySpan0Map.put(EnrichedSpanConstants.getValue(Api.API_BOUNDARY_TYPE), AttributeValue.newBuilder().setValue("ENTRY").build());
    enrichedEntrySpan0Map.put(EntityConstants.getValue(ServiceAttribute.SERVICE_ATTRIBUTE_ID), AttributeValue.newBuilder().setValue(SERVICE_ID).build());
    enrichedEntrySpan0Map.put(EntityConstants.getValue(ServiceAttribute.SERVICE_ATTRIBUTE_NAME), AttributeValue.newBuilder().setValue(SERVICE_NAME).build());
    enrichedEntrySpan0Map.put(EntityConstants.getValue(ApiAttribute.API_ATTRIBUTE_ID), AttributeValue.newBuilder().setValue(API_ID).build());
    enrichedEntrySpan0Map.put(EntityConstants.getValue(ApiAttribute.API_ATTRIBUTE_URL_PATTERN), AttributeValue.newBuilder().setValue(API_PATTERN_VAL).build());
    enrichedEntrySpan0Map.put(EntityConstants.getValue(ApiAttribute.API_ATTRIBUTE_NAME), AttributeValue.newBuilder().setValue(API_NAME_VAL).build());
    Event event0 = Event.newBuilder().setCustomerId(TENANT_ID).setAttributes(Attributes.newBuilder().setAttributeMap(entrySpan0Map).build()).setEnrichedAttributes(Attributes.newBuilder().setAttributeMap(enrichedEntrySpan0Map).build()).setEventId(createByteBuffer("event0")).setEventName(API_PATTERN_VAL).build();
    RawSpan rawSpan0 = RawSpan.newBuilder().setCustomerId(TENANT_ID).setEvent(event0).setTraceId(createByteBuffer("trace")).build();
    // 1st intermediate raw span
    EventRef eventRef0 = EventRef.newBuilder().setRefType(EventRefType.CHILD_OF).setTraceId(createByteBuffer("trace")).setEventId(createByteBuffer("event0")).build();
    Map<String, AttributeValue> exitSpanMap = new HashMap<>();
    Map<String, AttributeValue> enrichedExitSpanMap = new HashMap<>();
    Event event1 = Event.newBuilder().setCustomerId(TENANT_ID).setEventId(createByteBuffer("event1")).setAttributes(Attributes.newBuilder().setAttributeMap(exitSpanMap).build()).setEnrichedAttributes(Attributes.newBuilder().setAttributeMap(enrichedExitSpanMap).build()).setEventRefList(Collections.singletonList(eventRef0)).build();
    RawSpan rawSpan1 = RawSpan.newBuilder().setCustomerId(TENANT_ID).setEvent(event1).setTraceId(createByteBuffer("trace")).build();
    // 2nd exit raw span
    EventRef eventRef1 = EventRef.newBuilder().setRefType(EventRefType.CHILD_OF).setTraceId(createByteBuffer("trace")).setEventId(createByteBuffer("event1")).build();
    Map<String, AttributeValue> entrySpan2Map = new HashMap<>();
    entrySpan2Map.put(EnrichedSpanConstants.getValue(Http.HTTP_REQUEST_URL), AttributeValue.newBuilder().setValue("http://nextservice.ai/login").build());
    Map<String, AttributeValue> enrichedEntrySpan2Map = new HashMap<>();
    enrichedEntrySpan2Map.put(EnrichedSpanConstants.getValue(Api.API_BOUNDARY_TYPE), AttributeValue.newBuilder().setValue("EXIT").build());
    Event event2 = Event.newBuilder().setCustomerId(TENANT_ID).setAttributes(Attributes.newBuilder().setAttributeMap(entrySpan2Map).build()).setEnrichedAttributes(Attributes.newBuilder().setAttributeMap(enrichedEntrySpan2Map).build()).setEventId(createByteBuffer("event2")).setEventRefList(Collections.singletonList(eventRef1)).build();
    RawSpan rawSpan2 = RawSpan.newBuilder().setCustomerId(TENANT_ID).setEvent(event2).setTraceId(createByteBuffer("trace")).build();
    return StructuredTraceBuilder.buildStructuredTraceFromRawSpans(List.of(rawSpan0, rawSpan1, rawSpan2), createByteBuffer("trace"), TENANT_ID);
}

Example 10 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class MigrationTestRpc method testGetGrpcAuthority.

@Test
public void testGetGrpcAuthority() throws Exception {
    String rpcRequestMetadataAuthorityValue = "grpc authority";
    Map<String, String> tagsMap = Map.of(RPC_REQUEST_METADATA_AUTHORITY.getValue(), rpcRequestMetadataAuthorityValue);
    Span span = createSpanFromTags(tagsMap);
    RawSpan rawSpan = normalizer.convert("tenant-key", span, buildEvent("tenant-key", span, Optional.empty()));
    assertNull(rawSpan.getEvent().getGrpc());
    assertTrue(RpcSemanticConventionUtils.getGrpcAuthority(rawSpan.getEvent()).isEmpty());
    tagsMap = Map.of(RPC_REQUEST_METADATA_AUTHORITY.getValue(), rpcRequestMetadataAuthorityValue, OTEL_SPAN_TAG_RPC_SYSTEM.getValue(), OTEL_RPC_SYSTEM_GRPC.getValue());
    span = createSpanFromTags(tagsMap);
    rawSpan = normalizer.convert("tenant-key", span, buildEvent("tenant-key", span, Optional.empty()));
    // now, we are not populating first class fields. So, it should be null.
    assertNull(rawSpan.getEvent().getGrpc());
    assertEquals(rpcRequestMetadataAuthorityValue, RpcSemanticConventionUtils.getGrpcAuthority(rawSpan.getEvent()).get());
}