Examples with StructuredTrace org.hypertrace.core.datamodel.StructuredTrace used on opensource projects

Example 6 with StructuredTrace

use of org.hypertrace.core.datamodel.StructuredTrace in project hypertrace-ingester by hypertrace.

the class StructuredTraceGraphBuilderTest method testBuildGraph.

@Test
void testBuildGraph() {
    Entity entity = mock(Entity.class);
    Event parent = mock(Event.class);
    Event child = mock(Event.class);
    Edge eventEdge = mock(Edge.class);
    StructuredTrace underTestTrace = mock(StructuredTrace.class);
    when(underTestTrace.getCustomerId()).thenReturn("__defaultTenant");
    when(underTestTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes()));
    when(underTestTrace.getEntityList()).thenReturn(List.of(entity));
    when(underTestTrace.getEventList()).thenReturn(List.of(parent, child));
    when(underTestTrace.getEntityEdgeList()).thenReturn(List.of());
    when(underTestTrace.getEntityEventEdgeList()).thenReturn(List.of());
    when(underTestTrace.getEventEdgeList()).thenReturn(List.of(eventEdge));
    try (MockedStatic<StructuredTrace> builderMockedStatic = mockStatic(StructuredTrace.class)) {
        StructuredTrace.Builder builder = mock(StructuredTrace.Builder.class);
        when(builder.build()).thenReturn(underTestTrace);
        builderMockedStatic.when(() -> StructuredTrace.newBuilder(underTestTrace)).thenReturn(builder);
        try (MockedConstruction<StructuredTraceGraph> mocked = mockConstruction(StructuredTraceGraph.class)) {
            // calls first time
            StructuredTraceGraph actual = StructuredTraceGraphBuilder.buildGraph(underTestTrace);
            // calls second time, this time it returns from cache
            StructuredTraceGraph cached = StructuredTraceGraphBuilder.buildGraph(underTestTrace);
            Assertions.assertEquals(actual, cached);
        }
    }
}

Example 7 with StructuredTrace

use of org.hypertrace.core.datamodel.StructuredTrace in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenByPassedExpectStructuredTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenByPassedExpectStructuredTraceToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<StructuredTrace> structuredTraceSerde = new AvroSerde<>();
    structuredTraceSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic bypassOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.BYPASS_OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), structuredTraceSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // with logs event, with bypass key
    // expects no output to raw-span-grouper
    // expects output to trace-enricher
    // expects log output
    Span span1 = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("true").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span1);
    // validate output for trace-enricher
    assertFalse(bypassOutputTopic.isEmpty());
    KeyValue<String, StructuredTrace> kv1 = bypassOutputTopic.readKeyValue();
    assertEquals("__default", kv1.value.getCustomerId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv1.value.getTraceId().array()));
    // validate no output for raw-spans-grouper
    assertTrue(outputTopic.isEmpty());
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    LogEvents logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, without bypass key
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span2);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv2 = outputTopic.readKeyValue();
    assertEquals("__default", kv2.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv2.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, with bypass key but false value
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("false").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span3);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv3 = outputTopic.readKeyValue();
    assertEquals("__default", kv3.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-3".getBytes()).toByteArray()), HexUtils.getHex(kv3.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
}

Example 8 with StructuredTrace

use of org.hypertrace.core.datamodel.StructuredTrace in project hypertrace-ingester by hypertrace.

the class ViewGeneratorStateTest method testGetTraceState.

@Test
public void testGetTraceState() {
    StructuredTrace trace = getTestTrace(customerId, traceId1);
    TraceState traceState = ViewGeneratorState.getTraceState(trace);
    assertNotNull(traceState);
    assertEquals(trace, traceState.getTrace());
    TraceState sameTraceState = ViewGeneratorState.getTraceState(trace);
    assertEquals(sameTraceState, traceState);
    StructuredTrace modifiedTrace = getTestTrace(customerId, traceId1);
    modifiedTrace.setEntityList(Arrays.asList(Entity.newBuilder().setCustomerId(customerId).setEntityId("entity-2").setEntityName("entity-2").setEntityType("service").build()));
    // same trace id but different object should result in rebuilding of trace state
    TraceState differentTraceState1 = ViewGeneratorState.getTraceState(modifiedTrace);
    assertNotEquals(traceState, differentTraceState1);
    StructuredTrace differentTrace = getTestTrace(customerId, traceId2);
    TraceState differentTraceState2 = ViewGeneratorState.getTraceState(differentTrace);
    assertEquals(differentTrace, differentTraceState2.getTrace());
}

Example 9 with StructuredTrace

use of org.hypertrace.core.datamodel.StructuredTrace in project hypertrace-ingester by hypertrace.

the class RawSpansGrouper method buildTopology.

public StreamsBuilder buildTopology(Map<String, Object> properties, StreamsBuilder streamsBuilder, Map<String, KStream<?, ?>> inputStreams) {
    Config jobConfig = getJobConfig(properties);
    String inputTopic = jobConfig.getString(INPUT_TOPIC_CONFIG_KEY);
    String outputTopic = jobConfig.getString(OUTPUT_TOPIC_CONFIG_KEY);
    KStream<TraceIdentity, RawSpan> inputStream = (KStream<TraceIdentity, RawSpan>) inputStreams.get(inputTopic);
    if (inputStream == null) {
        inputStream = streamsBuilder.stream(inputTopic);
        inputStreams.put(inputTopic, inputStream);
    }
    // Retrieve the default value serde defined in config and use it
    Serde valueSerde = defaultValueSerde(properties);
    Serde keySerde = defaultKeySerde(properties);
    StoreBuilder<KeyValueStore<TraceIdentity, TraceState>> traceStateStoreBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(TRACE_STATE_STORE), keySerde, valueSerde).withCachingEnabled();
    StoreBuilder<KeyValueStore<SpanIdentity, RawSpan>> spanStoreBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(SPAN_STATE_STORE_NAME), keySerde, valueSerde).withCachingEnabled();
    streamsBuilder.addStateStore(spanStoreBuilder);
    streamsBuilder.addStateStore(traceStateStoreBuilder);
    Produced<String, StructuredTrace> outputTopicProducer = Produced.with(Serdes.String(), null);
    outputTopicProducer = outputTopicProducer.withName(OUTPUT_TOPIC_PRODUCER);
    inputStream.transform(RawSpansProcessor::new, Named.as(RawSpansProcessor.class.getSimpleName()), SPAN_STATE_STORE_NAME, TRACE_STATE_STORE).to(outputTopic, outputTopicProducer);
    return streamsBuilder;
}

Example 10 with StructuredTrace

use of org.hypertrace.core.datamodel.StructuredTrace in project hypertrace-ingester by hypertrace.

the class RawSpansProcessor method transform.

public KeyValue<String, StructuredTrace> transform(TraceIdentity key, RawSpan value) {
    Instant start = Instant.now();
    long currentTimeMs = System.currentTimeMillis();
    TraceState traceState = traceStateStore.get(key);
    boolean firstEntry = (traceState == null);
    if (shouldDropSpan(key, traceState)) {
        return null;
    }
    String tenantId = key.getTenantId();
    ByteBuffer traceId = value.getTraceId();
    ByteBuffer spanId = value.getEvent().getEventId();
    spanStore.put(new SpanIdentity(tenantId, traceId, spanId), value);
    /*
     the trace emit ts is essentially currentTs + groupingWindowTimeoutMs
     i.e. if there is no span added in the next 'groupingWindowTimeoutMs' interval
     then the trace can be finalized and emitted
    */
    long traceEmitTs = currentTimeMs + groupingWindowTimeoutMs;
    if (logger.isDebugEnabled()) {
        logger.debug("Updating trigger_ts=[{}] for for tenant_id=[{}], trace_id=[{}]", Instant.ofEpochMilli(traceEmitTs), key.getTenantId(), HexUtils.getHex(traceId));
    }
    if (firstEntry) {
        traceState = fastNewBuilder(TraceState.Builder.class).setTraceStartTimestamp(currentTimeMs).setTraceEndTimestamp(currentTimeMs).setEmitTs(traceEmitTs).setTenantId(tenantId).setTraceId(traceId).setSpanIds(List.of(spanId)).build();
        schedulePunctuator(key);
    } else {
        traceState.getSpanIds().add(spanId);
        traceState.setTraceEndTimestamp(currentTimeMs);
        traceState.setEmitTs(traceEmitTs);
    }
    traceStateStore.put(key, traceState);
    tenantToSpansGroupingTimer.computeIfAbsent(value.getCustomerId(), k -> PlatformMetricsRegistry.registerTimer(PROCESSING_LATENCY_TIMER, Map.of("tenantId", k))).record(Duration.between(start, Instant.now()).toMillis(), TimeUnit.MILLISECONDS);
    // the punctuator will emit the trace
    return null;
}