Examples with RawSpan org.hypertrace.core.datamodel.RawSpan used on opensource projects

Example 1 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method testLaterArrivalJaegerSpans.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void testLaterArrivalJaegerSpans() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // case 1: within threshold, expect output
    Instant instant = Instant.now();
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).build();
    inputTopic.pipeInput(span);
    KeyValue<TraceIdentity, RawSpan> kv = outputTopic.readKeyValue();
    assertEquals("__default", kv.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv.key.getTraceId().array()));
    RawSpan value = kv.value;
    assertEquals(HexUtils.getHex("1".getBytes()), HexUtils.getHex((value).getEvent().getEventId()));
    assertEquals(SERVICE_NAME, value.getEvent().getServiceName());
    // outside threshold, except no output to RawSpan
    Instant instant1 = Instant.now().minus(25, ChronoUnit.HOURS);
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant1.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).build();
    inputTopic.pipeInput(span2);
    Assertions.assertTrue(outputTopic.isEmpty());
}

Example 2 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenByPassedExpectStructuredTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenByPassedExpectStructuredTraceToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<StructuredTrace> structuredTraceSerde = new AvroSerde<>();
    structuredTraceSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic bypassOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.BYPASS_OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), structuredTraceSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // with logs event, with bypass key
    // expects no output to raw-span-grouper
    // expects output to trace-enricher
    // expects log output
    Span span1 = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("true").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span1);
    // validate output for trace-enricher
    assertFalse(bypassOutputTopic.isEmpty());
    KeyValue<String, StructuredTrace> kv1 = bypassOutputTopic.readKeyValue();
    assertEquals("__default", kv1.value.getCustomerId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv1.value.getTraceId().array()));
    // validate no output for raw-spans-grouper
    assertTrue(outputTopic.isEmpty());
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    LogEvents logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, without bypass key
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span2);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv2 = outputTopic.readKeyValue();
    assertEquals("__default", kv2.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv2.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, with bypass key but false value
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("false").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span3);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv3 = outputTopic.readKeyValue();
    assertEquals("__default", kv3.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-3".getBytes()).toByteArray()), HexUtils.getHex(kv3.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
}

Example 3 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class JaegerSpanNormalizerTest method testServiceNameNotAddedToEvent.

@Test
public void testServiceNameNotAddedToEvent() throws Exception {
    String tenantId = "tenant-" + random.nextLong();
    Map<String, Object> configs = new HashMap<>(getCommonConfig());
    configs.putAll(Map.of("processor", Map.of("defaultTenantId", tenantId)));
    JaegerSpanNormalizer normalizer = JaegerSpanNormalizer.get(ConfigFactory.parseMap(configs));
    Process process = Process.newBuilder().build();
    Span span = Span.newBuilder().setProcess(process).build();
    RawSpan rawSpan = normalizer.convert(tenantId, span, buildEvent(tenantId, span, Optional.empty()));
    Assertions.assertNull(rawSpan.getEvent().getServiceName());
    Assertions.assertNull(rawSpan.getEvent().getAttributes().getAttributeMap().get(RawSpanConstants.getValue(JaegerAttribute.JAEGER_ATTRIBUTE_SERVICE_NAME)));
    Timer timer = normalizer.getSpanNormalizationTimer(tenantId);
    Assertions.assertNotNull(timer);
    // Assert that metrics are collected.
    Assertions.assertEquals(1, timer.count());
}

Example 4 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class SpanNormalizer method buildTopology.

@Override
public StreamsBuilder buildTopology(Map<String, Object> streamsProperties, StreamsBuilder streamsBuilder, Map<String, KStream<?, ?>> inputStreams) {
    Config jobConfig = getJobConfig(streamsProperties);
    String inputTopic = jobConfig.getString(INPUT_TOPIC_CONFIG_KEY);
    String outputTopic = jobConfig.getString(OUTPUT_TOPIC_CONFIG_KEY);
    String bypassOutputTopic = jobConfig.getString(BYPASS_OUTPUT_TOPIC_CONFIG_KEY);
    String outputTopicRawLogs = jobConfig.getString(OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY);
    KStream<byte[], Span> inputStream = (KStream<byte[], Span>) inputStreams.get(inputTopic);
    if (inputStream == null) {
        inputStream = streamsBuilder.stream(inputTopic, Consumed.with(Serdes.ByteArray(), new JaegerSpanSerde()));
        inputStreams.put(inputTopic, inputStream);
    }
    KStream<byte[], PreProcessedSpan> preProcessedStream = inputStream.transform(JaegerSpanPreProcessor::new);
    KStream<TraceIdentity, RawSpan>[] branches = preProcessedStream.transform(JaegerSpanToAvroRawSpanTransformer::new).branch(new ByPassPredicate(jobConfig), (key, value) -> true);
    branches[0].transform(RawSpanToStructuredTraceTransformer::new).to(bypassOutputTopic);
    branches[1].to(outputTopic);
    preProcessedStream.transform(JaegerSpanToLogRecordsTransformer::new).to(outputTopicRawLogs);
    return streamsBuilder;
}

Example 5 with RawSpan

use of org.hypertrace.core.datamodel.RawSpan in project hypertrace-ingester by hypertrace.

the class RawSpansGrouper method buildTopology.

public StreamsBuilder buildTopology(Map<String, Object> properties, StreamsBuilder streamsBuilder, Map<String, KStream<?, ?>> inputStreams) {
    Config jobConfig = getJobConfig(properties);
    String inputTopic = jobConfig.getString(INPUT_TOPIC_CONFIG_KEY);
    String outputTopic = jobConfig.getString(OUTPUT_TOPIC_CONFIG_KEY);
    KStream<TraceIdentity, RawSpan> inputStream = (KStream<TraceIdentity, RawSpan>) inputStreams.get(inputTopic);
    if (inputStream == null) {
        inputStream = streamsBuilder.stream(inputTopic);
        inputStreams.put(inputTopic, inputStream);
    }
    // Retrieve the default value serde defined in config and use it
    Serde valueSerde = defaultValueSerde(properties);
    Serde keySerde = defaultKeySerde(properties);
    StoreBuilder<KeyValueStore<TraceIdentity, TraceState>> traceStateStoreBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(TRACE_STATE_STORE), keySerde, valueSerde).withCachingEnabled();
    StoreBuilder<KeyValueStore<SpanIdentity, RawSpan>> spanStoreBuilder = Stores.keyValueStoreBuilder(Stores.persistentKeyValueStore(SPAN_STATE_STORE_NAME), keySerde, valueSerde).withCachingEnabled();
    streamsBuilder.addStateStore(spanStoreBuilder);
    streamsBuilder.addStateStore(traceStateStoreBuilder);
    Produced<String, StructuredTrace> outputTopicProducer = Produced.with(Serdes.String(), null);
    outputTopicProducer = outputTopicProducer.withName(OUTPUT_TOPIC_PRODUCER);
    inputStream.transform(RawSpansProcessor::new, Named.as(RawSpansProcessor.class.getSimpleName()), SPAN_STATE_STORE_NAME, TRACE_STATE_STORE).to(outputTopic, outputTopicProducer);
    return streamsBuilder;
}