Examples with JaegerSpanSerde org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde used on opensource projects

Example 1 with JaegerSpanSerde

use of org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method testLaterArrivalJaegerSpans.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void testLaterArrivalJaegerSpans() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // case 1: within threshold, expect output
    Instant instant = Instant.now();
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).build();
    inputTopic.pipeInput(span);
    KeyValue<TraceIdentity, RawSpan> kv = outputTopic.readKeyValue();
    assertEquals("__default", kv.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv.key.getTraceId().array()));
    RawSpan value = kv.value;
    assertEquals(HexUtils.getHex("1".getBytes()), HexUtils.getHex((value).getEvent().getEventId()));
    assertEquals(SERVICE_NAME, value.getEvent().getServiceName());
    // outside threshold, except no output to RawSpan
    Instant instant1 = Instant.now().minus(25, ChronoUnit.HOURS);
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant1.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).build();
    inputTopic.pipeInput(span2);
    Assertions.assertTrue(outputTopic.isEmpty());
}

Example 2 with JaegerSpanSerde

use of org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenByPassedExpectStructuredTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenByPassedExpectStructuredTraceToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<StructuredTrace> structuredTraceSerde = new AvroSerde<>();
    structuredTraceSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic bypassOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.BYPASS_OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), structuredTraceSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // with logs event, with bypass key
    // expects no output to raw-span-grouper
    // expects output to trace-enricher
    // expects log output
    Span span1 = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("true").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span1);
    // validate output for trace-enricher
    assertFalse(bypassOutputTopic.isEmpty());
    KeyValue<String, StructuredTrace> kv1 = bypassOutputTopic.readKeyValue();
    assertEquals("__default", kv1.value.getCustomerId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv1.value.getTraceId().array()));
    // validate no output for raw-spans-grouper
    assertTrue(outputTopic.isEmpty());
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    LogEvents logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, without bypass key
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span2);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv2 = outputTopic.readKeyValue();
    assertEquals("__default", kv2.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv2.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, with bypass key but false value
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("false").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span3);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv3 = outputTopic.readKeyValue();
    assertEquals("__default", kv3.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-3".getBytes()).toByteArray()), HexUtils.getHex(kv3.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
}

Example 3 with JaegerSpanSerde

use of org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizer method buildTopology.

@Override
public StreamsBuilder buildTopology(Map<String, Object> streamsProperties, StreamsBuilder streamsBuilder, Map<String, KStream<?, ?>> inputStreams) {
    Config jobConfig = getJobConfig(streamsProperties);
    String inputTopic = jobConfig.getString(INPUT_TOPIC_CONFIG_KEY);
    String outputTopic = jobConfig.getString(OUTPUT_TOPIC_CONFIG_KEY);
    String bypassOutputTopic = jobConfig.getString(BYPASS_OUTPUT_TOPIC_CONFIG_KEY);
    String outputTopicRawLogs = jobConfig.getString(OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY);
    KStream<byte[], Span> inputStream = (KStream<byte[], Span>) inputStreams.get(inputTopic);
    if (inputStream == null) {
        inputStream = streamsBuilder.stream(inputTopic, Consumed.with(Serdes.ByteArray(), new JaegerSpanSerde()));
        inputStreams.put(inputTopic, inputStream);
    }
    KStream<byte[], PreProcessedSpan> preProcessedStream = inputStream.transform(JaegerSpanPreProcessor::new);
    KStream<TraceIdentity, RawSpan>[] branches = preProcessedStream.transform(JaegerSpanToAvroRawSpanTransformer::new).branch(new ByPassPredicate(jobConfig), (key, value) -> true);
    branches[0].transform(RawSpanToStructuredTraceTransformer::new).to(bypassOutputTopic);
    branches[1].to(outputTopic);
    preProcessedStream.transform(JaegerSpanToLogRecordsTransformer::new).to(outputTopicRawLogs);
    return streamsBuilder;
}

Example 4 with JaegerSpanSerde

use of org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde in project hypertrace-ingester by hypertrace.

the class HypertraceIngesterTest method testIngestionPacketFlow.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "hypertrace-ingester")
public void testIngestionPacketFlow(@TempDir Path tempDir) {
    File file = tempDir.resolve("state").toFile();
    Map<String, Object> baseProps = underTest.getBaseStreamsConfig();
    Map<String, Object> streamsProps = underTest.getStreamsConfig(underTestConfig);
    streamsProps.forEach(baseProps::put);
    Map<String, Object> mergedProps = baseProps;
    mergedProps.put(underTest.getJobConfigKey(), underTestConfig);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    mergedProps.put(StreamsConfig.STATE_DIR_CONFIG, file.getAbsolutePath());
    // create topology test driver for ingester
    TopologyTestDriver topologyTestDriver = new TopologyTestDriver(streamsBuilder.build(), props);
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).build();
    TestInputTopic<byte[], Span> spanNormalizerInputTopic = topologyTestDriver.createInputTopic(spanNormalizerConfig.getString(org.hypertrace.core.spannormalizer.constants.SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    spanNormalizerInputTopic.pipeInput(span);
    // create output topic for span-normalizer topology
    TestOutputTopic spanNormalizerOutputTopic = topologyTestDriver.createOutputTopic(spanNormalizerConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(spanNormalizerOutputTopic.readKeyValue());
    topologyTestDriver.advanceWallClockTime(Duration.ofSeconds(32));
    // create output topic for span-grouper topology
    TestOutputTopic spanGrouperOutputTopic = topologyTestDriver.createOutputTopic(rawSpansGrouperConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(spanGrouperOutputTopic.readKeyValue());
    // create output topic for trace-enricher topology
    TestOutputTopic traceEnricherOutputTopic = topologyTestDriver.createOutputTopic(traceEnricherConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(traceEnricherOutputTopic.readKeyValue());
    // create output topic for  topology
    TestOutputTopic spanEventViewOutputTopic = topologyTestDriver.createOutputTopic(spanEventViewGeneratorConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    SpanEventView spanEventView = (SpanEventView) spanEventViewOutputTopic.readValue();
    assertEquals(HexUtils.getHex(spanEventView.getSpanId()), HexUtils.getHex(span.getSpanId().toByteArray()));
    assertEquals(HexUtils.getHex(spanEventView.getTraceId()), HexUtils.getHex(span.getTraceId().toByteArray()));
}

Example 5 with JaegerSpanSerde

use of org.hypertrace.core.spannormalizer.jaeger.JaegerSpanSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenJaegerSpansAreProcessedExpectRawSpansToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenJaegerSpansAreProcessedExpectRawSpansToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(5).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("e1").setVStr("some event detail").build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("e2").setVStr("some event detail").build())).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span);
    KeyValue<TraceIdentity, RawSpan> kv = outputTopic.readKeyValue();
    assertEquals("__default", kv.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv.key.getTraceId().array()));
    RawSpan value = kv.value;
    assertEquals(HexUtils.getHex("1".getBytes()), HexUtils.getHex((value).getEvent().getEventId()));
    assertEquals(SERVICE_NAME, value.getEvent().getServiceName());
    KeyValue<String, LogEvents> keyValue = rawLogOutputTopic.readKeyValue();
    LogEvents logEvents = keyValue.value;
    Assertions.assertEquals(2, logEvents.getLogEvents().size());
    // pipe in one more span which doesn't match spanDropFilters
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).build();
    inputTopic.pipeInput(span2);
    KeyValue<TraceIdentity, RawSpan> kv1 = outputTopic.readKeyValue();
    assertNotNull(kv1);
    assertEquals("__default", kv1.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv1.key.getTraceId().array()));
    // pipe in one more span which match one of spanDropFilters (http.method & http.url)
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.url").setVStr("http://xyz.com/health/check").build()).build();
    inputTopic.pipeInput(span3);
    assertTrue(outputTopic.isEmpty());
    // pipe in one more span which match one of spanDropFilters (grpc.url)
    Span span4 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("grpc.url").setVStr("doesn't match with input filter set").build()).build();
    inputTopic.pipeInput(span4);
    assertTrue(outputTopic.isEmpty());
}