Examples with AvroSerde org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde used on opensource projects

Example 1 with AvroSerde

use of org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method testLaterArrivalJaegerSpans.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void testLaterArrivalJaegerSpans() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // case 1: within threshold, expect output
    Instant instant = Instant.now();
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).build();
    inputTopic.pipeInput(span);
    KeyValue<TraceIdentity, RawSpan> kv = outputTopic.readKeyValue();
    assertEquals("__default", kv.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv.key.getTraceId().array()));
    RawSpan value = kv.value;
    assertEquals(HexUtils.getHex("1".getBytes()), HexUtils.getHex((value).getEvent().getEventId()));
    assertEquals(SERVICE_NAME, value.getEvent().getServiceName());
    // outside threshold, except no output to RawSpan
    Instant instant1 = Instant.now().minus(25, ChronoUnit.HOURS);
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).setStartTime(Timestamp.newBuilder().setSeconds(instant1.getEpochSecond()).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).build();
    inputTopic.pipeInput(span2);
    Assertions.assertTrue(outputTopic.isEmpty());
}

Example 2 with AvroSerde

use of org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenByPassedExpectStructuredTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenByPassedExpectStructuredTraceToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<StructuredTrace> structuredTraceSerde = new AvroSerde<>();
    structuredTraceSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic bypassOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.BYPASS_OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), structuredTraceSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // with logs event, with bypass key
    // expects no output to raw-span-grouper
    // expects output to trace-enricher
    // expects log output
    Span span1 = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("true").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span1);
    // validate output for trace-enricher
    assertFalse(bypassOutputTopic.isEmpty());
    KeyValue<String, StructuredTrace> kv1 = bypassOutputTopic.readKeyValue();
    assertEquals("__default", kv1.value.getCustomerId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv1.value.getTraceId().array()));
    // validate no output for raw-spans-grouper
    assertTrue(outputTopic.isEmpty());
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    LogEvents logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, without bypass key
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span2);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv2 = outputTopic.readKeyValue();
    assertEquals("__default", kv2.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv2.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, with bypass key but false value
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("false").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span3);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv3 = outputTopic.readKeyValue();
    assertEquals("__default", kv3.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-3".getBytes()).toByteArray()), HexUtils.getHex(kv3.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
}

Example 3 with AvroSerde

use of org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde in project hypertrace-ingester by hypertrace.

the class TraceEmitPunctuatorTest method setUp.

@BeforeEach
public void setUp() {
    AvroSerde avroSerde = new AvroSerde();
    ProcessorContext context = mock(ProcessorContext.class);
    when(context.keySerde()).thenReturn(avroSerde);
    spanStore = mock(KeyValueStore.class);
    traceStateStore = mock(KeyValueStore.class);
    To outputTopicProducer = mock(To.class);
    underTest = new TraceEmitPunctuator(TraceIdentity.newBuilder().setTenantId("__default").setTraceId(ByteBuffer.wrap("trace-1".getBytes())).build(), context, spanStore, traceStateStore, outputTopicProducer, 100, -1);
    underTest.setCancellable(mock(Cancellable.class));
}

Example 4 with AvroSerde

use of org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde in project hypertrace-ingester by hypertrace.

the class HypertraceTraceEnricherTest method testTraceEnricherTopology.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "hypertrace-trace-enricher")
public void testTraceEnricherTopology() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/hypertrace-trace-enricher/application.conf")).resolve();
    Map<String, Object> baseProps = underTest.getBaseStreamsConfig();
    Map<String, Object> streamsProps = underTest.getStreamsConfig(config);
    streamsProps.forEach(baseProps::put);
    Map<String, Object> mergedProps = baseProps;
    mergedProps.put(underTest.getJobConfigKey(), config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    // create topology test driver for trace-enricher
    TopologyTestDriver topologyTestDriver = new TopologyTestDriver(streamsBuilder.build(), props);
    Serde<StructuredTrace> htStructuredTraceSerde = new AvroSerde<>();
    // create input topic for HT-model StructuredTrace
    TestInputTopic<String, StructuredTrace> inputTopic = topologyTestDriver.createInputTopic(config.getString(StructuredTraceEnricherConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.String().serializer(), htStructuredTraceSerde.serializer());
    // create output topic for closed-model StructuredTrace
    TestOutputTopic outputTopic = topologyTestDriver.createOutputTopic(config.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), htStructuredTraceSerde.deserializer());
    // create instance of HT-model StructuredTrace
    StructuredTrace htStructuredTrace = createHTStructuredTrace("customer1", "1234");
    // Write an input record into input topic
    inputTopic.pipeInput(htStructuredTrace);
    // Read the output record from output topic
    StructuredTrace structuredTrace = (StructuredTrace) outputTopic.readValue();
    Assertions.assertEquals(HexUtils.getHex("1234".getBytes()), HexUtils.getHex(structuredTrace.getTraceId()));
}

Example 5 with AvroSerde

use of org.hypertrace.core.kafkastreams.framework.serdes.AvroSerde in project hypertrace-ingester by hypertrace.

the class HypertraceIngesterTest method testIngestionPacketFlow.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "hypertrace-ingester")
public void testIngestionPacketFlow(@TempDir Path tempDir) {
    File file = tempDir.resolve("state").toFile();
    Map<String, Object> baseProps = underTest.getBaseStreamsConfig();
    Map<String, Object> streamsProps = underTest.getStreamsConfig(underTestConfig);
    streamsProps.forEach(baseProps::put);
    Map<String, Object> mergedProps = baseProps;
    mergedProps.put(underTest.getJobConfigKey(), underTestConfig);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    mergedProps.put(StreamsConfig.STATE_DIR_CONFIG, file.getAbsolutePath());
    // create topology test driver for ingester
    TopologyTestDriver topologyTestDriver = new TopologyTestDriver(streamsBuilder.build(), props);
    Span span = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).build();
    TestInputTopic<byte[], Span> spanNormalizerInputTopic = topologyTestDriver.createInputTopic(spanNormalizerConfig.getString(org.hypertrace.core.spannormalizer.constants.SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    spanNormalizerInputTopic.pipeInput(span);
    // create output topic for span-normalizer topology
    TestOutputTopic spanNormalizerOutputTopic = topologyTestDriver.createOutputTopic(spanNormalizerConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(spanNormalizerOutputTopic.readKeyValue());
    topologyTestDriver.advanceWallClockTime(Duration.ofSeconds(32));
    // create output topic for span-grouper topology
    TestOutputTopic spanGrouperOutputTopic = topologyTestDriver.createOutputTopic(rawSpansGrouperConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(spanGrouperOutputTopic.readKeyValue());
    // create output topic for trace-enricher topology
    TestOutputTopic traceEnricherOutputTopic = topologyTestDriver.createOutputTopic(traceEnricherConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    assertNotNull(traceEnricherOutputTopic.readKeyValue());
    // create output topic for  topology
    TestOutputTopic spanEventViewOutputTopic = topologyTestDriver.createOutputTopic(spanEventViewGeneratorConfig.getString(StructuredTraceEnricherConstants.OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), new AvroSerde<>().deserializer());
    SpanEventView spanEventView = (SpanEventView) spanEventViewOutputTopic.readValue();
    assertEquals(HexUtils.getHex(spanEventView.getSpanId()), HexUtils.getHex(span.getSpanId().toByteArray()));
    assertEquals(HexUtils.getHex(spanEventView.getTraceId()), HexUtils.getHex(span.getTraceId().toByteArray()));
}