Examples with LogEvents org.hypertrace.core.datamodel.LogEvents used on opensource projects

Example 1 with LogEvents

use of org.hypertrace.core.datamodel.LogEvents in project hypertrace-ingester by hypertrace.

the class JaegerSpanToLogRecordsTransformerTest method testDropLogEventRecords.

@Test
void testDropLogEventRecords() {
    Map<String, Object> configs = new HashMap<>();
    configs.putAll(Map.of("processor", Map.of("tenantIdTagKey", "tenant-key", "excludeLogsTenantIds", List.of("tenant-1"))));
    ProcessorContext processorContext = Mockito.mock(ProcessorContext.class);
    Mockito.when(processorContext.appConfigs()).thenReturn(Map.of("span-normalizer-job-config", ConfigFactory.parseMap(configs)));
    JaegerSpanToLogRecordsTransformer jaegerSpanToLogRecordsTransformer = new JaegerSpanToLogRecordsTransformer();
    jaegerSpanToLogRecordsTransformer.init(processorContext);
    KeyValue<String, LogEvents> keyValue = jaegerSpanToLogRecordsTransformer.transform(null, new PreProcessedSpan("tenant-1", getTestSpan(), buildEvent("tenant-1", getTestSpan(), Optional.of("tenant-key"))));
    Assertions.assertNull(keyValue);
}

Example 2 with LogEvents

use of org.hypertrace.core.datamodel.LogEvents in project hypertrace-ingester by hypertrace.

the class SpanNormalizerTest method whenByPassedExpectStructuredTraceToBeOutput.

@Test
@SetEnvironmentVariable(key = "SERVICE_NAME", value = "span-normalizer")
public void whenByPassedExpectStructuredTraceToBeOutput() {
    Config config = ConfigFactory.parseURL(getClass().getClassLoader().getResource("configs/span-normalizer/application.conf"));
    Map<String, Object> mergedProps = new HashMap<>();
    underTest.getBaseStreamsConfig().forEach(mergedProps::put);
    underTest.getStreamsConfig(config).forEach(mergedProps::put);
    mergedProps.put(SpanNormalizerConstants.SPAN_NORMALIZER_JOB_CONFIG, config);
    StreamsBuilder streamsBuilder = underTest.buildTopology(mergedProps, new StreamsBuilder(), new HashMap<>());
    Properties props = new Properties();
    mergedProps.forEach(props::put);
    TopologyTestDriver td = new TopologyTestDriver(streamsBuilder.build(), props);
    TestInputTopic<byte[], Span> inputTopic = td.createInputTopic(config.getString(SpanNormalizerConstants.INPUT_TOPIC_CONFIG_KEY), Serdes.ByteArray().serializer(), new JaegerSpanSerde().serializer());
    Serde<RawSpan> rawSpanSerde = new AvroSerde<>();
    rawSpanSerde.configure(Map.of(), false);
    Serde<StructuredTrace> structuredTraceSerde = new AvroSerde<>();
    structuredTraceSerde.configure(Map.of(), false);
    Serde<TraceIdentity> spanIdentitySerde = new AvroSerde<>();
    spanIdentitySerde.configure(Map.of(), true);
    TestOutputTopic outputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_CONFIG_KEY), spanIdentitySerde.deserializer(), rawSpanSerde.deserializer());
    TestOutputTopic bypassOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.BYPASS_OUTPUT_TOPIC_CONFIG_KEY), Serdes.String().deserializer(), structuredTraceSerde.deserializer());
    TestOutputTopic rawLogOutputTopic = td.createOutputTopic(config.getString(SpanNormalizerConstants.OUTPUT_TOPIC_RAW_LOGS_CONFIG_KEY), spanIdentitySerde.deserializer(), new AvroSerde<>().deserializer());
    // with logs event, with bypass key
    // expects no output to raw-span-grouper
    // expects output to trace-enricher
    // expects log output
    Span span1 = Span.newBuilder().setSpanId(ByteString.copyFrom("1".getBytes())).setTraceId(ByteString.copyFrom("trace-1".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("true").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span1);
    // validate output for trace-enricher
    assertFalse(bypassOutputTopic.isEmpty());
    KeyValue<String, StructuredTrace> kv1 = bypassOutputTopic.readKeyValue();
    assertEquals("__default", kv1.value.getCustomerId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-1".getBytes()).toByteArray()), HexUtils.getHex(kv1.value.getTraceId().array()));
    // validate no output for raw-spans-grouper
    assertTrue(outputTopic.isEmpty());
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    LogEvents logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, without bypass key
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span2 = Span.newBuilder().setSpanId(ByteString.copyFrom("2".getBytes())).setTraceId(ByteString.copyFrom("trace-2".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span2);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv2 = outputTopic.readKeyValue();
    assertEquals("__default", kv2.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-2".getBytes()).toByteArray()), HexUtils.getHex(kv2.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
    // with logs event, with bypass key but false value
    // expects output to raw-span-grouper
    // expects no output to trace-enricher
    // expects log output
    Span span3 = Span.newBuilder().setSpanId(ByteString.copyFrom("3".getBytes())).setTraceId(ByteString.copyFrom("trace-3".getBytes())).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("jaeger.servicename").setVStr(SERVICE_NAME).build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("http.method").setVStr("GET").build()).addTags(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("test.bypass").setVStr("false").build()).addLogs(Log.newBuilder().setTimestamp(Timestamp.newBuilder().setSeconds(10).build()).addFields(JaegerSpanInternalModel.KeyValue.newBuilder().setKey("z2").setVStr("some event detail").build())).build();
    inputTopic.pipeInput(span3);
    // validate that no output to trace-enricher
    assertTrue(bypassOutputTopic.isEmpty());
    // validate that output to raw-spans-grouper
    assertFalse(outputTopic.isEmpty());
    KeyValue<TraceIdentity, RawSpan> kv3 = outputTopic.readKeyValue();
    assertEquals("__default", kv3.key.getTenantId());
    assertEquals(HexUtils.getHex(ByteString.copyFrom("trace-3".getBytes()).toByteArray()), HexUtils.getHex(kv3.key.getTraceId().array()));
    // validate that no change in log traffic
    assertFalse(rawLogOutputTopic.isEmpty());
    logEvents = (LogEvents) rawLogOutputTopic.readKeyValue().value;
    Assertions.assertEquals(1, logEvents.getLogEvents().size());
}

Example 3 with LogEvents

use of org.hypertrace.core.datamodel.LogEvents in project hypertrace-ingester by hypertrace.

the class JaegerSpanToLogRecordsTransformer method transform.

@Override
public KeyValue<String, LogEvents> transform(byte[] key, PreProcessedSpan preProcessedSpan) {
    try {
        Span value = preProcessedSpan.getSpan();
        String tenantId = preProcessedSpan.getTenantId();
        if (value.getLogsCount() == 0 || tenantIdsToExclude.contains(tenantId)) {
            return null;
        }
        tenantToSpanWithLogsReceivedCount.computeIfAbsent(tenantId, tenant -> PlatformMetricsRegistry.registerCounter(VALID_SPAN_WITH_LOGS_RECEIVED_COUNT, Map.of("tenantId", tenantId))).increment();
        return new KeyValue<>(null, buildLogEventRecords(value, tenantId));
    } catch (Exception e) {
        LOG.error("Exception processing log records", e);
        return null;
    }
}

Example 4 with LogEvents

use of org.hypertrace.core.datamodel.LogEvents in project hypertrace-ingester by hypertrace.

the class LogEventViewGenerator method process.

@Override
public List<LogEventView> process(LogEvents logEvents) {
    try {
        List<LogEventView> list = new ArrayList<>();
        for (LogEvent logEventRecord : logEvents.getLogEvents()) {
            String attributes = convertAttributes(logEventRecord.getAttributes());
            LogEventView logEventView = fastNewBuilder(LogEventView.Builder.class).setSpanId(logEventRecord.getSpanId()).setTraceId(logEventRecord.getTraceId()).setTimestampNanos(logEventRecord.getTimestampNanos()).setTenantId(logEventRecord.getTenantId()).setAttributes(attributes).setSummary(getSummary(logEventRecord.getAttributes())).build();
            if (!StringUtils.isEmpty(logEventRecord.getTenantId()) && null != attributes) {
                logEventAttributeSizeGauge.computeIfAbsent(logEventRecord.getTenantId(), v -> PlatformMetricsRegistry.registerGauge(LOG_EVENT_ATTRIBUTE_SIZE_METRIC, Map.of("tenantId", logEventRecord.getTenantId()), new AtomicInteger(0))).set(attributes.length());
            }
            list.add(logEventView);
        }
        return list;
    } catch (Exception e) {
        LOG.error("Exception processing log records", e);
        return null;
    }
}

Example 5 with LogEvents

use of org.hypertrace.core.datamodel.LogEvents in project hypertrace-ingester by hypertrace.

the class LogEventViewGeneratorTest method testSummaryField.

@Test
void testSummaryField() {
    LogEventViewGenerator logEventViewGenerator = new LogEventViewGenerator();
    Map<String, String> attributes = new HashMap<>();
    List<String> summaryKeys = new ArrayList<>(LogEventViewGenerator.SUMMARY_KEYS);
    AtomicInteger attributeVal = new AtomicInteger();
    summaryKeys.forEach(key -> attributes.put(key, String.valueOf(attributeVal.getAndIncrement())));
    for (String summaryKey : LogEventViewGenerator.SUMMARY_KEYS) {
        LogEvents logEvents = getLogEventsWithAttribute(attributes);
        List<LogEventView> list = logEventViewGenerator.process(logEvents);
        assertEquals(attributes.get(summaryKey), list.get(0).getSummary());
        attributes.remove(summaryKey);
    }
}