Examples with ReportableEntityPreprocessor com.wavefront.agent.preprocessor.ReportableEntityPreprocessor used on opensource projects

Example 11 with ReportableEntityPreprocessor

use of com.wavefront.agent.preprocessor.ReportableEntityPreprocessor in project java by wavefrontHQ.

the class RawLogsIngesterPortUnificationHandler method processLine.

@VisibleForTesting
@Override
public void processLine(final ChannelHandlerContext ctx, @Nonnull String message, @Nullable DataFormat format) {
    received.inc();
    ReportableEntityPreprocessor preprocessor = preprocessorSupplier == null ? null : preprocessorSupplier.get();
    String processedMessage = preprocessor == null ? message : preprocessor.forPointLine().transform(message);
    if (preprocessor != null && !preprocessor.forPointLine().filter(message, null))
        return;
    logsIngester.ingestLog(new LogsMessage() {

        @Override
        public String getLogLine() {
            return processedMessage;
        }

        @Override
        public String hostOrDefault(String fallbackHost) {
            String hostname = hostnameResolver.apply(getRemoteAddress(ctx));
            return StringUtils.isBlank(hostname) ? fallbackHost : hostname;
        }
    });
}

Example 12 with ReportableEntityPreprocessor

use of com.wavefront.agent.preprocessor.ReportableEntityPreprocessor in project java by wavefrontHQ.

the class SpanUtils method handleSpanLogs.

/**
 * Handle spanLogs.
 *
 * @param message              encoded spanLogs data.
 * @param spanLogsDecoder      spanLogs decoder.
 * @param spanDecoder          span decoder.
 * @param handler              spanLogs handler.
 * @param preprocessorSupplier spanLogs preprocessor.
 * @param ctx                  channel handler context.
 * @param samplerFunc          span sampler.
 */
public static void handleSpanLogs(String message, ReportableEntityDecoder<JsonNode, SpanLogs> spanLogsDecoder, ReportableEntityDecoder<String, Span> spanDecoder, ReportableEntityHandler<SpanLogs, String> handler, @Nullable Supplier<ReportableEntityPreprocessor> preprocessorSupplier, @Nullable ChannelHandlerContext ctx, Function<Span, Boolean> samplerFunc) {
    List<SpanLogs> spanLogsOutput = new ArrayList<>(1);
    try {
        spanLogsDecoder.decode(JSON_PARSER.readTree(message), spanLogsOutput, "dummy");
    } catch (Exception e) {
        handler.reject(message, formatErrorMessage(message, e, ctx));
        return;
    }
    for (SpanLogs spanLogs : spanLogsOutput) {
        String spanMessage = spanLogs.getSpan();
        if (spanMessage == null) {
            // For backwards compatibility, report the span logs if span line data is not
            // included
            handler.report(spanLogs);
        } else {
            ReportableEntityPreprocessor preprocessor = preprocessorSupplier == null ? null : preprocessorSupplier.get();
            String[] spanMessageHolder = new String[1];
            // transform the line if needed
            if (preprocessor != null) {
                spanMessage = preprocessor.forPointLine().transform(spanMessage);
                if (!preprocessor.forPointLine().filter(message, spanMessageHolder)) {
                    if (spanMessageHolder[0] != null) {
                        handler.reject(spanLogs, spanMessageHolder[0]);
                    } else {
                        handler.block(spanLogs);
                    }
                    return;
                }
            }
            List<Span> spanOutput = new ArrayList<>(1);
            try {
                spanDecoder.decode(spanMessage, spanOutput, "dummy");
            } catch (Exception e) {
                handler.reject(spanLogs, formatErrorMessage(message, e, ctx));
                return;
            }
            if (!spanOutput.isEmpty()) {
                Span span = spanOutput.get(0);
                if (preprocessor != null) {
                    preprocessor.forSpan().transform(span);
                    if (!preprocessor.forSpan().filter(span, spanMessageHolder)) {
                        if (spanMessageHolder[0] != null) {
                            handler.reject(spanLogs, spanMessageHolder[0]);
                        } else {
                            handler.block(spanLogs);
                        }
                        return;
                    }
                }
                if (samplerFunc.apply(span)) {
                    // after sampling, span line data is no longer needed
                    spanLogs.setSpan(null);
                    handler.report(spanLogs);
                }
            }
        }
    }
}

Example 13 with ReportableEntityPreprocessor

use of com.wavefront.agent.preprocessor.ReportableEntityPreprocessor in project java by wavefrontHQ.

the class JaegerProtobufUtils method processSpan.

private static void processSpan(Model.Span span, String serviceName, String sourceName, String applicationName, String cluster, String shard, List<Annotation> processAnnotations, ReportableEntityHandler<Span, String> spanHandler, ReportableEntityHandler<SpanLogs, String> spanLogsHandler, @Nullable WavefrontInternalReporter wfInternalReporter, Supplier<Boolean> spanLogsDisabled, Supplier<ReportableEntityPreprocessor> preprocessorSupplier, SpanSampler sampler, Set<String> traceDerivedCustomTagKeys, Counter discardedSpansBySampler, Set<Pair<Map<String, String>, String>> discoveredHeartbeatMetrics) {
    List<Annotation> annotations = new ArrayList<>(processAnnotations);
    // serviceName is mandatory in Jaeger
    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
    String componentTagValue = NULL_TAG_VAL;
    boolean isError = false;
    if (span.getTagsList() != null) {
        for (Model.KeyValue tag : span.getTagsList()) {
            if (IGNORE_TAGS.contains(tag.getKey()) || (tag.getVType() == Model.ValueType.STRING && StringUtils.isBlank(tag.getVStr()))) {
                continue;
            }
            Annotation annotation = tagToAnnotation(tag);
            if (annotation != null) {
                switch(annotation.getKey()) {
                    case APPLICATION_TAG_KEY:
                        applicationName = annotation.getValue();
                        continue;
                    case CLUSTER_TAG_KEY:
                        cluster = annotation.getValue();
                        continue;
                    case SHARD_TAG_KEY:
                        shard = annotation.getValue();
                        continue;
                    case SOURCE_KEY:
                        // Do not add source to annotation span tag list.
                        sourceName = annotation.getValue();
                        continue;
                    case SERVICE_TAG_KEY:
                        // Do not use service tag from annotations, use field instead
                        continue;
                    case COMPONENT_TAG_KEY:
                        componentTagValue = annotation.getValue();
                        break;
                    case ERROR_TAG_KEY:
                        // only error=true is supported
                        isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                        break;
                }
                annotations.add(annotation);
            }
        }
    }
    // Add all wavefront indexed tags. These are set based on below hierarchy.
    // Span Level > Process Level > Proxy Level > Default
    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
    if (span.getReferencesList() != null) {
        for (Model.SpanRef reference : span.getReferencesList()) {
            switch(reference.getRefType()) {
                case CHILD_OF:
                    if (!reference.getSpanId().isEmpty()) {
                        annotations.add(new Annotation(TraceConstants.PARENT_KEY, toStringId(reference.getSpanId())));
                    }
                    break;
                case FOLLOWS_FROM:
                    if (!reference.getSpanId().isEmpty()) {
                        annotations.add(new Annotation(TraceConstants.FOLLOWS_FROM_KEY, toStringId(reference.getSpanId())));
                    }
                default:
            }
        }
    }
    if (!spanLogsDisabled.get() && span.getLogsCount() > 0) {
        annotations.add(new Annotation("_spanLogs", "true"));
    }
    Span wavefrontSpan = Span.newBuilder().setCustomer("dummy").setName(span.getOperationName()).setSource(sourceName).setSpanId(toStringId(span.getSpanId())).setTraceId(toStringId(span.getTraceId())).setStartMillis(toMillis(span.getStartTime())).setDuration(toMillis(span.getDuration())).setAnnotations(annotations).build();
    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
        JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
        JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
    }
    if (preprocessorSupplier != null) {
        ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
        String[] messageHolder = new String[1];
        preprocessor.forSpan().transform(wavefrontSpan);
        if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
            if (messageHolder[0] != null) {
                spanHandler.reject(wavefrontSpan, messageHolder[0]);
            } else {
                spanHandler.block(wavefrontSpan);
            }
            return;
        }
    }
    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
        spanHandler.report(wavefrontSpan);
        if (span.getLogsCount() > 0 && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
            SpanLogs spanLogs = SpanLogs.newBuilder().setCustomer("default").setTraceId(wavefrontSpan.getTraceId()).setSpanId(wavefrontSpan.getSpanId()).setLogs(span.getLogsList().stream().map(x -> {
                Map<String, String> fields = new HashMap<>(x.getFieldsCount());
                x.getFieldsList().forEach(t -> {
                    switch(t.getVType()) {
                        case STRING:
                            fields.put(t.getKey(), t.getVStr());
                            break;
                        case BOOL:
                            fields.put(t.getKey(), String.valueOf(t.getVBool()));
                            break;
                        case INT64:
                            fields.put(t.getKey(), String.valueOf(t.getVInt64()));
                            break;
                        case FLOAT64:
                            fields.put(t.getKey(), String.valueOf(t.getVFloat64()));
                            break;
                        case BINARY:
                        // ignore
                        default:
                    }
                });
                return SpanLog.newBuilder().setTimestamp(toMicros(x.getTimestamp())).setFields(fields).build();
            }).collect(Collectors.toList())).build();
            spanLogsHandler.report(spanLogs);
        }
    }
    // report stats irrespective of span sampling.
    if (wfInternalReporter != null) {
        // Set post preprocessor rule values and report converted metrics/histograms from the span
        List<Annotation> processedAnnotations = wavefrontSpan.getAnnotations();
        for (Annotation processedAnnotation : processedAnnotations) {
            switch(processedAnnotation.getKey()) {
                case APPLICATION_TAG_KEY:
                    applicationName = processedAnnotation.getValue();
                    continue;
                case SERVICE_TAG_KEY:
                    serviceName = processedAnnotation.getValue();
                    continue;
                case CLUSTER_TAG_KEY:
                    cluster = processedAnnotation.getValue();
                    continue;
                case SHARD_TAG_KEY:
                    shard = processedAnnotation.getValue();
                    continue;
                case COMPONENT_TAG_KEY:
                    componentTagValue = processedAnnotation.getValue();
                    continue;
                case ERROR_TAG_KEY:
                    isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                    continue;
            }
        }
        List<Pair<String, String>> spanTags = processedAnnotations.stream().map(a -> new Pair<>(a.getKey(), a.getValue())).collect(Collectors.toList());
        discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, wavefrontSpan.getName(), applicationName, serviceName, cluster, shard, wavefrontSpan.getSource(), componentTagValue, isError, toMicros(span.getDuration()), traceDerivedCustomTagKeys, spanTags, true));
    }
}

Example 14 with ReportableEntityPreprocessor

use of com.wavefront.agent.preprocessor.ReportableEntityPreprocessor in project java by wavefrontHQ.

the class JaegerThriftUtils method processSpan.

private static void processSpan(io.jaegertracing.thriftjava.Span span, String serviceName, String sourceName, String applicationName, String cluster, String shard, List<Annotation> processAnnotations, ReportableEntityHandler<Span, String> spanHandler, ReportableEntityHandler<SpanLogs, String> spanLogsHandler, @Nullable WavefrontInternalReporter wfInternalReporter, Supplier<Boolean> spanLogsDisabled, Supplier<ReportableEntityPreprocessor> preprocessorSupplier, SpanSampler sampler, Set<String> traceDerivedCustomTagKeys, Counter discardedSpansBySampler, Set<Pair<Map<String, String>, String>> discoveredHeartbeatMetrics) {
    List<Annotation> annotations = new ArrayList<>(processAnnotations);
    String traceId = new UUID(span.getTraceIdHigh(), span.getTraceIdLow()).toString();
    String strippedTraceId = StringUtils.stripStart(traceId.replace("-", ""), "0");
    strippedTraceId = strippedTraceId.length() > 0 ? strippedTraceId : "0";
    annotations.add(new Annotation("jaegerSpanId", Long.toHexString(span.getSpanId())));
    annotations.add(new Annotation("jaegerTraceId", strippedTraceId));
    // serviceName is mandatory in Jaeger
    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
    long parentSpanId = span.getParentSpanId();
    if (parentSpanId != 0) {
        annotations.add(new Annotation("parent", new UUID(0, parentSpanId).toString()));
    }
    String componentTagValue = NULL_TAG_VAL;
    boolean isError = false;
    if (span.getTags() != null) {
        for (Tag tag : span.getTags()) {
            if (IGNORE_TAGS.contains(tag.getKey()) || (tag.vType == TagType.STRING && StringUtils.isBlank(tag.getVStr()))) {
                continue;
            }
            Annotation annotation = tagToAnnotation(tag);
            if (annotation != null) {
                switch(annotation.getKey()) {
                    case APPLICATION_TAG_KEY:
                        applicationName = annotation.getValue();
                        continue;
                    case CLUSTER_TAG_KEY:
                        cluster = annotation.getValue();
                        continue;
                    case SHARD_TAG_KEY:
                        shard = annotation.getValue();
                        continue;
                    case SOURCE_KEY:
                        // Do not add source to annotation span tag list.
                        sourceName = annotation.getValue();
                        continue;
                    case SERVICE_TAG_KEY:
                        // Do not use service tag from annotations, use field instead
                        continue;
                    case COMPONENT_TAG_KEY:
                        componentTagValue = annotation.getValue();
                        break;
                    case ERROR_TAG_KEY:
                        // only error=true is supported
                        isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                        break;
                }
                annotations.add(annotation);
            }
        }
    }
    // Add all wavefront indexed tags. These are set based on below hierarchy.
    // Span Level > Process Level > Proxy Level > Default
    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
    if (span.getReferences() != null) {
        for (SpanRef reference : span.getReferences()) {
            switch(reference.refType) {
                case CHILD_OF:
                    if (reference.getSpanId() != 0 && reference.getSpanId() != parentSpanId) {
                        annotations.add(new Annotation(TraceConstants.PARENT_KEY, new UUID(0, reference.getSpanId()).toString()));
                    }
                case FOLLOWS_FROM:
                    if (reference.getSpanId() != 0) {
                        annotations.add(new Annotation(TraceConstants.FOLLOWS_FROM_KEY, new UUID(0, reference.getSpanId()).toString()));
                    }
                default:
            }
        }
    }
    if (!spanLogsDisabled.get() && span.getLogs() != null && !span.getLogs().isEmpty()) {
        annotations.add(new Annotation("_spanLogs", "true"));
    }
    Span wavefrontSpan = Span.newBuilder().setCustomer("dummy").setName(span.getOperationName()).setSource(sourceName).setSpanId(new UUID(0, span.getSpanId()).toString()).setTraceId(traceId).setStartMillis(span.getStartTime() / 1000).setDuration(span.getDuration() / 1000).setAnnotations(annotations).build();
    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
        JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
        JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
    }
    if (preprocessorSupplier != null) {
        ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
        String[] messageHolder = new String[1];
        preprocessor.forSpan().transform(wavefrontSpan);
        if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
            if (messageHolder[0] != null) {
                spanHandler.reject(wavefrontSpan, messageHolder[0]);
            } else {
                spanHandler.block(wavefrontSpan);
            }
            return;
        }
    }
    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
        spanHandler.report(wavefrontSpan);
        if (span.getLogs() != null && !span.getLogs().isEmpty() && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
            SpanLogs spanLogs = SpanLogs.newBuilder().setCustomer("default").setTraceId(wavefrontSpan.getTraceId()).setSpanId(wavefrontSpan.getSpanId()).setLogs(span.getLogs().stream().map(x -> {
                Map<String, String> fields = new HashMap<>(x.fields.size());
                x.fields.forEach(t -> {
                    switch(t.vType) {
                        case STRING:
                            fields.put(t.getKey(), t.getVStr());
                            break;
                        case BOOL:
                            fields.put(t.getKey(), String.valueOf(t.isVBool()));
                            break;
                        case LONG:
                            fields.put(t.getKey(), String.valueOf(t.getVLong()));
                            break;
                        case DOUBLE:
                            fields.put(t.getKey(), String.valueOf(t.getVDouble()));
                            break;
                        case BINARY:
                        // ignore
                        default:
                    }
                });
                return SpanLog.newBuilder().setTimestamp(x.timestamp).setFields(fields).build();
            }).collect(Collectors.toList())).build();
            spanLogsHandler.report(spanLogs);
        }
    }
    // report stats irrespective of span sampling.
    if (wfInternalReporter != null) {
        // Set post preprocessor rule values and report converted metrics/histograms from the span
        List<Annotation> processedAnnotations = wavefrontSpan.getAnnotations();
        for (Annotation processedAnnotation : processedAnnotations) {
            switch(processedAnnotation.getKey()) {
                case APPLICATION_TAG_KEY:
                    applicationName = processedAnnotation.getValue();
                    continue;
                case SERVICE_TAG_KEY:
                    serviceName = processedAnnotation.getValue();
                    continue;
                case CLUSTER_TAG_KEY:
                    cluster = processedAnnotation.getValue();
                    continue;
                case SHARD_TAG_KEY:
                    shard = processedAnnotation.getValue();
                    continue;
                case COMPONENT_TAG_KEY:
                    componentTagValue = processedAnnotation.getValue();
                    continue;
                case ERROR_TAG_KEY:
                    isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                    continue;
            }
        }
        List<Pair<String, String>> spanTags = processedAnnotations.stream().map(a -> new Pair<>(a.getKey(), a.getValue())).collect(Collectors.toList());
        // TODO: Modify to use new method from wavefront internal reporter.
        discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, wavefrontSpan.getName(), applicationName, serviceName, cluster, shard, wavefrontSpan.getSource(), componentTagValue, isError, span.getDuration(), traceDerivedCustomTagKeys, spanTags, true));
    }
}

Example 15 with ReportableEntityPreprocessor

use of com.wavefront.agent.preprocessor.ReportableEntityPreprocessor in project java by wavefrontHQ.

the class PushAgentTest method setUserPreprocessorForTraceDerivedREDMetrics.

private void setUserPreprocessorForTraceDerivedREDMetrics(int port) {
    ReportableEntityPreprocessor preprocessor = new ReportableEntityPreprocessor();
    PreprocessorRuleMetrics preprocessorRuleMetrics = new PreprocessorRuleMetrics(null, null, null);
    preprocessor.forSpan().addTransformer(new SpanAddAnnotationIfNotExistsTransformer("application", PREPROCESSED_APPLICATION_TAG_VALUE, x -> true, preprocessorRuleMetrics));
    preprocessor.forSpan().addTransformer(new SpanAddAnnotationIfNotExistsTransformer("service", PREPROCESSED_SERVICE_TAG_VALUE, x -> true, preprocessorRuleMetrics));
    preprocessor.forSpan().addTransformer(new SpanAddAnnotationIfNotExistsTransformer("cluster", PREPROCESSED_CLUSTER_TAG_VALUE, x -> true, preprocessorRuleMetrics));
    preprocessor.forSpan().addTransformer(new SpanAddAnnotationIfNotExistsTransformer("shard", PREPROCESSED_SHARD_TAG_VALUE, x -> true, preprocessorRuleMetrics));
    preprocessor.forSpan().addTransformer(new SpanReplaceRegexTransformer("sourceName", "^test.*", PREPROCESSED_SOURCE_VALUE, null, null, false, x -> true, preprocessorRuleMetrics));
    Map<String, ReportableEntityPreprocessor> userPreprocessorMap = new HashMap<>();
    userPreprocessorMap.put(String.valueOf(port), preprocessor);
    proxy.preprocessors.userPreprocessors = userPreprocessorMap;
}