Search in sources :

Example 6 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class JaegerProtobufUtils method processSpan.

private static void processSpan(Model.Span span, String serviceName, String sourceName, String applicationName, String cluster, String shard, List<Annotation> processAnnotations, ReportableEntityHandler<Span, String> spanHandler, ReportableEntityHandler<SpanLogs, String> spanLogsHandler, @Nullable WavefrontInternalReporter wfInternalReporter, Supplier<Boolean> spanLogsDisabled, Supplier<ReportableEntityPreprocessor> preprocessorSupplier, SpanSampler sampler, Set<String> traceDerivedCustomTagKeys, Counter discardedSpansBySampler, Set<Pair<Map<String, String>, String>> discoveredHeartbeatMetrics) {
    List<Annotation> annotations = new ArrayList<>(processAnnotations);
    // serviceName is mandatory in Jaeger
    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
    String componentTagValue = NULL_TAG_VAL;
    boolean isError = false;
    if (span.getTagsList() != null) {
        for (Model.KeyValue tag : span.getTagsList()) {
            if (IGNORE_TAGS.contains(tag.getKey()) || (tag.getVType() == Model.ValueType.STRING && StringUtils.isBlank(tag.getVStr()))) {
                continue;
            }
            Annotation annotation = tagToAnnotation(tag);
            if (annotation != null) {
                switch(annotation.getKey()) {
                    case APPLICATION_TAG_KEY:
                        applicationName = annotation.getValue();
                        continue;
                    case CLUSTER_TAG_KEY:
                        cluster = annotation.getValue();
                        continue;
                    case SHARD_TAG_KEY:
                        shard = annotation.getValue();
                        continue;
                    case SOURCE_KEY:
                        // Do not add source to annotation span tag list.
                        sourceName = annotation.getValue();
                        continue;
                    case SERVICE_TAG_KEY:
                        // Do not use service tag from annotations, use field instead
                        continue;
                    case COMPONENT_TAG_KEY:
                        componentTagValue = annotation.getValue();
                        break;
                    case ERROR_TAG_KEY:
                        // only error=true is supported
                        isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                        break;
                }
                annotations.add(annotation);
            }
        }
    }
    // Add all wavefront indexed tags. These are set based on below hierarchy.
    // Span Level > Process Level > Proxy Level > Default
    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
    if (span.getReferencesList() != null) {
        for (Model.SpanRef reference : span.getReferencesList()) {
            switch(reference.getRefType()) {
                case CHILD_OF:
                    if (!reference.getSpanId().isEmpty()) {
                        annotations.add(new Annotation(TraceConstants.PARENT_KEY, toStringId(reference.getSpanId())));
                    }
                    break;
                case FOLLOWS_FROM:
                    if (!reference.getSpanId().isEmpty()) {
                        annotations.add(new Annotation(TraceConstants.FOLLOWS_FROM_KEY, toStringId(reference.getSpanId())));
                    }
                default:
            }
        }
    }
    if (!spanLogsDisabled.get() && span.getLogsCount() > 0) {
        annotations.add(new Annotation("_spanLogs", "true"));
    }
    Span wavefrontSpan = Span.newBuilder().setCustomer("dummy").setName(span.getOperationName()).setSource(sourceName).setSpanId(toStringId(span.getSpanId())).setTraceId(toStringId(span.getTraceId())).setStartMillis(toMillis(span.getStartTime())).setDuration(toMillis(span.getDuration())).setAnnotations(annotations).build();
    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
        JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
        JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
    }
    if (preprocessorSupplier != null) {
        ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
        String[] messageHolder = new String[1];
        preprocessor.forSpan().transform(wavefrontSpan);
        if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
            if (messageHolder[0] != null) {
                spanHandler.reject(wavefrontSpan, messageHolder[0]);
            } else {
                spanHandler.block(wavefrontSpan);
            }
            return;
        }
    }
    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
        spanHandler.report(wavefrontSpan);
        if (span.getLogsCount() > 0 && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
            SpanLogs spanLogs = SpanLogs.newBuilder().setCustomer("default").setTraceId(wavefrontSpan.getTraceId()).setSpanId(wavefrontSpan.getSpanId()).setLogs(span.getLogsList().stream().map(x -> {
                Map<String, String> fields = new HashMap<>(x.getFieldsCount());
                x.getFieldsList().forEach(t -> {
                    switch(t.getVType()) {
                        case STRING:
                            fields.put(t.getKey(), t.getVStr());
                            break;
                        case BOOL:
                            fields.put(t.getKey(), String.valueOf(t.getVBool()));
                            break;
                        case INT64:
                            fields.put(t.getKey(), String.valueOf(t.getVInt64()));
                            break;
                        case FLOAT64:
                            fields.put(t.getKey(), String.valueOf(t.getVFloat64()));
                            break;
                        case BINARY:
                        // ignore
                        default:
                    }
                });
                return SpanLog.newBuilder().setTimestamp(toMicros(x.getTimestamp())).setFields(fields).build();
            }).collect(Collectors.toList())).build();
            spanLogsHandler.report(spanLogs);
        }
    }
    // report stats irrespective of span sampling.
    if (wfInternalReporter != null) {
        // Set post preprocessor rule values and report converted metrics/histograms from the span
        List<Annotation> processedAnnotations = wavefrontSpan.getAnnotations();
        for (Annotation processedAnnotation : processedAnnotations) {
            switch(processedAnnotation.getKey()) {
                case APPLICATION_TAG_KEY:
                    applicationName = processedAnnotation.getValue();
                    continue;
                case SERVICE_TAG_KEY:
                    serviceName = processedAnnotation.getValue();
                    continue;
                case CLUSTER_TAG_KEY:
                    cluster = processedAnnotation.getValue();
                    continue;
                case SHARD_TAG_KEY:
                    shard = processedAnnotation.getValue();
                    continue;
                case COMPONENT_TAG_KEY:
                    componentTagValue = processedAnnotation.getValue();
                    continue;
                case ERROR_TAG_KEY:
                    isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                    continue;
            }
        }
        List<Pair<String, String>> spanTags = processedAnnotations.stream().map(a -> new Pair<>(a.getKey(), a.getValue())).collect(Collectors.toList());
        discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, wavefrontSpan.getName(), applicationName, serviceName, cluster, shard, wavefrontSpan.getSource(), componentTagValue, isError, toMicros(span.getDuration()), traceDerivedCustomTagKeys, spanTags, true));
    }
}
Also used : Annotation(wavefront.report.Annotation) ERROR_SPAN_TAG_VAL(com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL) StringUtils(org.apache.commons.lang.StringUtils) FeatureCheckUtils.isFeatureDisabled(com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled) SpanSampler(com.wavefront.agent.sampler.SpanSampler) SPANLOGS_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED) HashMap(java.util.HashMap) NULL_TAG_VAL(com.wavefront.sdk.common.Constants.NULL_TAG_VAL) SpanLogs(wavefront.report.SpanLogs) ERROR_TAG_KEY(com.wavefront.sdk.common.Constants.ERROR_TAG_KEY) Supplier(java.util.function.Supplier) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) APPLICATION_TAG_KEY(com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) Durations.toMillis(com.google.protobuf.util.Durations.toMillis) COMPONENT_TAG_KEY(com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY) Map(java.util.Map) SpanDerivedMetricsUtils.reportWavefrontGeneratedData(com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData) TraceConstants(com.wavefront.common.TraceConstants) BigInteger(java.math.BigInteger) Timestamps.toMicros(com.google.protobuf.util.Timestamps.toMicros) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) Nullable(javax.annotation.Nullable) SHARD_TAG_KEY(com.wavefront.sdk.common.Constants.SHARD_TAG_KEY) ImmutableSet(com.google.common.collect.ImmutableSet) Counter(com.yammer.metrics.core.Counter) Durations.toMicros(com.google.protobuf.util.Durations.toMicros) ReportableEntityPreprocessor(com.wavefront.agent.preprocessor.ReportableEntityPreprocessor) Set(java.util.Set) SPAN_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED) UUID(java.util.UUID) Span(wavefront.report.Span) Logger(java.util.logging.Logger) SpanLog(wavefront.report.SpanLog) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) List(java.util.List) SERVICE_TAG_KEY(com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY) Pair(com.wavefront.sdk.common.Pair) Timestamps.toMillis(com.google.protobuf.util.Timestamps.toMillis) Model(io.opentelemetry.exporters.jaeger.proto.api_v2.Model) VisibleForTesting(com.google.common.annotations.VisibleForTesting) CLUSTER_TAG_KEY(com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY) SOURCE_KEY(com.wavefront.sdk.common.Constants.SOURCE_KEY) ReportableEntityPreprocessor(com.wavefront.agent.preprocessor.ReportableEntityPreprocessor) ArrayList(java.util.ArrayList) SpanLogs(wavefront.report.SpanLogs) ByteString(com.google.protobuf.ByteString) Span(wavefront.report.Span) Annotation(wavefront.report.Annotation) Model(io.opentelemetry.exporters.jaeger.proto.api_v2.Model) HashMap(java.util.HashMap) Map(java.util.Map) Pair(com.wavefront.sdk.common.Pair)

Example 7 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class JaegerThriftUtils method processSpan.

private static void processSpan(io.jaegertracing.thriftjava.Span span, String serviceName, String sourceName, String applicationName, String cluster, String shard, List<Annotation> processAnnotations, ReportableEntityHandler<Span, String> spanHandler, ReportableEntityHandler<SpanLogs, String> spanLogsHandler, @Nullable WavefrontInternalReporter wfInternalReporter, Supplier<Boolean> spanLogsDisabled, Supplier<ReportableEntityPreprocessor> preprocessorSupplier, SpanSampler sampler, Set<String> traceDerivedCustomTagKeys, Counter discardedSpansBySampler, Set<Pair<Map<String, String>, String>> discoveredHeartbeatMetrics) {
    List<Annotation> annotations = new ArrayList<>(processAnnotations);
    String traceId = new UUID(span.getTraceIdHigh(), span.getTraceIdLow()).toString();
    String strippedTraceId = StringUtils.stripStart(traceId.replace("-", ""), "0");
    strippedTraceId = strippedTraceId.length() > 0 ? strippedTraceId : "0";
    annotations.add(new Annotation("jaegerSpanId", Long.toHexString(span.getSpanId())));
    annotations.add(new Annotation("jaegerTraceId", strippedTraceId));
    // serviceName is mandatory in Jaeger
    annotations.add(new Annotation(SERVICE_TAG_KEY, serviceName));
    long parentSpanId = span.getParentSpanId();
    if (parentSpanId != 0) {
        annotations.add(new Annotation("parent", new UUID(0, parentSpanId).toString()));
    }
    String componentTagValue = NULL_TAG_VAL;
    boolean isError = false;
    if (span.getTags() != null) {
        for (Tag tag : span.getTags()) {
            if (IGNORE_TAGS.contains(tag.getKey()) || (tag.vType == TagType.STRING && StringUtils.isBlank(tag.getVStr()))) {
                continue;
            }
            Annotation annotation = tagToAnnotation(tag);
            if (annotation != null) {
                switch(annotation.getKey()) {
                    case APPLICATION_TAG_KEY:
                        applicationName = annotation.getValue();
                        continue;
                    case CLUSTER_TAG_KEY:
                        cluster = annotation.getValue();
                        continue;
                    case SHARD_TAG_KEY:
                        shard = annotation.getValue();
                        continue;
                    case SOURCE_KEY:
                        // Do not add source to annotation span tag list.
                        sourceName = annotation.getValue();
                        continue;
                    case SERVICE_TAG_KEY:
                        // Do not use service tag from annotations, use field instead
                        continue;
                    case COMPONENT_TAG_KEY:
                        componentTagValue = annotation.getValue();
                        break;
                    case ERROR_TAG_KEY:
                        // only error=true is supported
                        isError = annotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                        break;
                }
                annotations.add(annotation);
            }
        }
    }
    // Add all wavefront indexed tags. These are set based on below hierarchy.
    // Span Level > Process Level > Proxy Level > Default
    annotations.add(new Annotation(APPLICATION_TAG_KEY, applicationName));
    annotations.add(new Annotation(CLUSTER_TAG_KEY, cluster));
    annotations.add(new Annotation(SHARD_TAG_KEY, shard));
    if (span.getReferences() != null) {
        for (SpanRef reference : span.getReferences()) {
            switch(reference.refType) {
                case CHILD_OF:
                    if (reference.getSpanId() != 0 && reference.getSpanId() != parentSpanId) {
                        annotations.add(new Annotation(TraceConstants.PARENT_KEY, new UUID(0, reference.getSpanId()).toString()));
                    }
                case FOLLOWS_FROM:
                    if (reference.getSpanId() != 0) {
                        annotations.add(new Annotation(TraceConstants.FOLLOWS_FROM_KEY, new UUID(0, reference.getSpanId()).toString()));
                    }
                default:
            }
        }
    }
    if (!spanLogsDisabled.get() && span.getLogs() != null && !span.getLogs().isEmpty()) {
        annotations.add(new Annotation("_spanLogs", "true"));
    }
    Span wavefrontSpan = Span.newBuilder().setCustomer("dummy").setName(span.getOperationName()).setSource(sourceName).setSpanId(new UUID(0, span.getSpanId()).toString()).setTraceId(traceId).setStartMillis(span.getStartTime() / 1000).setDuration(span.getDuration() / 1000).setAnnotations(annotations).build();
    // Log Jaeger spans as well as Wavefront spans for debugging purposes.
    if (JAEGER_DATA_LOGGER.isLoggable(Level.FINEST)) {
        JAEGER_DATA_LOGGER.info("Inbound Jaeger span: " + span.toString());
        JAEGER_DATA_LOGGER.info("Converted Wavefront span: " + wavefrontSpan.toString());
    }
    if (preprocessorSupplier != null) {
        ReportableEntityPreprocessor preprocessor = preprocessorSupplier.get();
        String[] messageHolder = new String[1];
        preprocessor.forSpan().transform(wavefrontSpan);
        if (!preprocessor.forSpan().filter(wavefrontSpan, messageHolder)) {
            if (messageHolder[0] != null) {
                spanHandler.reject(wavefrontSpan, messageHolder[0]);
            } else {
                spanHandler.block(wavefrontSpan);
            }
            return;
        }
    }
    if (sampler.sample(wavefrontSpan, discardedSpansBySampler)) {
        spanHandler.report(wavefrontSpan);
        if (span.getLogs() != null && !span.getLogs().isEmpty() && !isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, null)) {
            SpanLogs spanLogs = SpanLogs.newBuilder().setCustomer("default").setTraceId(wavefrontSpan.getTraceId()).setSpanId(wavefrontSpan.getSpanId()).setLogs(span.getLogs().stream().map(x -> {
                Map<String, String> fields = new HashMap<>(x.fields.size());
                x.fields.forEach(t -> {
                    switch(t.vType) {
                        case STRING:
                            fields.put(t.getKey(), t.getVStr());
                            break;
                        case BOOL:
                            fields.put(t.getKey(), String.valueOf(t.isVBool()));
                            break;
                        case LONG:
                            fields.put(t.getKey(), String.valueOf(t.getVLong()));
                            break;
                        case DOUBLE:
                            fields.put(t.getKey(), String.valueOf(t.getVDouble()));
                            break;
                        case BINARY:
                        // ignore
                        default:
                    }
                });
                return SpanLog.newBuilder().setTimestamp(x.timestamp).setFields(fields).build();
            }).collect(Collectors.toList())).build();
            spanLogsHandler.report(spanLogs);
        }
    }
    // report stats irrespective of span sampling.
    if (wfInternalReporter != null) {
        // Set post preprocessor rule values and report converted metrics/histograms from the span
        List<Annotation> processedAnnotations = wavefrontSpan.getAnnotations();
        for (Annotation processedAnnotation : processedAnnotations) {
            switch(processedAnnotation.getKey()) {
                case APPLICATION_TAG_KEY:
                    applicationName = processedAnnotation.getValue();
                    continue;
                case SERVICE_TAG_KEY:
                    serviceName = processedAnnotation.getValue();
                    continue;
                case CLUSTER_TAG_KEY:
                    cluster = processedAnnotation.getValue();
                    continue;
                case SHARD_TAG_KEY:
                    shard = processedAnnotation.getValue();
                    continue;
                case COMPONENT_TAG_KEY:
                    componentTagValue = processedAnnotation.getValue();
                    continue;
                case ERROR_TAG_KEY:
                    isError = processedAnnotation.getValue().equals(ERROR_SPAN_TAG_VAL);
                    continue;
            }
        }
        List<Pair<String, String>> spanTags = processedAnnotations.stream().map(a -> new Pair<>(a.getKey(), a.getValue())).collect(Collectors.toList());
        // TODO: Modify to use new method from wavefront internal reporter.
        discoveredHeartbeatMetrics.add(reportWavefrontGeneratedData(wfInternalReporter, wavefrontSpan.getName(), applicationName, serviceName, cluster, shard, wavefrontSpan.getSource(), componentTagValue, isError, span.getDuration(), traceDerivedCustomTagKeys, spanTags, true));
    }
}
Also used : Annotation(wavefront.report.Annotation) ERROR_SPAN_TAG_VAL(com.wavefront.internal.SpanDerivedMetricsUtils.ERROR_SPAN_TAG_VAL) StringUtils(org.apache.commons.lang.StringUtils) FeatureCheckUtils.isFeatureDisabled(com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled) SpanSampler(com.wavefront.agent.sampler.SpanSampler) SPANLOGS_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED) HashMap(java.util.HashMap) NULL_TAG_VAL(com.wavefront.sdk.common.Constants.NULL_TAG_VAL) SpanLogs(wavefront.report.SpanLogs) ERROR_TAG_KEY(com.wavefront.sdk.common.Constants.ERROR_TAG_KEY) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) APPLICATION_TAG_KEY(com.wavefront.sdk.common.Constants.APPLICATION_TAG_KEY) TagType(io.jaegertracing.thriftjava.TagType) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) Batch(io.jaegertracing.thriftjava.Batch) COMPONENT_TAG_KEY(com.wavefront.sdk.common.Constants.COMPONENT_TAG_KEY) Map(java.util.Map) SpanDerivedMetricsUtils.reportWavefrontGeneratedData(com.wavefront.internal.SpanDerivedMetricsUtils.reportWavefrontGeneratedData) TraceConstants(com.wavefront.common.TraceConstants) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) Nullable(javax.annotation.Nullable) SHARD_TAG_KEY(com.wavefront.sdk.common.Constants.SHARD_TAG_KEY) ImmutableSet(com.google.common.collect.ImmutableSet) Counter(com.yammer.metrics.core.Counter) SpanRef(io.jaegertracing.thriftjava.SpanRef) ReportableEntityPreprocessor(com.wavefront.agent.preprocessor.ReportableEntityPreprocessor) Set(java.util.Set) SPAN_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED) UUID(java.util.UUID) Span(wavefront.report.Span) Logger(java.util.logging.Logger) SpanLog(wavefront.report.SpanLog) Collectors(java.util.stream.Collectors) List(java.util.List) SERVICE_TAG_KEY(com.wavefront.sdk.common.Constants.SERVICE_TAG_KEY) Pair(com.wavefront.sdk.common.Pair) CLUSTER_TAG_KEY(com.wavefront.sdk.common.Constants.CLUSTER_TAG_KEY) SOURCE_KEY(com.wavefront.sdk.common.Constants.SOURCE_KEY) Tag(io.jaegertracing.thriftjava.Tag) ReportableEntityPreprocessor(com.wavefront.agent.preprocessor.ReportableEntityPreprocessor) SpanRef(io.jaegertracing.thriftjava.SpanRef) ArrayList(java.util.ArrayList) SpanLogs(wavefront.report.SpanLogs) Span(wavefront.report.Span) Annotation(wavefront.report.Annotation) Tag(io.jaegertracing.thriftjava.Tag) UUID(java.util.UUID) HashMap(java.util.HashMap) Map(java.util.Map) Pair(com.wavefront.sdk.common.Pair)

Example 8 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class LogsIngesterTest method setup.

private void setup(LogsIngestionConfig config) throws IOException, GrokException, ConfigurationException {
    logsIngestionConfig = config;
    // HACK: Never call flush automatically.
    logsIngestionConfig.aggregationIntervalSeconds = 10000;
    logsIngestionConfig.verifyAndInit();
    mockPointHandler = createMock(ReportableEntityHandler.class);
    mockHistogramHandler = createMock(ReportableEntityHandler.class);
    mockFactory = createMock(ReportableEntityHandlerFactory.class);
    expect((ReportableEntityHandler) mockFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, "logs-ingester"))).andReturn(mockPointHandler).anyTimes();
    expect((ReportableEntityHandler) mockFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, "logs-ingester"))).andReturn(mockHistogramHandler).anyTimes();
    replay(mockFactory);
    logsIngesterUnderTest = new LogsIngester(mockFactory, () -> logsIngestionConfig, null, now::get, nanos::get);
    logsIngesterUnderTest.start();
    filebeatIngesterUnderTest = new FilebeatIngester(logsIngesterUnderTest, now::get);
    rawLogsIngesterUnderTest = new RawLogsIngesterPortUnificationHandler("12345", logsIngesterUnderTest, x -> "testHost", TokenAuthenticatorBuilder.create().build(), new NoopHealthCheckManager(), null);
}
Also used : RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) EasyMock.reset(org.easymock.EasyMock.reset) Map(java.util.Map) After(org.junit.After) HandlerKey(com.wavefront.agent.handlers.HandlerKey) ReportPoint(wavefront.report.ReportPoint) GrokException(oi.thekraken.grok.api.exception.GrokException) LogsIngestionConfig(com.wavefront.agent.config.LogsIngestionConfig) ImmutableMap(com.google.common.collect.ImmutableMap) InetSocketAddress(java.net.InetSocketAddress) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) PointMatchers(com.wavefront.agent.PointMatchers) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) MetricMatcher(com.wavefront.agent.config.MetricMatcher) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) YAMLFactory(com.fasterxml.jackson.dataformat.yaml.YAMLFactory) Matchers.lessThan(org.hamcrest.Matchers.lessThan) Matchers.hasSize(org.hamcrest.Matchers.hasSize) ConfigurationException(com.wavefront.agent.config.ConfigurationException) EasyMock.replay(org.easymock.EasyMock.replay) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) EasyMock.createMock(org.easymock.EasyMock.createMock) TokenAuthenticatorBuilder(com.wavefront.agent.auth.TokenAuthenticatorBuilder) MetricConstants(com.wavefront.common.MetricConstants) Capture(org.easymock.Capture) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) IOException(java.io.IOException) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) EasyMock.expect(org.easymock.EasyMock.expect) Maps(com.google.common.collect.Maps) File(java.io.File) Message(org.logstash.beats.Message) Consumer(java.util.function.Consumer) Channel(io.netty.channel.Channel) EasyMock.expectLastCall(org.easymock.EasyMock.expectLastCall) AtomicLong(java.util.concurrent.atomic.AtomicLong) Matchers.emptyIterable(org.hamcrest.Matchers.emptyIterable) CaptureType(org.easymock.CaptureType) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) NoopHealthCheckManager(com.wavefront.agent.channel.NoopHealthCheckManager) EasyMock.verify(org.easymock.EasyMock.verify) Histogram(wavefront.report.Histogram) NoopHealthCheckManager(com.wavefront.agent.channel.NoopHealthCheckManager) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory)

Example 9 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class PushAgent method startRelayListener.

@VisibleForTesting
protected void startRelayListener(String strPort, ReportableEntityHandlerFactory handlerFactory, SharedGraphiteHostAnnotator hostAnnotator) {
    final int port = Integer.parseInt(strPort);
    registerPrefixFilter(strPort);
    registerTimestampFilter(strPort);
    if (proxyConfig.isHttpHealthCheckAllPorts())
        healthCheckManager.enableHealthcheck(port);
    ReportableEntityHandlerFactory handlerFactoryDelegate = proxyConfig.isPushRelayHistogramAggregator() ? new DelegatingReportableEntityHandlerFactoryImpl(handlerFactory) {

        @Override
        public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
            if (handlerKey.getEntityType() == ReportableEntityType.HISTOGRAM) {
                ChronicleMap<HistogramKey, AgentDigest> accumulator = ChronicleMap.of(HistogramKey.class, AgentDigest.class).keyMarshaller(HistogramKeyMarshaller.get()).valueMarshaller(AgentDigestMarshaller.get()).entries(proxyConfig.getPushRelayHistogramAggregatorAccumulatorSize()).averageKeySize(proxyConfig.getHistogramDistAvgKeyBytes()).averageValueSize(proxyConfig.getHistogramDistAvgDigestBytes()).maxBloatFactor(1000).create();
                AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> (short) Math.min(proxyConfig.getPushRelayHistogramAggregatorCompression(), entityProps.getGlobalProperties().getHistogramStorageAccuracy()), TimeUnit.SECONDS.toMillis(proxyConfig.getPushRelayHistogramAggregatorFlushSecs()), proxyConfig.getTimeProvider());
                AccumulationCache cachedAccumulator = new AccumulationCache(accumulator, agentDigestFactory, 0, "histogram.accumulator.distributionRelay", null);
                // noinspection unchecked
                return (ReportableEntityHandler<T, U>) new HistogramAccumulationHandlerImpl(handlerKey, cachedAccumulator, proxyConfig.getPushBlockedSamples(), null, validationConfiguration, true, rate -> entityProps.get(ReportableEntityType.HISTOGRAM).reportReceivedRate(handlerKey.getHandle(), rate), blockedHistogramsLogger, VALID_HISTOGRAMS_LOGGER);
            }
            return delegate.getHandler(handlerKey);
        }
    } : handlerFactory;
    Map<ReportableEntityType, ReportableEntityDecoder<?, ?>> filteredDecoders = decoderSupplier.get().entrySet().stream().filter(x -> !x.getKey().equals(ReportableEntityType.SOURCE_TAG)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    ChannelHandler channelHandler = new RelayPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, filteredDecoders, handlerFactoryDelegate, preprocessors.get(strPort), hostAnnotator, () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE_SPAN_LOGS).isFeatureDisabled());
    startAsManagedThread(port, new TcpIngester(createInitializer(channelHandler, port, proxyConfig.getPushListenerMaxReceivedLength(), proxyConfig.getPushListenerHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-relay-" + port);
}
Also used : HandlerKey(com.wavefront.agent.handlers.HandlerKey) QueueingReason(com.wavefront.agent.data.QueueingReason) CustomTracingPortUnificationHandler(com.wavefront.agent.listeners.tracing.CustomTracingPortUnificationHandler) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) EntityProperties(com.wavefront.agent.data.EntityProperties) NettyServerBuilder(io.grpc.netty.NettyServerBuilder) RequestConfig(org.apache.http.client.config.RequestConfig) StringUtils(org.apache.commons.lang3.StringUtils) SpanSanitizeTransformer(com.wavefront.agent.preprocessor.SpanSanitizeTransformer) BooleanUtils(org.apache.commons.lang.BooleanUtils) InetAddress(java.net.InetAddress) SQSQueueFactoryImpl(com.wavefront.agent.queueing.SQSQueueFactoryImpl) PointHandlerDispatcher(com.wavefront.agent.histogram.PointHandlerDispatcher) Map(java.util.Map) HandlerKey(com.wavefront.agent.handlers.HandlerKey) ByteArrayDecoder(io.netty.handler.codec.bytes.ByteArrayDecoder) RecyclableRateLimiter(com.google.common.util.concurrent.RecyclableRateLimiter) HealthCheckManagerImpl(com.wavefront.agent.channel.HealthCheckManagerImpl) HistogramKey(com.wavefront.agent.histogram.HistogramKey) AdminPortUnificationHandler(com.wavefront.agent.listeners.AdminPortUnificationHandler) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) ProxyUtil.createInitializer(com.wavefront.agent.ProxyUtil.createInitializer) Executors(java.util.concurrent.Executors) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ByteOrder(java.nio.ByteOrder) AgentConfiguration(com.wavefront.api.agent.AgentConfiguration) OpenTSDBDecoder(com.wavefront.ingester.OpenTSDBDecoder) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) HealthCheckManager(com.wavefront.agent.channel.HealthCheckManager) ChannelOption(io.netty.channel.ChannelOption) SpanSampler(com.wavefront.agent.sampler.SpanSampler) TChannel(com.uber.tchannel.api.TChannel) Supplier(java.util.function.Supplier) RelayPortUnificationHandler(com.wavefront.agent.listeners.RelayPortUnificationHandler) TcpIngester(com.wavefront.ingester.TcpIngester) ArrayList(java.util.ArrayList) HttpClient(org.apache.http.client.HttpClient) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConfigurationException(com.wavefront.agent.config.ConfigurationException) Utils.lazySupplier(com.wavefront.common.Utils.lazySupplier) Utils.csvToList(com.wavefront.common.Utils.csvToList) Server(org.logstash.beats.Server) TokenAuthenticatorBuilder(com.wavefront.agent.auth.TokenAuthenticatorBuilder) MapLoader(com.wavefront.agent.histogram.MapLoader) Nullable(javax.annotation.Nullable) HistogramUtils(com.wavefront.agent.histogram.HistogramUtils) DataDogPortUnificationHandler(com.wavefront.agent.listeners.DataDogPortUnificationHandler) SslContext(io.netty.handler.ssl.SslContext) WavefrontSender(com.wavefront.sdk.common.WavefrontSender) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) File(java.io.File) CachingHostnameLookupResolver(com.wavefront.agent.channel.CachingHostnameLookupResolver) NamedThreadFactory(com.wavefront.common.NamedThreadFactory) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) ReportSourceTagDecoder(com.wavefront.ingester.ReportSourceTagDecoder) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) VALID_POINTS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_POINTS_LOGGER) Preconditions(com.google.common.base.Preconditions) WriteHttpJsonPortUnificationHandler(com.wavefront.agent.listeners.WriteHttpJsonPortUnificationHandler) Metrics(com.yammer.metrics.Metrics) HttpClientBuilder(org.apache.http.impl.client.HttpClientBuilder) QueueingFactory(com.wavefront.agent.queueing.QueueingFactory) SpanSamplerUtils(com.wavefront.agent.sampler.SpanSamplerUtils) ReportPointDecoder(com.wavefront.ingester.ReportPointDecoder) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) SpanLogsDecoder(com.wavefront.ingester.SpanLogsDecoder) Sampler(com.wavefront.sdk.entities.tracing.sampling.Sampler) ZipkinPortUnificationHandler(com.wavefront.agent.listeners.tracing.ZipkinPortUnificationHandler) TaskQueueFactory(com.wavefront.agent.queueing.TaskQueueFactory) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TokenAuthenticator(com.wavefront.agent.auth.TokenAuthenticator) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) Granularity(com.wavefront.agent.histogram.Granularity) DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) ReportPoint(wavefront.report.ReportPoint) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) HistogramRecompressor(com.wavefront.agent.histogram.HistogramRecompressor) CorsConfigBuilder(io.netty.handler.codec.http.cors.CorsConfigBuilder) FilebeatIngester(com.wavefront.agent.logsharvesting.FilebeatIngester) PickleProtocolDecoder(com.wavefront.ingester.PickleProtocolDecoder) JaegerPortUnificationHandler(com.wavefront.agent.listeners.tracing.JaegerPortUnificationHandler) IdentityHashMap(java.util.IdentityHashMap) TaggedMetricName(com.wavefront.common.TaggedMetricName) ImmutableMap(com.google.common.collect.ImmutableMap) HttpHealthCheckEndpointHandler(com.wavefront.agent.listeners.HttpHealthCheckEndpointHandler) Accumulator(com.wavefront.agent.histogram.accumulator.Accumulator) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) ReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl) Collectors(java.util.stream.Collectors) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) ExpectedAgentMetric(com.wavefront.metrics.ExpectedAgentMetric) ChannelByteArrayHandler(com.wavefront.agent.listeners.ChannelByteArrayHandler) QueueingFactoryImpl(com.wavefront.agent.queueing.QueueingFactoryImpl) JaegerTChannelCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerTChannelCollectorHandler) PreprocessorRuleMetrics(com.wavefront.agent.preprocessor.PreprocessorRuleMetrics) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) HashMap(java.util.HashMap) BindException(java.net.BindException) Function(java.util.function.Function) OpenTSDBPortUnificationHandler(com.wavefront.agent.listeners.OpenTSDBPortUnificationHandler) Level(java.util.logging.Level) JsonMetricsPortUnificationHandler(com.wavefront.agent.listeners.JsonMetricsPortUnificationHandler) TaskQueueFactoryImpl(com.wavefront.agent.queueing.TaskQueueFactoryImpl) ReportPointAddPrefixTransformer(com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer) ImmutableList(com.google.common.collect.ImmutableList) TrafficShapingRateLimitAdjuster(com.wavefront.agent.handlers.TrafficShapingRateLimitAdjuster) ObjectUtils(org.apache.commons.lang3.ObjectUtils) RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) HistogramKeyMarshaller(com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller) EventDecoder(com.wavefront.ingester.EventDecoder) ReportPointDecoderWrapper(com.wavefront.ingester.ReportPointDecoderWrapper) Nonnull(javax.annotation.Nonnull) InternalProxyWavefrontClient(com.wavefront.agent.handlers.InternalProxyWavefrontClient) LengthFieldBasedFrameDecoder(io.netty.handler.codec.LengthFieldBasedFrameDecoder) Counter(com.yammer.metrics.core.Counter) SenderTaskFactory(com.wavefront.agent.handlers.SenderTaskFactory) AgentDigestMarshaller(com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) SpanDecoder(com.wavefront.ingester.SpanDecoder) CorsConfig(io.netty.handler.codec.http.cors.CorsConfig) VALID_HISTOGRAMS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_HISTOGRAMS_LOGGER) HttpMethod(io.netty.handler.codec.http.HttpMethod) AgentDigest(com.tdunning.math.stats.AgentDigest) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) TracePortUnificationHandler(com.wavefront.agent.listeners.tracing.TracePortUnificationHandler) TimeUnit(java.util.concurrent.TimeUnit) Connection(com.uber.tchannel.channels.Connection) RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) SenderTaskFactoryImpl(com.wavefront.agent.handlers.SenderTaskFactoryImpl) JaegerGrpcCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerGrpcCollectorHandler) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Histogram(wavefront.report.Histogram) RelayPortUnificationHandler(com.wavefront.agent.listeners.RelayPortUnificationHandler) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ChannelHandler(io.netty.channel.ChannelHandler) ReportPoint(wavefront.report.ReportPoint) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) Map(java.util.Map) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) IdentityHashMap(java.util.IdentityHashMap) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ReportableEntityType(com.wavefront.data.ReportableEntityType) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) TcpIngester(com.wavefront.ingester.TcpIngester) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 10 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class PushAgent method startHistogramListeners.

protected void startHistogramListeners(List<String> ports, ReportableEntityHandler<ReportPoint, String> pointHandler, SharedGraphiteHostAnnotator hostAnnotator, @Nullable Granularity granularity, int flushSecs, boolean memoryCacheEnabled, File baseDirectory, Long accumulatorSize, int avgKeyBytes, int avgDigestBytes, short compression, boolean persist, SpanSampler sampler) throws Exception {
    if (ports.size() == 0)
        return;
    String listenerBinType = HistogramUtils.granularityToString(granularity);
    // Accumulator
    if (persist) {
        // Check directory
        checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!");
        checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!");
    }
    MapLoader<HistogramKey, AgentDigest, HistogramKeyMarshaller, AgentDigestMarshaller> mapLoader = new MapLoader<>(HistogramKey.class, AgentDigest.class, accumulatorSize, avgKeyBytes, avgDigestBytes, HistogramKeyMarshaller.get(), AgentDigestMarshaller.get(), persist);
    File accumulationFile = new File(baseDirectory, "accumulator." + listenerBinType);
    ChronicleMap<HistogramKey, AgentDigest> accumulator = mapLoader.get(accumulationFile);
    histogramExecutor.scheduleWithFixedDelay(() -> {
        // as ChronicleMap starts losing efficiency
        if (accumulator.size() > accumulatorSize * 5) {
            logger.severe("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 5x higher than currently configured size (" + accumulatorSize + "), which may cause severe performance degradation issues " + "or data loss! If the data volume is expected to stay at this level, we strongly " + "recommend increasing the value for accumulator size in wavefront.conf and " + "restarting the proxy.");
        } else if (accumulator.size() > accumulatorSize * 2) {
            logger.warning("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 2x higher than currently configured size (" + accumulatorSize + "), which may cause performance issues. If the data volume is " + "expected to stay at this level, we strongly recommend increasing the value " + "for accumulator size in wavefront.conf and restarting the proxy.");
        }
    }, 10, 10, TimeUnit.SECONDS);
    AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> (short) Math.min(compression, entityProps.getGlobalProperties().getHistogramStorageAccuracy()), TimeUnit.SECONDS.toMillis(flushSecs), proxyConfig.getTimeProvider());
    Accumulator cachedAccumulator = new AccumulationCache(accumulator, agentDigestFactory, (memoryCacheEnabled ? accumulatorSize : 0), "histogram.accumulator." + HistogramUtils.granularityToString(granularity), null);
    // Schedule write-backs
    histogramExecutor.scheduleWithFixedDelay(cachedAccumulator::flush, proxyConfig.getHistogramAccumulatorResolveInterval(), proxyConfig.getHistogramAccumulatorResolveInterval(), TimeUnit.MILLISECONDS);
    histogramFlushRunnables.add(cachedAccumulator::flush);
    PointHandlerDispatcher dispatcher = new PointHandlerDispatcher(cachedAccumulator, pointHandler, proxyConfig.getTimeProvider(), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), proxyConfig.getHistogramAccumulatorFlushMaxBatchSize() < 0 ? null : proxyConfig.getHistogramAccumulatorFlushMaxBatchSize(), granularity);
    histogramExecutor.scheduleWithFixedDelay(dispatcher, proxyConfig.getHistogramAccumulatorFlushInterval(), proxyConfig.getHistogramAccumulatorFlushInterval(), TimeUnit.MILLISECONDS);
    histogramFlushRunnables.add(dispatcher);
    // gracefully shutdown persisted accumulator (ChronicleMap) on proxy exit
    shutdownTasks.add(() -> {
        try {
            logger.fine("Flushing in-flight histogram accumulator digests: " + listenerBinType);
            cachedAccumulator.flush();
            logger.fine("Shutting down histogram accumulator cache: " + listenerBinType);
            accumulator.close();
        } catch (Throwable t) {
            logger.log(Level.SEVERE, "Error flushing " + listenerBinType + " accumulator, possibly unclean shutdown: ", t);
        }
    });
    ReportableEntityHandlerFactory histogramHandlerFactory = new ReportableEntityHandlerFactory() {

        private final Map<HandlerKey, ReportableEntityHandler<?, ?>> handlers = new ConcurrentHashMap<>();

        @SuppressWarnings("unchecked")
        @Override
        public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
            return (ReportableEntityHandler<T, U>) handlers.computeIfAbsent(handlerKey, k -> new HistogramAccumulationHandlerImpl(handlerKey, cachedAccumulator, proxyConfig.getPushBlockedSamples(), granularity, validationConfiguration, granularity == null, null, blockedHistogramsLogger, VALID_HISTOGRAMS_LOGGER));
        }

        @Override
        public void shutdown(@Nonnull String handle) {
            handlers.values().forEach(ReportableEntityHandler::shutdown);
        }
    };
    ports.forEach(strPort -> {
        int port = Integer.parseInt(strPort);
        registerPrefixFilter(strPort);
        registerTimestampFilter(strPort);
        if (proxyConfig.isHttpHealthCheckAllPorts()) {
            healthCheckManager.enableHealthcheck(port);
        }
        WavefrontPortUnificationHandler wavefrontPortUnificationHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, decoderSupplier.get(), histogramHandlerFactory, hostAnnotator, preprocessors.get(strPort), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE_SPAN_LOGS).isFeatureDisabled(), sampler);
        startAsManagedThread(port, new TcpIngester(createInitializer(wavefrontPortUnificationHandler, port, proxyConfig.getHistogramMaxReceivedLength(), proxyConfig.getHistogramHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-histogram-" + port);
        logger.info("listening on port: " + port + " for histogram samples, accumulating to the " + listenerBinType);
    });
}
Also used : Accumulator(com.wavefront.agent.histogram.accumulator.Accumulator) HandlerKey(com.wavefront.agent.handlers.HandlerKey) QueueingReason(com.wavefront.agent.data.QueueingReason) CustomTracingPortUnificationHandler(com.wavefront.agent.listeners.tracing.CustomTracingPortUnificationHandler) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) EntityProperties(com.wavefront.agent.data.EntityProperties) NettyServerBuilder(io.grpc.netty.NettyServerBuilder) RequestConfig(org.apache.http.client.config.RequestConfig) StringUtils(org.apache.commons.lang3.StringUtils) SpanSanitizeTransformer(com.wavefront.agent.preprocessor.SpanSanitizeTransformer) BooleanUtils(org.apache.commons.lang.BooleanUtils) InetAddress(java.net.InetAddress) SQSQueueFactoryImpl(com.wavefront.agent.queueing.SQSQueueFactoryImpl) PointHandlerDispatcher(com.wavefront.agent.histogram.PointHandlerDispatcher) Map(java.util.Map) HandlerKey(com.wavefront.agent.handlers.HandlerKey) ByteArrayDecoder(io.netty.handler.codec.bytes.ByteArrayDecoder) RecyclableRateLimiter(com.google.common.util.concurrent.RecyclableRateLimiter) HealthCheckManagerImpl(com.wavefront.agent.channel.HealthCheckManagerImpl) HistogramKey(com.wavefront.agent.histogram.HistogramKey) AdminPortUnificationHandler(com.wavefront.agent.listeners.AdminPortUnificationHandler) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) ProxyUtil.createInitializer(com.wavefront.agent.ProxyUtil.createInitializer) Executors(java.util.concurrent.Executors) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ByteOrder(java.nio.ByteOrder) AgentConfiguration(com.wavefront.api.agent.AgentConfiguration) OpenTSDBDecoder(com.wavefront.ingester.OpenTSDBDecoder) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) HealthCheckManager(com.wavefront.agent.channel.HealthCheckManager) ChannelOption(io.netty.channel.ChannelOption) SpanSampler(com.wavefront.agent.sampler.SpanSampler) TChannel(com.uber.tchannel.api.TChannel) Supplier(java.util.function.Supplier) RelayPortUnificationHandler(com.wavefront.agent.listeners.RelayPortUnificationHandler) TcpIngester(com.wavefront.ingester.TcpIngester) ArrayList(java.util.ArrayList) HttpClient(org.apache.http.client.HttpClient) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConfigurationException(com.wavefront.agent.config.ConfigurationException) Utils.lazySupplier(com.wavefront.common.Utils.lazySupplier) Utils.csvToList(com.wavefront.common.Utils.csvToList) Server(org.logstash.beats.Server) TokenAuthenticatorBuilder(com.wavefront.agent.auth.TokenAuthenticatorBuilder) MapLoader(com.wavefront.agent.histogram.MapLoader) Nullable(javax.annotation.Nullable) HistogramUtils(com.wavefront.agent.histogram.HistogramUtils) DataDogPortUnificationHandler(com.wavefront.agent.listeners.DataDogPortUnificationHandler) SslContext(io.netty.handler.ssl.SslContext) WavefrontSender(com.wavefront.sdk.common.WavefrontSender) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) File(java.io.File) CachingHostnameLookupResolver(com.wavefront.agent.channel.CachingHostnameLookupResolver) NamedThreadFactory(com.wavefront.common.NamedThreadFactory) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) ReportSourceTagDecoder(com.wavefront.ingester.ReportSourceTagDecoder) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) VALID_POINTS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_POINTS_LOGGER) Preconditions(com.google.common.base.Preconditions) WriteHttpJsonPortUnificationHandler(com.wavefront.agent.listeners.WriteHttpJsonPortUnificationHandler) Metrics(com.yammer.metrics.Metrics) HttpClientBuilder(org.apache.http.impl.client.HttpClientBuilder) QueueingFactory(com.wavefront.agent.queueing.QueueingFactory) SpanSamplerUtils(com.wavefront.agent.sampler.SpanSamplerUtils) ReportPointDecoder(com.wavefront.ingester.ReportPointDecoder) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) SpanLogsDecoder(com.wavefront.ingester.SpanLogsDecoder) Sampler(com.wavefront.sdk.entities.tracing.sampling.Sampler) ZipkinPortUnificationHandler(com.wavefront.agent.listeners.tracing.ZipkinPortUnificationHandler) TaskQueueFactory(com.wavefront.agent.queueing.TaskQueueFactory) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TokenAuthenticator(com.wavefront.agent.auth.TokenAuthenticator) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) Granularity(com.wavefront.agent.histogram.Granularity) DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) ReportPoint(wavefront.report.ReportPoint) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) HistogramRecompressor(com.wavefront.agent.histogram.HistogramRecompressor) CorsConfigBuilder(io.netty.handler.codec.http.cors.CorsConfigBuilder) FilebeatIngester(com.wavefront.agent.logsharvesting.FilebeatIngester) PickleProtocolDecoder(com.wavefront.ingester.PickleProtocolDecoder) JaegerPortUnificationHandler(com.wavefront.agent.listeners.tracing.JaegerPortUnificationHandler) IdentityHashMap(java.util.IdentityHashMap) TaggedMetricName(com.wavefront.common.TaggedMetricName) ImmutableMap(com.google.common.collect.ImmutableMap) HttpHealthCheckEndpointHandler(com.wavefront.agent.listeners.HttpHealthCheckEndpointHandler) Accumulator(com.wavefront.agent.histogram.accumulator.Accumulator) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) ReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl) Collectors(java.util.stream.Collectors) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) ExpectedAgentMetric(com.wavefront.metrics.ExpectedAgentMetric) ChannelByteArrayHandler(com.wavefront.agent.listeners.ChannelByteArrayHandler) QueueingFactoryImpl(com.wavefront.agent.queueing.QueueingFactoryImpl) JaegerTChannelCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerTChannelCollectorHandler) PreprocessorRuleMetrics(com.wavefront.agent.preprocessor.PreprocessorRuleMetrics) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) HashMap(java.util.HashMap) BindException(java.net.BindException) Function(java.util.function.Function) OpenTSDBPortUnificationHandler(com.wavefront.agent.listeners.OpenTSDBPortUnificationHandler) Level(java.util.logging.Level) JsonMetricsPortUnificationHandler(com.wavefront.agent.listeners.JsonMetricsPortUnificationHandler) TaskQueueFactoryImpl(com.wavefront.agent.queueing.TaskQueueFactoryImpl) ReportPointAddPrefixTransformer(com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer) ImmutableList(com.google.common.collect.ImmutableList) TrafficShapingRateLimitAdjuster(com.wavefront.agent.handlers.TrafficShapingRateLimitAdjuster) ObjectUtils(org.apache.commons.lang3.ObjectUtils) RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) HistogramKeyMarshaller(com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller) EventDecoder(com.wavefront.ingester.EventDecoder) ReportPointDecoderWrapper(com.wavefront.ingester.ReportPointDecoderWrapper) Nonnull(javax.annotation.Nonnull) InternalProxyWavefrontClient(com.wavefront.agent.handlers.InternalProxyWavefrontClient) LengthFieldBasedFrameDecoder(io.netty.handler.codec.LengthFieldBasedFrameDecoder) Counter(com.yammer.metrics.core.Counter) SenderTaskFactory(com.wavefront.agent.handlers.SenderTaskFactory) AgentDigestMarshaller(com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) SpanDecoder(com.wavefront.ingester.SpanDecoder) CorsConfig(io.netty.handler.codec.http.cors.CorsConfig) VALID_HISTOGRAMS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_HISTOGRAMS_LOGGER) HttpMethod(io.netty.handler.codec.http.HttpMethod) AgentDigest(com.tdunning.math.stats.AgentDigest) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) TracePortUnificationHandler(com.wavefront.agent.listeners.tracing.TracePortUnificationHandler) TimeUnit(java.util.concurrent.TimeUnit) Connection(com.uber.tchannel.channels.Connection) RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) SenderTaskFactoryImpl(com.wavefront.agent.handlers.SenderTaskFactoryImpl) JaegerGrpcCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerGrpcCollectorHandler) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Histogram(wavefront.report.Histogram) AgentDigestMarshaller(com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller) PointHandlerDispatcher(com.wavefront.agent.histogram.PointHandlerDispatcher) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) HistogramKey(com.wavefront.agent.histogram.HistogramKey) Nonnull(javax.annotation.Nonnull) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) ReportPoint(wavefront.report.ReportPoint) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) MapLoader(com.wavefront.agent.histogram.MapLoader) AgentDigest(com.tdunning.math.stats.AgentDigest) HistogramKeyMarshaller(com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller) File(java.io.File) Map(java.util.Map) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) IdentityHashMap(java.util.IdentityHashMap) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) TcpIngester(com.wavefront.ingester.TcpIngester)

Aggregations

ReportableEntityHandler (com.wavefront.agent.handlers.ReportableEntityHandler)10 ArrayList (java.util.ArrayList)8 List (java.util.List)8 Nullable (javax.annotation.Nullable)7 ReportPoint (wavefront.report.ReportPoint)7 HandlerKey (com.wavefront.agent.handlers.HandlerKey)6 ReportableEntityHandlerFactory (com.wavefront.agent.handlers.ReportableEntityHandlerFactory)6 SpanSampler (com.wavefront.agent.sampler.SpanSampler)6 ImmutableList (com.google.common.collect.ImmutableList)5 SharedGraphiteHostAnnotator (com.wavefront.agent.channel.SharedGraphiteHostAnnotator)5 ConfigurationException (com.wavefront.agent.config.ConfigurationException)5 ReportableEntityType (com.wavefront.data.ReportableEntityType)5 HashMap (java.util.HashMap)5 Map (java.util.Map)5 AgentDigest (com.tdunning.math.stats.AgentDigest)4 TokenAuthenticator (com.wavefront.agent.auth.TokenAuthenticator)4 CachingHostnameLookupResolver (com.wavefront.agent.channel.CachingHostnameLookupResolver)4 HealthCheckManager (com.wavefront.agent.channel.HealthCheckManager)4 HealthCheckManagerImpl (com.wavefront.agent.channel.HealthCheckManagerImpl)4 WavefrontInternalReporter (com.wavefront.internal.reporter.WavefrontInternalReporter)4