Search in sources :

Example 1 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class PointHandlerDispatcherTest method setup.

@Before
public void setup() {
    timeMillis = new AtomicLong(0L);
    backingStore = new ConcurrentHashMap<>();
    AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> COMPRESSION, 100L, timeMillis::get);
    in = new AccumulationCache(backingStore, agentDigestFactory, 0, "", timeMillis::get);
    pointOut = new LinkedList<>();
    debugLineOut = new LinkedList<>();
    blockedOut = new LinkedList<>();
    digestA = new AgentDigest(COMPRESSION, 100L);
    digestB = new AgentDigest(COMPRESSION, 1000L);
    subject = new PointHandlerDispatcher(in, new ReportableEntityHandler<ReportPoint, String>() {

        @Override
        public void report(ReportPoint reportPoint) {
            pointOut.add(reportPoint);
        }

        @Override
        public void block(ReportPoint reportPoint) {
            blockedOut.add(reportPoint);
        }

        @Override
        public void block(@Nullable ReportPoint reportPoint, @Nullable String message) {
            blockedOut.add(reportPoint);
        }

        @Override
        public void reject(@Nullable ReportPoint reportPoint, @Nullable String message) {
            blockedOut.add(reportPoint);
        }

        @Override
        public void reject(@Nonnull String t, @Nullable String message) {
        }

        @Override
        public void shutdown() {
        }
    }, timeMillis::get, () -> false, null, null);
}
Also used : Nonnull(javax.annotation.Nonnull) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) AtomicLong(java.util.concurrent.atomic.AtomicLong) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) AgentDigest(com.tdunning.math.stats.AgentDigest) ReportPoint(wavefront.report.ReportPoint) Nullable(javax.annotation.Nullable) Before(org.junit.Before)

Example 2 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class PushAgent method startListeners.

@Override
protected void startListeners() throws Exception {
    blockedPointsLogger = Logger.getLogger(proxyConfig.getBlockedPointsLoggerName());
    blockedHistogramsLogger = Logger.getLogger(proxyConfig.getBlockedHistogramsLoggerName());
    blockedSpansLogger = Logger.getLogger(proxyConfig.getBlockedSpansLoggerName());
    if (proxyConfig.getSoLingerTime() >= 0) {
        childChannelOptions.put(ChannelOption.SO_LINGER, proxyConfig.getSoLingerTime());
    }
    hostnameResolver = new CachingHostnameLookupResolver(proxyConfig.isDisableRdnsLookup(), ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName);
    if (proxyConfig.isSqsQueueBuffer()) {
        taskQueueFactory = new SQSQueueFactoryImpl(proxyConfig.getSqsQueueNameTemplate(), proxyConfig.getSqsQueueRegion(), proxyConfig.getSqsQueueIdentifier(), proxyConfig.isPurgeBuffer());
    } else {
        taskQueueFactory = new TaskQueueFactoryImpl(proxyConfig.getBufferFile(), proxyConfig.isPurgeBuffer(), proxyConfig.isDisableBufferSharding(), proxyConfig.getBufferShardSize());
    }
    remoteHostAnnotator = new SharedGraphiteHostAnnotator(proxyConfig.getCustomSourceTags(), hostnameResolver);
    queueingFactory = new QueueingFactoryImpl(apiContainer, agentId, taskQueueFactory, entityProps);
    senderTaskFactory = new SenderTaskFactoryImpl(apiContainer, agentId, taskQueueFactory, queueingFactory, entityProps);
    if (proxyConfig.isHistogramPassthroughRecompression()) {
        histogramRecompressor = new HistogramRecompressor(() -> entityProps.getGlobalProperties().getHistogramStorageAccuracy());
    }
    handlerFactory = new ReportableEntityHandlerFactoryImpl(senderTaskFactory, proxyConfig.getPushBlockedSamples(), validationConfiguration, blockedPointsLogger, blockedHistogramsLogger, blockedSpansLogger, histogramRecompressor, entityProps);
    if (proxyConfig.isTrafficShaping()) {
        new TrafficShapingRateLimitAdjuster(entityProps, proxyConfig.getTrafficShapingWindowSeconds(), proxyConfig.getTrafficShapingHeadroom()).start();
    }
    healthCheckManager = new HealthCheckManagerImpl(proxyConfig);
    tokenAuthenticator = configureTokenAuthenticator();
    shutdownTasks.add(() -> senderTaskFactory.shutdown());
    shutdownTasks.add(() -> senderTaskFactory.drainBuffersToQueue(null));
    // sampler for spans
    rateSampler.setSamplingRate(entityProps.getGlobalProperties().getTraceSamplingRate());
    Sampler durationSampler = SpanSamplerUtils.getDurationSampler(proxyConfig.getTraceSamplingDuration());
    List<Sampler> samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler);
    SpanSampler spanSampler = new SpanSampler(new CompositeSampler(samplers), () -> entityProps.getGlobalProperties().getActiveSpanSamplingPolicies());
    if (proxyConfig.getAdminApiListenerPort() > 0) {
        startAdminListener(proxyConfig.getAdminApiListenerPort());
    }
    csvToList(proxyConfig.getHttpHealthCheckPorts()).forEach(strPort -> startHealthCheckListener(Integer.parseInt(strPort)));
    csvToList(proxyConfig.getPushListenerPorts()).forEach(strPort -> {
        startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront metrics");
    });
    csvToList(proxyConfig.getDeltaCountersAggregationListenerPorts()).forEach(strPort -> {
        startDeltaCounterListener(strPort, remoteHostAnnotator, senderTaskFactory, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront delta counter metrics");
    });
    {
        // Histogram bootstrap.
        List<String> histMinPorts = csvToList(proxyConfig.getHistogramMinuteListenerPorts());
        List<String> histHourPorts = csvToList(proxyConfig.getHistogramHourListenerPorts());
        List<String> histDayPorts = csvToList(proxyConfig.getHistogramDayListenerPorts());
        List<String> histDistPorts = csvToList(proxyConfig.getHistogramDistListenerPorts());
        int activeHistogramAggregationTypes = (histDayPorts.size() > 0 ? 1 : 0) + (histHourPorts.size() > 0 ? 1 : 0) + (histMinPorts.size() > 0 ? 1 : 0) + (histDistPorts.size() > 0 ? 1 : 0);
        if (activeHistogramAggregationTypes > 0) {
            /*Histograms enabled*/
            histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
            histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
            managedExecutors.add(histogramExecutor);
            managedExecutors.add(histogramFlushExecutor);
            File baseDirectory = new File(proxyConfig.getHistogramStateDirectory());
            // Central dispatch
            ReportableEntityHandler<ReportPoint, String> pointHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports"));
            startHistogramListeners(histMinPorts, pointHandler, remoteHostAnnotator, Granularity.MINUTE, proxyConfig.getHistogramMinuteFlushSecs(), proxyConfig.isHistogramMinuteMemoryCache(), baseDirectory, proxyConfig.getHistogramMinuteAccumulatorSize(), proxyConfig.getHistogramMinuteAvgKeyBytes(), proxyConfig.getHistogramMinuteAvgDigestBytes(), proxyConfig.getHistogramMinuteCompression(), proxyConfig.isHistogramMinuteAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histHourPorts, pointHandler, remoteHostAnnotator, Granularity.HOUR, proxyConfig.getHistogramHourFlushSecs(), proxyConfig.isHistogramHourMemoryCache(), baseDirectory, proxyConfig.getHistogramHourAccumulatorSize(), proxyConfig.getHistogramHourAvgKeyBytes(), proxyConfig.getHistogramHourAvgDigestBytes(), proxyConfig.getHistogramHourCompression(), proxyConfig.isHistogramHourAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDayPorts, pointHandler, remoteHostAnnotator, Granularity.DAY, proxyConfig.getHistogramDayFlushSecs(), proxyConfig.isHistogramDayMemoryCache(), baseDirectory, proxyConfig.getHistogramDayAccumulatorSize(), proxyConfig.getHistogramDayAvgKeyBytes(), proxyConfig.getHistogramDayAvgDigestBytes(), proxyConfig.getHistogramDayCompression(), proxyConfig.isHistogramDayAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDistPorts, pointHandler, remoteHostAnnotator, null, proxyConfig.getHistogramDistFlushSecs(), proxyConfig.isHistogramDistMemoryCache(), baseDirectory, proxyConfig.getHistogramDistAccumulatorSize(), proxyConfig.getHistogramDistAvgKeyBytes(), proxyConfig.getHistogramDistAvgDigestBytes(), proxyConfig.getHistogramDistCompression(), proxyConfig.isHistogramDistAccumulatorPersisted(), spanSampler);
        }
    }
    if (StringUtils.isNotBlank(proxyConfig.getGraphitePorts()) || StringUtils.isNotBlank(proxyConfig.getPicklePorts())) {
        if (tokenAuthenticator.authRequired()) {
            logger.warning("Graphite mode is not compatible with HTTP authentication, ignoring");
        } else {
            Preconditions.checkNotNull(proxyConfig.getGraphiteFormat(), "graphiteFormat must be supplied to enable graphite support");
            Preconditions.checkNotNull(proxyConfig.getGraphiteDelimiters(), "graphiteDelimiters must be supplied to enable graphite support");
            GraphiteFormatter graphiteFormatter = new GraphiteFormatter(proxyConfig.getGraphiteFormat(), proxyConfig.getGraphiteDelimiters(), proxyConfig.getGraphiteFieldsToRemove());
            csvToList(proxyConfig.getGraphitePorts()).forEach(strPort -> {
                preprocessors.getSystemPreprocessor(strPort).forPointLine().addTransformer(0, graphiteFormatter);
                startGraphiteListener(strPort, handlerFactory, null, spanSampler);
                logger.info("listening on port: " + strPort + " for graphite metrics");
            });
            csvToList(proxyConfig.getPicklePorts()).forEach(strPort -> startPickleListener(strPort, handlerFactory, graphiteFormatter));
        }
    }
    csvToList(proxyConfig.getOpentsdbPorts()).forEach(strPort -> startOpenTsdbListener(strPort, handlerFactory));
    if (proxyConfig.getDataDogJsonPorts() != null) {
        HttpClient httpClient = HttpClientBuilder.create().useSystemProperties().setUserAgent(proxyConfig.getHttpUserAgent()).setConnectionTimeToLive(1, TimeUnit.MINUTES).setMaxConnPerRoute(100).setMaxConnTotal(100).setRetryHandler(new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)).setDefaultRequestConfig(RequestConfig.custom().setContentCompressionEnabled(true).setRedirectsEnabled(true).setConnectTimeout(proxyConfig.getHttpConnectTimeout()).setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()).setSocketTimeout(proxyConfig.getHttpRequestTimeout()).build()).build();
        csvToList(proxyConfig.getDataDogJsonPorts()).forEach(strPort -> startDataDogListener(strPort, handlerFactory, httpClient));
    }
    csvToList(proxyConfig.getTraceListenerPorts()).forEach(strPort -> startTraceListener(strPort, handlerFactory, spanSampler));
    csvToList(proxyConfig.getCustomTracingListenerPorts()).forEach(strPort -> startCustomTracingListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler));
    csvToList(proxyConfig.getTraceJaegerListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerGrpcListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerGrpcListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerHttpListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerHttpListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceZipkinListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceZipkinListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getPushRelayListenerPorts()).forEach(strPort -> startRelayListener(strPort, handlerFactory, remoteHostAnnotator));
    csvToList(proxyConfig.getJsonListenerPorts()).forEach(strPort -> startJsonListener(strPort, handlerFactory));
    csvToList(proxyConfig.getWriteHttpJsonListenerPorts()).forEach(strPort -> startWriteHttpJsonListener(strPort, handlerFactory));
    // Logs ingestion.
    if (proxyConfig.getFilebeatPort() > 0 || proxyConfig.getRawLogsPort() > 0) {
        if (loadLogsIngestionConfig() != null) {
            logger.info("Initializing logs ingestion");
            try {
                final LogsIngester logsIngester = new LogsIngester(handlerFactory, this::loadLogsIngestionConfig, proxyConfig.getPrefix());
                logsIngester.start();
                if (proxyConfig.getFilebeatPort() > 0) {
                    startLogsIngestionListener(proxyConfig.getFilebeatPort(), logsIngester);
                }
                if (proxyConfig.getRawLogsPort() > 0) {
                    startRawLogsIngestionListener(proxyConfig.getRawLogsPort(), logsIngester);
                }
            } catch (ConfigurationException e) {
                logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
            }
        } else {
            logger.warning("Cannot start logsIngestion: invalid configuration or no config specified");
        }
    }
    setupMemoryGuard();
}
Also used : DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) TaggedMetricName(com.wavefront.common.TaggedMetricName) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) ReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) PreprocessorRuleMetrics(com.wavefront.agent.preprocessor.PreprocessorRuleMetrics) ConfigurationException(com.wavefront.agent.config.ConfigurationException) HistogramRecompressor(com.wavefront.agent.histogram.HistogramRecompressor) ArrayList(java.util.ArrayList) Utils.csvToList(com.wavefront.common.Utils.csvToList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) HealthCheckManagerImpl(com.wavefront.agent.channel.HealthCheckManagerImpl) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) TaskQueueFactoryImpl(com.wavefront.agent.queueing.TaskQueueFactoryImpl) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) NamedThreadFactory(com.wavefront.common.NamedThreadFactory) SpanSampler(com.wavefront.agent.sampler.SpanSampler) SQSQueueFactoryImpl(com.wavefront.agent.queueing.SQSQueueFactoryImpl) TrafficShapingRateLimitAdjuster(com.wavefront.agent.handlers.TrafficShapingRateLimitAdjuster) QueueingFactoryImpl(com.wavefront.agent.queueing.QueueingFactoryImpl) CachingHostnameLookupResolver(com.wavefront.agent.channel.CachingHostnameLookupResolver) SenderTaskFactoryImpl(com.wavefront.agent.handlers.SenderTaskFactoryImpl) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) SpanSampler(com.wavefront.agent.sampler.SpanSampler) Sampler(com.wavefront.sdk.entities.tracing.sampling.Sampler) RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) HttpClient(org.apache.http.client.HttpClient) SpanSanitizeTransformer(com.wavefront.agent.preprocessor.SpanSanitizeTransformer) File(java.io.File) InternalProxyWavefrontClient(com.wavefront.agent.handlers.InternalProxyWavefrontClient)

Example 3 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class PushAgent method startDeltaCounterListener.

@VisibleForTesting
protected void startDeltaCounterListener(String strPort, SharedGraphiteHostAnnotator hostAnnotator, SenderTaskFactory senderTaskFactory, SpanSampler sampler) {
    final int port = Integer.parseInt(strPort);
    registerPrefixFilter(strPort);
    registerTimestampFilter(strPort);
    if (proxyConfig.isHttpHealthCheckAllPorts())
        healthCheckManager.enableHealthcheck(port);
    if (this.deltaCounterHandlerFactory == null) {
        this.deltaCounterHandlerFactory = new ReportableEntityHandlerFactory() {

            private final Map<String, ReportableEntityHandler<?, ?>> handlers = new ConcurrentHashMap<>();

            @Override
            public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
                // noinspection unchecked
                return (ReportableEntityHandler<T, U>) handlers.computeIfAbsent(handlerKey.getHandle(), k -> new DeltaCounterAccumulationHandlerImpl(handlerKey, proxyConfig.getPushBlockedSamples(), senderTaskFactory.createSenderTasks(handlerKey), validationConfiguration, proxyConfig.getDeltaCountersAggregationIntervalSeconds(), rate -> entityProps.get(ReportableEntityType.POINT).reportReceivedRate(handlerKey.getHandle(), rate), blockedPointsLogger, VALID_POINTS_LOGGER));
            }

            @Override
            public void shutdown(@Nonnull String handle) {
                if (handlers.containsKey(handle)) {
                    handlers.values().forEach(ReportableEntityHandler::shutdown);
                }
            }
        };
    }
    shutdownTasks.add(() -> deltaCounterHandlerFactory.shutdown(strPort));
    WavefrontPortUnificationHandler wavefrontPortUnificationHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, decoderSupplier.get(), deltaCounterHandlerFactory, hostAnnotator, preprocessors.get(strPort), () -> false, () -> false, () -> false, sampler);
    startAsManagedThread(port, new TcpIngester(createInitializer(wavefrontPortUnificationHandler, port, proxyConfig.getPushListenerMaxReceivedLength(), proxyConfig.getPushListenerHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-deltaCounter-" + port);
}
Also used : HandlerKey(com.wavefront.agent.handlers.HandlerKey) QueueingReason(com.wavefront.agent.data.QueueingReason) CustomTracingPortUnificationHandler(com.wavefront.agent.listeners.tracing.CustomTracingPortUnificationHandler) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) EntityProperties(com.wavefront.agent.data.EntityProperties) NettyServerBuilder(io.grpc.netty.NettyServerBuilder) RequestConfig(org.apache.http.client.config.RequestConfig) StringUtils(org.apache.commons.lang3.StringUtils) SpanSanitizeTransformer(com.wavefront.agent.preprocessor.SpanSanitizeTransformer) BooleanUtils(org.apache.commons.lang.BooleanUtils) InetAddress(java.net.InetAddress) SQSQueueFactoryImpl(com.wavefront.agent.queueing.SQSQueueFactoryImpl) PointHandlerDispatcher(com.wavefront.agent.histogram.PointHandlerDispatcher) Map(java.util.Map) HandlerKey(com.wavefront.agent.handlers.HandlerKey) ByteArrayDecoder(io.netty.handler.codec.bytes.ByteArrayDecoder) RecyclableRateLimiter(com.google.common.util.concurrent.RecyclableRateLimiter) HealthCheckManagerImpl(com.wavefront.agent.channel.HealthCheckManagerImpl) HistogramKey(com.wavefront.agent.histogram.HistogramKey) AdminPortUnificationHandler(com.wavefront.agent.listeners.AdminPortUnificationHandler) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) ProxyUtil.createInitializer(com.wavefront.agent.ProxyUtil.createInitializer) Executors(java.util.concurrent.Executors) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ByteOrder(java.nio.ByteOrder) AgentConfiguration(com.wavefront.api.agent.AgentConfiguration) OpenTSDBDecoder(com.wavefront.ingester.OpenTSDBDecoder) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) HealthCheckManager(com.wavefront.agent.channel.HealthCheckManager) ChannelOption(io.netty.channel.ChannelOption) SpanSampler(com.wavefront.agent.sampler.SpanSampler) TChannel(com.uber.tchannel.api.TChannel) Supplier(java.util.function.Supplier) RelayPortUnificationHandler(com.wavefront.agent.listeners.RelayPortUnificationHandler) TcpIngester(com.wavefront.ingester.TcpIngester) ArrayList(java.util.ArrayList) HttpClient(org.apache.http.client.HttpClient) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConfigurationException(com.wavefront.agent.config.ConfigurationException) Utils.lazySupplier(com.wavefront.common.Utils.lazySupplier) Utils.csvToList(com.wavefront.common.Utils.csvToList) Server(org.logstash.beats.Server) TokenAuthenticatorBuilder(com.wavefront.agent.auth.TokenAuthenticatorBuilder) MapLoader(com.wavefront.agent.histogram.MapLoader) Nullable(javax.annotation.Nullable) HistogramUtils(com.wavefront.agent.histogram.HistogramUtils) DataDogPortUnificationHandler(com.wavefront.agent.listeners.DataDogPortUnificationHandler) SslContext(io.netty.handler.ssl.SslContext) WavefrontSender(com.wavefront.sdk.common.WavefrontSender) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) File(java.io.File) CachingHostnameLookupResolver(com.wavefront.agent.channel.CachingHostnameLookupResolver) NamedThreadFactory(com.wavefront.common.NamedThreadFactory) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) ReportSourceTagDecoder(com.wavefront.ingester.ReportSourceTagDecoder) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) VALID_POINTS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_POINTS_LOGGER) Preconditions(com.google.common.base.Preconditions) WriteHttpJsonPortUnificationHandler(com.wavefront.agent.listeners.WriteHttpJsonPortUnificationHandler) Metrics(com.yammer.metrics.Metrics) HttpClientBuilder(org.apache.http.impl.client.HttpClientBuilder) QueueingFactory(com.wavefront.agent.queueing.QueueingFactory) SpanSamplerUtils(com.wavefront.agent.sampler.SpanSamplerUtils) ReportPointDecoder(com.wavefront.ingester.ReportPointDecoder) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) SpanLogsDecoder(com.wavefront.ingester.SpanLogsDecoder) Sampler(com.wavefront.sdk.entities.tracing.sampling.Sampler) ZipkinPortUnificationHandler(com.wavefront.agent.listeners.tracing.ZipkinPortUnificationHandler) TaskQueueFactory(com.wavefront.agent.queueing.TaskQueueFactory) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TokenAuthenticator(com.wavefront.agent.auth.TokenAuthenticator) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) Granularity(com.wavefront.agent.histogram.Granularity) DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) ReportPoint(wavefront.report.ReportPoint) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) HistogramRecompressor(com.wavefront.agent.histogram.HistogramRecompressor) CorsConfigBuilder(io.netty.handler.codec.http.cors.CorsConfigBuilder) FilebeatIngester(com.wavefront.agent.logsharvesting.FilebeatIngester) PickleProtocolDecoder(com.wavefront.ingester.PickleProtocolDecoder) JaegerPortUnificationHandler(com.wavefront.agent.listeners.tracing.JaegerPortUnificationHandler) IdentityHashMap(java.util.IdentityHashMap) TaggedMetricName(com.wavefront.common.TaggedMetricName) ImmutableMap(com.google.common.collect.ImmutableMap) HttpHealthCheckEndpointHandler(com.wavefront.agent.listeners.HttpHealthCheckEndpointHandler) Accumulator(com.wavefront.agent.histogram.accumulator.Accumulator) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) ReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl) Collectors(java.util.stream.Collectors) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) ExpectedAgentMetric(com.wavefront.metrics.ExpectedAgentMetric) ChannelByteArrayHandler(com.wavefront.agent.listeners.ChannelByteArrayHandler) QueueingFactoryImpl(com.wavefront.agent.queueing.QueueingFactoryImpl) JaegerTChannelCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerTChannelCollectorHandler) PreprocessorRuleMetrics(com.wavefront.agent.preprocessor.PreprocessorRuleMetrics) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) HashMap(java.util.HashMap) BindException(java.net.BindException) Function(java.util.function.Function) OpenTSDBPortUnificationHandler(com.wavefront.agent.listeners.OpenTSDBPortUnificationHandler) Level(java.util.logging.Level) JsonMetricsPortUnificationHandler(com.wavefront.agent.listeners.JsonMetricsPortUnificationHandler) TaskQueueFactoryImpl(com.wavefront.agent.queueing.TaskQueueFactoryImpl) ReportPointAddPrefixTransformer(com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer) ImmutableList(com.google.common.collect.ImmutableList) TrafficShapingRateLimitAdjuster(com.wavefront.agent.handlers.TrafficShapingRateLimitAdjuster) ObjectUtils(org.apache.commons.lang3.ObjectUtils) RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) HistogramKeyMarshaller(com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller) EventDecoder(com.wavefront.ingester.EventDecoder) ReportPointDecoderWrapper(com.wavefront.ingester.ReportPointDecoderWrapper) Nonnull(javax.annotation.Nonnull) InternalProxyWavefrontClient(com.wavefront.agent.handlers.InternalProxyWavefrontClient) LengthFieldBasedFrameDecoder(io.netty.handler.codec.LengthFieldBasedFrameDecoder) Counter(com.yammer.metrics.core.Counter) SenderTaskFactory(com.wavefront.agent.handlers.SenderTaskFactory) AgentDigestMarshaller(com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) SpanDecoder(com.wavefront.ingester.SpanDecoder) CorsConfig(io.netty.handler.codec.http.cors.CorsConfig) VALID_HISTOGRAMS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_HISTOGRAMS_LOGGER) HttpMethod(io.netty.handler.codec.http.HttpMethod) AgentDigest(com.tdunning.math.stats.AgentDigest) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) TracePortUnificationHandler(com.wavefront.agent.listeners.tracing.TracePortUnificationHandler) TimeUnit(java.util.concurrent.TimeUnit) Connection(com.uber.tchannel.channels.Connection) RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) SenderTaskFactoryImpl(com.wavefront.agent.handlers.SenderTaskFactoryImpl) JaegerGrpcCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerGrpcCollectorHandler) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Histogram(wavefront.report.Histogram) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) ReportPoint(wavefront.report.ReportPoint) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) TcpIngester(com.wavefront.ingester.TcpIngester) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 4 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class RelayPortUnificationHandler method handleHttpMessage.

@Override
protected void handleHttpMessage(final ChannelHandlerContext ctx, final FullHttpRequest request) {
    URI uri = URI.create(request.uri());
    StringBuilder output = new StringBuilder();
    String path = uri.getPath();
    final boolean isDirectIngestion = path.startsWith("/report");
    if (path.endsWith("/checkin") && (path.startsWith("/api/daemon") || path.contains("wfproxy"))) {
        // simulate checkin response for proxy chaining
        ObjectNode jsonResponse = JsonNodeFactory.instance.objectNode();
        jsonResponse.put("currentTime", Clock.now());
        jsonResponse.put("allowAnyHostKeys", true);
        writeHttpResponse(ctx, HttpResponseStatus.OK, jsonResponse, request);
        return;
    }
    String format = URLEncodedUtils.parse(uri, CharsetUtil.UTF_8).stream().filter(x -> x.getName().equals("format") || x.getName().equals("f")).map(NameValuePair::getValue).findFirst().orElse(Constants.PUSH_FORMAT_WAVEFRONT);
    // Return HTTP 200 (OK) for payloads received on the proxy endpoint
    // Return HTTP 202 (ACCEPTED) for payloads received on the DDI endpoint
    // Return HTTP 204 (NO_CONTENT) for payloads received on all other endpoints
    HttpResponseStatus okStatus;
    if (isDirectIngestion) {
        okStatus = HttpResponseStatus.ACCEPTED;
    } else if (path.contains("/pushdata/") || path.contains("wfproxy/report")) {
        okStatus = HttpResponseStatus.OK;
    } else {
        okStatus = HttpResponseStatus.NO_CONTENT;
    }
    HttpResponseStatus status;
    switch(format) {
        case Constants.PUSH_FORMAT_HISTOGRAM:
            if (isFeatureDisabled(histogramDisabled, HISTO_DISABLED, discardedHistograms.get(), output, request)) {
                status = HttpResponseStatus.FORBIDDEN;
                break;
            }
        case Constants.PUSH_FORMAT_WAVEFRONT:
        case Constants.PUSH_FORMAT_GRAPHITE_V2:
            AtomicBoolean hasSuccessfulPoints = new AtomicBoolean(false);
            try {
                // noinspection unchecked
                ReportableEntityDecoder<String, ReportPoint> histogramDecoder = (ReportableEntityDecoder<String, ReportPoint>) decoders.get(ReportableEntityType.HISTOGRAM);
                Splitter.on('\n').trimResults().omitEmptyStrings().split(request.content().toString(CharsetUtil.UTF_8)).forEach(message -> {
                    DataFormat dataFormat = DataFormat.autodetect(message);
                    switch(dataFormat) {
                        case EVENT:
                            wavefrontHandler.reject(message, "Relay port does not support " + "event-formatted data!");
                            break;
                        case SOURCE_TAG:
                            wavefrontHandler.reject(message, "Relay port does not support " + "sourceTag-formatted data!");
                            break;
                        case HISTOGRAM:
                            if (isFeatureDisabled(histogramDisabled, HISTO_DISABLED, discardedHistograms.get(), output)) {
                                break;
                            }
                            preprocessAndHandlePoint(message, histogramDecoder, histogramHandlerSupplier.get(), preprocessorSupplier, ctx, "histogram");
                            hasSuccessfulPoints.set(true);
                            break;
                        default:
                            // only apply annotator if point received on the DDI endpoint
                            message = annotator != null && isDirectIngestion ? annotator.apply(ctx, message) : message;
                            preprocessAndHandlePoint(message, wavefrontDecoder, wavefrontHandler, preprocessorSupplier, ctx, "metric");
                            hasSuccessfulPoints.set(true);
                            break;
                    }
                });
                status = hasSuccessfulPoints.get() ? okStatus : HttpResponseStatus.BAD_REQUEST;
            } catch (Exception e) {
                status = HttpResponseStatus.BAD_REQUEST;
                output.append(errorMessageWithRootCause(e));
                logWarning("WF-300: Failed to handle HTTP POST", e, ctx);
            }
            break;
        case Constants.PUSH_FORMAT_TRACING:
            if (isFeatureDisabled(traceDisabled, SPAN_DISABLED, discardedSpans.get(), output, request)) {
                receivedSpansTotal.get().inc(discardedSpans.get().count());
                status = HttpResponseStatus.FORBIDDEN;
                break;
            }
            List<Span> spans = new ArrayList<>();
            // noinspection unchecked
            ReportableEntityDecoder<String, Span> spanDecoder = (ReportableEntityDecoder<String, Span>) decoders.get(ReportableEntityType.TRACE);
            ReportableEntityHandler<Span, String> spanHandler = spanHandlerSupplier.get();
            Splitter.on('\n').trimResults().omitEmptyStrings().split(request.content().toString(CharsetUtil.UTF_8)).forEach(line -> {
                try {
                    receivedSpansTotal.get().inc();
                    spanDecoder.decode(line, spans, "dummy");
                } catch (Exception e) {
                    spanHandler.reject(line, formatErrorMessage(line, e, ctx));
                }
            });
            spans.forEach(spanHandler::report);
            status = okStatus;
            break;
        case Constants.PUSH_FORMAT_TRACING_SPAN_LOGS:
            if (isFeatureDisabled(spanLogsDisabled, SPANLOGS_DISABLED, discardedSpanLogs.get(), output, request)) {
                status = HttpResponseStatus.FORBIDDEN;
                break;
            }
            List<SpanLogs> spanLogs = new ArrayList<>();
            // noinspection unchecked
            ReportableEntityDecoder<JsonNode, SpanLogs> spanLogDecoder = (ReportableEntityDecoder<JsonNode, SpanLogs>) decoders.get(ReportableEntityType.TRACE_SPAN_LOGS);
            ReportableEntityHandler<SpanLogs, String> spanLogsHandler = spanLogsHandlerSupplier.get();
            Splitter.on('\n').trimResults().omitEmptyStrings().split(request.content().toString(CharsetUtil.UTF_8)).forEach(line -> {
                try {
                    spanLogDecoder.decode(JSON_PARSER.readTree(line), spanLogs, "dummy");
                } catch (Exception e) {
                    spanLogsHandler.reject(line, formatErrorMessage(line, e, ctx));
                }
            });
            spanLogs.forEach(spanLogsHandler::report);
            status = okStatus;
            break;
        default:
            status = HttpResponseStatus.BAD_REQUEST;
            logger.warning("Unexpected format for incoming HTTP request: " + format);
    }
    writeHttpResponse(ctx, status, output, request);
}
Also used : HealthCheckManager(com.wavefront.agent.channel.HealthCheckManager) FeatureCheckUtils.isFeatureDisabled(com.wavefront.agent.listeners.FeatureCheckUtils.isFeatureDisabled) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SPANLOGS_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPANLOGS_DISABLED) SpanLogs(wavefront.report.SpanLogs) Supplier(java.util.function.Supplier) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) DataFormat(com.wavefront.agent.formatter.DataFormat) ArrayList(java.util.ArrayList) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) TokenAuthenticator(com.wavefront.agent.auth.TokenAuthenticator) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) ChannelUtils.errorMessageWithRootCause(com.wavefront.agent.channel.ChannelUtils.errorMessageWithRootCause) ChannelUtils.writeHttpResponse(com.wavefront.agent.channel.ChannelUtils.writeHttpResponse) Map(java.util.Map) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) HandlerKey(com.wavefront.agent.handlers.HandlerKey) Constants(com.wavefront.api.agent.Constants) CharsetUtil(io.netty.util.CharsetUtil) MetricName(com.yammer.metrics.core.MetricName) ReportPoint(wavefront.report.ReportPoint) JsonNode(com.fasterxml.jackson.databind.JsonNode) Clock(com.wavefront.common.Clock) URI(java.net.URI) Splitter(com.google.common.base.Splitter) Nullable(javax.annotation.Nullable) Counter(com.yammer.metrics.core.Counter) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ReportableEntityPreprocessor(com.wavefront.agent.preprocessor.ReportableEntityPreprocessor) SPAN_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.SPAN_DISABLED) HttpResponseStatus(io.netty.handler.codec.http.HttpResponseStatus) Span(wavefront.report.Span) Logger(java.util.logging.Logger) FullHttpRequest(io.netty.handler.codec.http.FullHttpRequest) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) ChannelUtils.formatErrorMessage(com.wavefront.agent.channel.ChannelUtils.formatErrorMessage) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) JsonNodeFactory(com.fasterxml.jackson.databind.node.JsonNodeFactory) Utils(com.wavefront.common.Utils) URLEncodedUtils(org.apache.http.client.utils.URLEncodedUtils) ChannelHandler(io.netty.channel.ChannelHandler) Metrics(com.yammer.metrics.Metrics) HISTO_DISABLED(com.wavefront.agent.listeners.FeatureCheckUtils.HISTO_DISABLED) NameValuePair(org.apache.http.NameValuePair) WavefrontPortUnificationHandler.preprocessAndHandlePoint(com.wavefront.agent.listeners.WavefrontPortUnificationHandler.preprocessAndHandlePoint) HttpResponseStatus(io.netty.handler.codec.http.HttpResponseStatus) ArrayList(java.util.ArrayList) SpanLogs(wavefront.report.SpanLogs) JsonNode(com.fasterxml.jackson.databind.JsonNode) URI(java.net.URI) Span(wavefront.report.Span) DataFormat(com.wavefront.agent.formatter.DataFormat) NameValuePair(org.apache.http.NameValuePair) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReportPoint(wavefront.report.ReportPoint)

Example 5 with ReportableEntityHandler

use of com.wavefront.agent.handlers.ReportableEntityHandler in project java by wavefrontHQ.

the class InteractiveLogsTester method interactiveTest.

/**
 * Read one line of stdin and print a message to stdout.
 */
@Override
public boolean interactiveTest() throws ConfigurationException {
    final AtomicBoolean reported = new AtomicBoolean(false);
    ReportableEntityHandlerFactory factory = new ReportableEntityHandlerFactory() {

        @SuppressWarnings("unchecked")
        @Override
        public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
            return (ReportableEntityHandler<T, U>) new ReportableEntityHandler<ReportPoint, String>() {

                @Override
                public void report(ReportPoint reportPoint) {
                    reported.set(true);
                    System.out.println(ReportPointSerializer.pointToString(reportPoint));
                }

                @Override
                public void block(ReportPoint reportPoint) {
                    System.out.println("Blocked: " + reportPoint);
                }

                @Override
                public void block(@Nullable ReportPoint reportPoint, @Nullable String message) {
                    System.out.println("Blocked: " + reportPoint);
                }

                @Override
                public void reject(@Nullable ReportPoint reportPoint, @Nullable String message) {
                    System.out.println("Rejected: " + reportPoint);
                }

                @Override
                public void reject(@Nonnull String t, @Nullable String message) {
                    System.out.println("Rejected: " + t);
                }

                @Override
                public void shutdown() {
                }
            };
        }

        @Override
        public void shutdown(@Nonnull String handle) {
        }
    };
    LogsIngester logsIngester = new LogsIngester(factory, logsIngestionConfigSupplier, prefix);
    String line = stdin.nextLine();
    logsIngester.ingestLog(new LogsMessage() {

        @Override
        public String getLogLine() {
            return line;
        }

        @Override
        public String hostOrDefault(String fallbackHost) {
            try {
                return InetAddress.getLocalHost().getHostName();
            } catch (UnknownHostException e) {
                return "localhost";
            }
        }
    });
    logsIngester.flush();
    if (!reported.get()) {
        System.out.println("Input matched no groks.");
    }
    return stdin.hasNext();
}
Also used : HandlerKey(com.wavefront.agent.handlers.HandlerKey) UnknownHostException(java.net.UnknownHostException) Nonnull(javax.annotation.Nonnull) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReportPoint(wavefront.report.ReportPoint) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory)

Aggregations

ReportableEntityHandler (com.wavefront.agent.handlers.ReportableEntityHandler)10 ArrayList (java.util.ArrayList)8 List (java.util.List)8 Nullable (javax.annotation.Nullable)7 ReportPoint (wavefront.report.ReportPoint)7 HandlerKey (com.wavefront.agent.handlers.HandlerKey)6 ReportableEntityHandlerFactory (com.wavefront.agent.handlers.ReportableEntityHandlerFactory)6 SpanSampler (com.wavefront.agent.sampler.SpanSampler)6 ImmutableList (com.google.common.collect.ImmutableList)5 SharedGraphiteHostAnnotator (com.wavefront.agent.channel.SharedGraphiteHostAnnotator)5 ConfigurationException (com.wavefront.agent.config.ConfigurationException)5 ReportableEntityType (com.wavefront.data.ReportableEntityType)5 HashMap (java.util.HashMap)5 Map (java.util.Map)5 AgentDigest (com.tdunning.math.stats.AgentDigest)4 TokenAuthenticator (com.wavefront.agent.auth.TokenAuthenticator)4 CachingHostnameLookupResolver (com.wavefront.agent.channel.CachingHostnameLookupResolver)4 HealthCheckManager (com.wavefront.agent.channel.HealthCheckManager)4 HealthCheckManagerImpl (com.wavefront.agent.channel.HealthCheckManagerImpl)4 WavefrontInternalReporter (com.wavefront.internal.reporter.WavefrontInternalReporter)4