Examples with TaggedMetricName com.wavefront.common.TaggedMetricName used on opensource projects

Example 6 with TaggedMetricName

use of com.wavefront.common.TaggedMetricName in project java by wavefrontHQ.

the class WavefrontYammerMetricsReporterTest method testTimerWithClear.

@Test(timeout = 1000)
public void testTimerWithClear() throws Exception {
    innerSetUp(false, null, false, true);
    Timer timer = metricsRegistry.newTimer(new TaggedMetricName("", "mytimer", "foo", "bar"), TimeUnit.SECONDS, TimeUnit.SECONDS);
    timer.time().stop();
    wavefrontYammerMetricsReporter.run();
    assertThat(receiveFromSocket(15, fromMetrics), containsInAnyOrder(equalTo("\"mytimer.rate.count\" 1.0 1485224035 foo=\"bar\""), startsWith("\"mytimer.duration.min\""), startsWith("\"mytimer.duration.max\""), startsWith("\"mytimer.duration.mean\""), startsWith("\"mytimer.duration.sum\""), startsWith("\"mytimer.duration.stddev\""), startsWith("\"mytimer.duration.median\""), startsWith("\"mytimer.duration.p75\""), startsWith("\"mytimer.duration.p95\""), startsWith("\"mytimer.duration.p99\""), startsWith("\"mytimer.duration.p999\""), startsWith("\"mytimer.rate.m1\""), startsWith("\"mytimer.rate.m5\""), startsWith("\"mytimer.rate.m15\""), startsWith("\"mytimer.rate.mean\"")));
    wavefrontYammerMetricsReporter.run();
    assertThat(receiveFromSocket(15, fromMetrics), hasItem("\"mytimer.rate.count\" 0.0 1485224035 foo=\"bar\""));
}

Example 7 with TaggedMetricName

use of com.wavefront.common.TaggedMetricName in project java by wavefrontHQ.

the class QueueController method run.

@Override
public void run() {
    // 1. grab current queue sizes (tasks count) and report to EntityProperties
    int backlog = processorTasks.stream().mapToInt(x -> x.getTaskQueue().size()).sum();
    queueSize.set(backlog);
    if (backlogSizeSink != null) {
        backlogSizeSink.accept(backlog);
    }
    // 2. grab queue sizes (points/etc count)
    Long totalWeight = 0L;
    for (QueueProcessor<T> task : processorTasks) {
        TaskQueue<T> taskQueue = task.getTaskQueue();
        // noinspection ConstantConditions
        totalWeight = taskQueue.weight() == null ? null : taskQueue.weight() + totalWeight;
        if (totalWeight == null)
            break;
    }
    if (totalWeight != null) {
        if (currentWeight == null) {
            currentWeight = new AtomicLong();
            Metrics.newGauge(new TaggedMetricName("buffer", handlerKey.getEntityType() + "-count", "port", handlerKey.getHandle()), new Gauge<Long>() {

                @Override
                public Long value() {
                    return currentWeight.get();
                }
            });
        }
        currentWeight.set(totalWeight);
    }
    // 3. adjust timing
    adjustTimingFactors(processorTasks);
    // 4. print stats when there's backlog
    if (backlog > 0) {
        printQueueStats();
    } else if (outputQueueingStats) {
        outputQueueingStats = false;
        logger.info("[" + handlerKey.getHandle() + "] " + handlerKey.getEntityType() + " backlog has been cleared!");
    }
}

Example 8 with TaggedMetricName

use of com.wavefront.common.TaggedMetricName in project java by wavefrontHQ.

the class PushAgent method startListeners.

@Override
protected void startListeners() throws Exception {
    blockedPointsLogger = Logger.getLogger(proxyConfig.getBlockedPointsLoggerName());
    blockedHistogramsLogger = Logger.getLogger(proxyConfig.getBlockedHistogramsLoggerName());
    blockedSpansLogger = Logger.getLogger(proxyConfig.getBlockedSpansLoggerName());
    if (proxyConfig.getSoLingerTime() >= 0) {
        childChannelOptions.put(ChannelOption.SO_LINGER, proxyConfig.getSoLingerTime());
    }
    hostnameResolver = new CachingHostnameLookupResolver(proxyConfig.isDisableRdnsLookup(), ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName);
    if (proxyConfig.isSqsQueueBuffer()) {
        taskQueueFactory = new SQSQueueFactoryImpl(proxyConfig.getSqsQueueNameTemplate(), proxyConfig.getSqsQueueRegion(), proxyConfig.getSqsQueueIdentifier(), proxyConfig.isPurgeBuffer());
    } else {
        taskQueueFactory = new TaskQueueFactoryImpl(proxyConfig.getBufferFile(), proxyConfig.isPurgeBuffer(), proxyConfig.isDisableBufferSharding(), proxyConfig.getBufferShardSize());
    }
    remoteHostAnnotator = new SharedGraphiteHostAnnotator(proxyConfig.getCustomSourceTags(), hostnameResolver);
    queueingFactory = new QueueingFactoryImpl(apiContainer, agentId, taskQueueFactory, entityProps);
    senderTaskFactory = new SenderTaskFactoryImpl(apiContainer, agentId, taskQueueFactory, queueingFactory, entityProps);
    if (proxyConfig.isHistogramPassthroughRecompression()) {
        histogramRecompressor = new HistogramRecompressor(() -> entityProps.getGlobalProperties().getHistogramStorageAccuracy());
    }
    handlerFactory = new ReportableEntityHandlerFactoryImpl(senderTaskFactory, proxyConfig.getPushBlockedSamples(), validationConfiguration, blockedPointsLogger, blockedHistogramsLogger, blockedSpansLogger, histogramRecompressor, entityProps);
    if (proxyConfig.isTrafficShaping()) {
        new TrafficShapingRateLimitAdjuster(entityProps, proxyConfig.getTrafficShapingWindowSeconds(), proxyConfig.getTrafficShapingHeadroom()).start();
    }
    healthCheckManager = new HealthCheckManagerImpl(proxyConfig);
    tokenAuthenticator = configureTokenAuthenticator();
    shutdownTasks.add(() -> senderTaskFactory.shutdown());
    shutdownTasks.add(() -> senderTaskFactory.drainBuffersToQueue(null));
    // sampler for spans
    rateSampler.setSamplingRate(entityProps.getGlobalProperties().getTraceSamplingRate());
    Sampler durationSampler = SpanSamplerUtils.getDurationSampler(proxyConfig.getTraceSamplingDuration());
    List<Sampler> samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler);
    SpanSampler spanSampler = new SpanSampler(new CompositeSampler(samplers), () -> entityProps.getGlobalProperties().getActiveSpanSamplingPolicies());
    if (proxyConfig.getAdminApiListenerPort() > 0) {
        startAdminListener(proxyConfig.getAdminApiListenerPort());
    }
    csvToList(proxyConfig.getHttpHealthCheckPorts()).forEach(strPort -> startHealthCheckListener(Integer.parseInt(strPort)));
    csvToList(proxyConfig.getPushListenerPorts()).forEach(strPort -> {
        startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront metrics");
    });
    csvToList(proxyConfig.getDeltaCountersAggregationListenerPorts()).forEach(strPort -> {
        startDeltaCounterListener(strPort, remoteHostAnnotator, senderTaskFactory, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront delta counter metrics");
    });
    {
        // Histogram bootstrap.
        List<String> histMinPorts = csvToList(proxyConfig.getHistogramMinuteListenerPorts());
        List<String> histHourPorts = csvToList(proxyConfig.getHistogramHourListenerPorts());
        List<String> histDayPorts = csvToList(proxyConfig.getHistogramDayListenerPorts());
        List<String> histDistPorts = csvToList(proxyConfig.getHistogramDistListenerPorts());
        int activeHistogramAggregationTypes = (histDayPorts.size() > 0 ? 1 : 0) + (histHourPorts.size() > 0 ? 1 : 0) + (histMinPorts.size() > 0 ? 1 : 0) + (histDistPorts.size() > 0 ? 1 : 0);
        if (activeHistogramAggregationTypes > 0) {
            /*Histograms enabled*/
            histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
            histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
            managedExecutors.add(histogramExecutor);
            managedExecutors.add(histogramFlushExecutor);
            File baseDirectory = new File(proxyConfig.getHistogramStateDirectory());
            // Central dispatch
            ReportableEntityHandler<ReportPoint, String> pointHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports"));
            startHistogramListeners(histMinPorts, pointHandler, remoteHostAnnotator, Granularity.MINUTE, proxyConfig.getHistogramMinuteFlushSecs(), proxyConfig.isHistogramMinuteMemoryCache(), baseDirectory, proxyConfig.getHistogramMinuteAccumulatorSize(), proxyConfig.getHistogramMinuteAvgKeyBytes(), proxyConfig.getHistogramMinuteAvgDigestBytes(), proxyConfig.getHistogramMinuteCompression(), proxyConfig.isHistogramMinuteAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histHourPorts, pointHandler, remoteHostAnnotator, Granularity.HOUR, proxyConfig.getHistogramHourFlushSecs(), proxyConfig.isHistogramHourMemoryCache(), baseDirectory, proxyConfig.getHistogramHourAccumulatorSize(), proxyConfig.getHistogramHourAvgKeyBytes(), proxyConfig.getHistogramHourAvgDigestBytes(), proxyConfig.getHistogramHourCompression(), proxyConfig.isHistogramHourAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDayPorts, pointHandler, remoteHostAnnotator, Granularity.DAY, proxyConfig.getHistogramDayFlushSecs(), proxyConfig.isHistogramDayMemoryCache(), baseDirectory, proxyConfig.getHistogramDayAccumulatorSize(), proxyConfig.getHistogramDayAvgKeyBytes(), proxyConfig.getHistogramDayAvgDigestBytes(), proxyConfig.getHistogramDayCompression(), proxyConfig.isHistogramDayAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDistPorts, pointHandler, remoteHostAnnotator, null, proxyConfig.getHistogramDistFlushSecs(), proxyConfig.isHistogramDistMemoryCache(), baseDirectory, proxyConfig.getHistogramDistAccumulatorSize(), proxyConfig.getHistogramDistAvgKeyBytes(), proxyConfig.getHistogramDistAvgDigestBytes(), proxyConfig.getHistogramDistCompression(), proxyConfig.isHistogramDistAccumulatorPersisted(), spanSampler);
        }
    }
    if (StringUtils.isNotBlank(proxyConfig.getGraphitePorts()) || StringUtils.isNotBlank(proxyConfig.getPicklePorts())) {
        if (tokenAuthenticator.authRequired()) {
            logger.warning("Graphite mode is not compatible with HTTP authentication, ignoring");
        } else {
            Preconditions.checkNotNull(proxyConfig.getGraphiteFormat(), "graphiteFormat must be supplied to enable graphite support");
            Preconditions.checkNotNull(proxyConfig.getGraphiteDelimiters(), "graphiteDelimiters must be supplied to enable graphite support");
            GraphiteFormatter graphiteFormatter = new GraphiteFormatter(proxyConfig.getGraphiteFormat(), proxyConfig.getGraphiteDelimiters(), proxyConfig.getGraphiteFieldsToRemove());
            csvToList(proxyConfig.getGraphitePorts()).forEach(strPort -> {
                preprocessors.getSystemPreprocessor(strPort).forPointLine().addTransformer(0, graphiteFormatter);
                startGraphiteListener(strPort, handlerFactory, null, spanSampler);
                logger.info("listening on port: " + strPort + " for graphite metrics");
            });
            csvToList(proxyConfig.getPicklePorts()).forEach(strPort -> startPickleListener(strPort, handlerFactory, graphiteFormatter));
        }
    }
    csvToList(proxyConfig.getOpentsdbPorts()).forEach(strPort -> startOpenTsdbListener(strPort, handlerFactory));
    if (proxyConfig.getDataDogJsonPorts() != null) {
        HttpClient httpClient = HttpClientBuilder.create().useSystemProperties().setUserAgent(proxyConfig.getHttpUserAgent()).setConnectionTimeToLive(1, TimeUnit.MINUTES).setMaxConnPerRoute(100).setMaxConnTotal(100).setRetryHandler(new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)).setDefaultRequestConfig(RequestConfig.custom().setContentCompressionEnabled(true).setRedirectsEnabled(true).setConnectTimeout(proxyConfig.getHttpConnectTimeout()).setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()).setSocketTimeout(proxyConfig.getHttpRequestTimeout()).build()).build();
        csvToList(proxyConfig.getDataDogJsonPorts()).forEach(strPort -> startDataDogListener(strPort, handlerFactory, httpClient));
    }
    csvToList(proxyConfig.getTraceListenerPorts()).forEach(strPort -> startTraceListener(strPort, handlerFactory, spanSampler));
    csvToList(proxyConfig.getCustomTracingListenerPorts()).forEach(strPort -> startCustomTracingListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler));
    csvToList(proxyConfig.getTraceJaegerListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerGrpcListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerGrpcListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerHttpListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerHttpListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceZipkinListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceZipkinListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getPushRelayListenerPorts()).forEach(strPort -> startRelayListener(strPort, handlerFactory, remoteHostAnnotator));
    csvToList(proxyConfig.getJsonListenerPorts()).forEach(strPort -> startJsonListener(strPort, handlerFactory));
    csvToList(proxyConfig.getWriteHttpJsonListenerPorts()).forEach(strPort -> startWriteHttpJsonListener(strPort, handlerFactory));
    // Logs ingestion.
    if (proxyConfig.getFilebeatPort() > 0 || proxyConfig.getRawLogsPort() > 0) {
        if (loadLogsIngestionConfig() != null) {
            logger.info("Initializing logs ingestion");
            try {
                final LogsIngester logsIngester = new LogsIngester(handlerFactory, this::loadLogsIngestionConfig, proxyConfig.getPrefix());
                logsIngester.start();
                if (proxyConfig.getFilebeatPort() > 0) {
                    startLogsIngestionListener(proxyConfig.getFilebeatPort(), logsIngester);
                }
                if (proxyConfig.getRawLogsPort() > 0) {
                    startRawLogsIngestionListener(proxyConfig.getRawLogsPort(), logsIngester);
                }
            } catch (ConfigurationException e) {
                logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
            }
        } else {
            logger.warning("Cannot start logsIngestion: invalid configuration or no config specified");
        }
    }
    setupMemoryGuard();
}

Example 9 with TaggedMetricName

use of com.wavefront.common.TaggedMetricName in project java by wavefrontHQ.

the class ProxyUtil method createInitializer.

/**
 * Create a {@link ChannelInitializer} with multiple dynamically created
 * {@link ChannelHandler} objects.
 *
 * @param channelHandlerSuppliers Suppliers of ChannelHandlers.
 * @param port                    port number.
 * @param idleTimeout             idle timeout in seconds.
 * @param sslContext              SSL context.
 * @return channel initializer
 */
static ChannelInitializer<SocketChannel> createInitializer(Iterable<Supplier<ChannelHandler>> channelHandlerSuppliers, int port, int idleTimeout, @Nullable SslContext sslContext) {
    String strPort = String.valueOf(port);
    ChannelHandler idleStateEventHandler = new IdleStateEventHandler(Metrics.newCounter(new TaggedMetricName("listeners", "connections.idle.closed", "port", strPort)));
    ChannelHandler connectionTracker = new ConnectionTrackingHandler(Metrics.newCounter(new TaggedMetricName("listeners", "connections.accepted", "port", strPort)), Metrics.newCounter(new TaggedMetricName("listeners", "connections.active", "port", strPort)));
    if (sslContext != null) {
        logger.info("TLS enabled on port: " + port);
    }
    return new ChannelInitializer<SocketChannel>() {

        @Override
        public void initChannel(SocketChannel ch) {
            ChannelPipeline pipeline = ch.pipeline();
            if (sslContext != null) {
                pipeline.addLast(sslContext.newHandler(ch.alloc()));
            }
            pipeline.addFirst("idlehandler", new IdleStateHandler(idleTimeout, 0, 0));
            pipeline.addLast("idlestateeventhandler", idleStateEventHandler);
            pipeline.addLast("connectiontracker", connectionTracker);
            channelHandlerSuppliers.forEach(x -> pipeline.addLast(x.get()));
        }
    };
}

Example 10 with TaggedMetricName

use of com.wavefront.common.TaggedMetricName in project java by wavefrontHQ.

the class AbstractDataSubmissionTask method execute.

public TaskResult execute() {
    if (enqueuedTimeMillis < Long.MAX_VALUE) {
        if (timeSpentInQueue == null) {
            timeSpentInQueue = Metrics.newHistogram(new TaggedMetricName("buffer", "queue-time", "port", handle, "content", entityType.toString()));
        }
        timeSpentInQueue.update(timeProvider.get() - enqueuedTimeMillis);
    }
    attempts += 1;
    TimerContext timer = Metrics.newTimer(new MetricName("push." + handle, "", "duration"), TimeUnit.MILLISECONDS, TimeUnit.MINUTES).time();
    try (Response response = doExecute()) {
        Metrics.newCounter(new TaggedMetricName("push", handle + ".http." + response.getStatus() + ".count")).inc();
        if (response.getStatus() >= 200 && response.getStatus() < 300) {
            Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered")).inc(this.weight());
            return TaskResult.DELIVERED;
        }
        switch(response.getStatus()) {
            case 406:
            case 429:
                if (enqueuedTimeMillis == Long.MAX_VALUE) {
                    if (properties.getTaskQueueLevel().isLessThan(TaskQueueLevel.PUSHBACK)) {
                        return TaskResult.RETRY_LATER;
                    }
                    enqueue(QueueingReason.PUSHBACK);
                    return TaskResult.PERSISTED;
                }
                if (properties.isSplitPushWhenRateLimited()) {
                    List<T> splitTasks = splitTask(properties.getMinBatchSplitSize(), properties.getItemsPerBatch());
                    if (splitTasks.size() == 1)
                        return TaskResult.RETRY_LATER;
                    splitTasks.forEach(x -> x.enqueue(null));
                    return TaskResult.PERSISTED;
                }
                return TaskResult.RETRY_LATER;
            case 401:
            case 403:
                log.warning("[" + handle + "] HTTP " + response.getStatus() + ": " + "Please verify that \"" + entityType + "\" is enabled for your account!");
                return checkStatusAndQueue(QueueingReason.AUTH, false);
            case 407:
            case 408:
                if (isWavefrontResponse(response)) {
                    log.warning("[" + handle + "] HTTP " + response.getStatus() + " (Unregistered proxy) " + "received while sending data to Wavefront - please verify that your token is " + "valid and has Proxy Management permissions!");
                } else {
                    log.warning("[" + handle + "] HTTP " + response.getStatus() + " " + "received while sending data to Wavefront - please verify your network/HTTP proxy" + " settings!");
                }
                return checkStatusAndQueue(QueueingReason.RETRY, false);
            case 413:
                splitTask(1, properties.getItemsPerBatch()).forEach(x -> x.enqueue(enqueuedTimeMillis == Long.MAX_VALUE ? QueueingReason.SPLIT : null));
                return TaskResult.PERSISTED_RETRY;
            default:
                serverErrors += 1;
                if (serverErrors > MAX_RETRIES && TRUE.equals(limitRetries)) {
                    log.info("[" + handle + "] HTTP " + response.getStatus() + " received while sending " + "data to Wavefront, max retries reached");
                    return TaskResult.DELIVERED;
                } else {
                    log.info("[" + handle + "] HTTP " + response.getStatus() + " received while sending " + "data to Wavefront, retrying");
                    return checkStatusAndQueue(QueueingReason.RETRY, true);
                }
        }
    } catch (DataSubmissionException ex) {
        if (ex instanceof IgnoreStatusCodeException) {
            Metrics.newCounter(new TaggedMetricName("push", handle + ".http.404.count")).inc();
            Metrics.newCounter(new MetricName(entityType + "." + handle, "", "delivered")).inc(this.weight());
            return TaskResult.DELIVERED;
        }
        throw new RuntimeException("Unhandled DataSubmissionException", ex);
    } catch (ProcessingException ex) {
        Throwable rootCause = Throwables.getRootCause(ex);
        if (rootCause instanceof UnknownHostException) {
            log.warning("[" + handle + "] Error sending data to Wavefront: Unknown host " + rootCause.getMessage() + ", please check your network!");
        } else if (rootCause instanceof ConnectException || rootCause instanceof SocketTimeoutException) {
            log.warning("[" + handle + "] Error sending data to Wavefront: " + rootCause.getMessage() + ", please verify your network/HTTP proxy settings!");
        } else if (ex.getCause() instanceof SSLHandshakeException) {
            log.warning("[" + handle + "] Error sending data to Wavefront: " + ex.getCause() + ", please verify that your environment has up-to-date root certificates!");
        } else {
            log.warning("[" + handle + "] Error sending data to Wavefront: " + rootCause);
        }
        if (log.isLoggable(Level.FINE)) {
            log.log(Level.FINE, "Full stacktrace: ", ex);
        }
        return checkStatusAndQueue(QueueingReason.RETRY, false);
    } catch (Exception ex) {
        log.warning("[" + handle + "] Error sending data to Wavefront: " + Throwables.getRootCause(ex));
        if (log.isLoggable(Level.FINE)) {
            log.log(Level.FINE, "Full stacktrace: ", ex);
        }
        return checkStatusAndQueue(QueueingReason.RETRY, true);
    } finally {
        timer.stop();
    }
}