Examples with HistogramKey com.wavefront.agent.histogram.HistogramKey used on opensource projects

Example 1 with HistogramKey

use of com.wavefront.agent.histogram.HistogramKey in project java by wavefrontHQ.

the class HistogramAccumulationHandlerImpl method reportInternal.

@Override
protected void reportInternal(ReportPoint point) {
    validatePoint(point, validationConfig);
    if (point.getValue() instanceof Double) {
        if (granularity == null) {
            pointRejectedCounter.get().inc();
            reject(point, "Wavefront data format is not supported on distribution ports!");
            return;
        }
        // Get key
        HistogramKey histogramKey = HistogramUtils.makeKey(point, granularity);
        double value = (Double) point.getValue();
        pointCounter.get().inc();
        // atomic update
        digests.put(histogramKey, value);
    } else if (point.getValue() instanceof Histogram) {
        Histogram value = (Histogram) point.getValue();
        Granularity pointGranularity = Granularity.fromMillis(value.getDuration());
        if (granularity != null && pointGranularity.getInMillis() > granularity.getInMillis()) {
            reject(point, "Attempting to send coarser granularity (" + granularityToString(pointGranularity) + ") distribution to a finer granularity (" + granularityToString(granularity) + ") port");
            histogramRejectedCounter.get().inc();
            return;
        }
        histogramBinCount.get().update(value.getCounts().size());
        histogramSampleCount.get().update(value.getCounts().stream().mapToLong(x -> x).sum());
        // Key
        HistogramKey histogramKey = HistogramUtils.makeKey(point, granularity == null ? pointGranularity : granularity);
        histogramCounter.get().inc();
        // atomic update
        digests.put(histogramKey, value);
    }
    if (validItemsLogger != null && validItemsLogger.isLoggable(Level.FINEST)) {
        validItemsLogger.info(serializer.apply(point));
    }
}

Example 2 with HistogramKey

use of com.wavefront.agent.histogram.HistogramKey in project java by wavefrontHQ.

the class AccumulationCacheTest method testChronicleMapOverflow.

@Test
public void testChronicleMapOverflow() {
    ConcurrentMap<HistogramKey, AgentDigest> chronicleMap = ChronicleMap.of(HistogramKey.class, AgentDigest.class).keyMarshaller(HistogramKeyMarshaller.get()).valueMarshaller(AgentDigest.AgentDigestMarshaller.get()).entries(10).averageKeySize(20).averageValueSize(20).maxBloatFactor(10).create();
    AtomicBoolean hasFailed = new AtomicBoolean(false);
    AccumulationCache ac = new AccumulationCache(chronicleMap, new AgentDigestFactory(() -> COMPRESSION, 100L, tickerTime::get), 10, "", tickerTime::get, () -> hasFailed.set(true));
    for (int i = 0; i < 1000; i++) {
        ac.put(TestUtils.makeKey("key-" + i), digestA);
        ac.flush();
        if (hasFailed.get()) {
            logger.info("Chronicle map overflow detected when adding object #" + i);
            break;
        }
    }
    assertThat(hasFailed.get()).isTrue();
}

Example 3 with HistogramKey

use of com.wavefront.agent.histogram.HistogramKey in project java by wavefrontHQ.

the class PushAgent method startRelayListener.

@VisibleForTesting
protected void startRelayListener(String strPort, ReportableEntityHandlerFactory handlerFactory, SharedGraphiteHostAnnotator hostAnnotator) {
    final int port = Integer.parseInt(strPort);
    registerPrefixFilter(strPort);
    registerTimestampFilter(strPort);
    if (proxyConfig.isHttpHealthCheckAllPorts())
        healthCheckManager.enableHealthcheck(port);
    ReportableEntityHandlerFactory handlerFactoryDelegate = proxyConfig.isPushRelayHistogramAggregator() ? new DelegatingReportableEntityHandlerFactoryImpl(handlerFactory) {

        @Override
        public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
            if (handlerKey.getEntityType() == ReportableEntityType.HISTOGRAM) {
                ChronicleMap<HistogramKey, AgentDigest> accumulator = ChronicleMap.of(HistogramKey.class, AgentDigest.class).keyMarshaller(HistogramKeyMarshaller.get()).valueMarshaller(AgentDigestMarshaller.get()).entries(proxyConfig.getPushRelayHistogramAggregatorAccumulatorSize()).averageKeySize(proxyConfig.getHistogramDistAvgKeyBytes()).averageValueSize(proxyConfig.getHistogramDistAvgDigestBytes()).maxBloatFactor(1000).create();
                AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> (short) Math.min(proxyConfig.getPushRelayHistogramAggregatorCompression(), entityProps.getGlobalProperties().getHistogramStorageAccuracy()), TimeUnit.SECONDS.toMillis(proxyConfig.getPushRelayHistogramAggregatorFlushSecs()), proxyConfig.getTimeProvider());
                AccumulationCache cachedAccumulator = new AccumulationCache(accumulator, agentDigestFactory, 0, "histogram.accumulator.distributionRelay", null);
                // noinspection unchecked
                return (ReportableEntityHandler<T, U>) new HistogramAccumulationHandlerImpl(handlerKey, cachedAccumulator, proxyConfig.getPushBlockedSamples(), null, validationConfiguration, true, rate -> entityProps.get(ReportableEntityType.HISTOGRAM).reportReceivedRate(handlerKey.getHandle(), rate), blockedHistogramsLogger, VALID_HISTOGRAMS_LOGGER);
            }
            return delegate.getHandler(handlerKey);
        }
    } : handlerFactory;
    Map<ReportableEntityType, ReportableEntityDecoder<?, ?>> filteredDecoders = decoderSupplier.get().entrySet().stream().filter(x -> !x.getKey().equals(ReportableEntityType.SOURCE_TAG)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    ChannelHandler channelHandler = new RelayPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, filteredDecoders, handlerFactoryDelegate, preprocessors.get(strPort), hostAnnotator, () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE_SPAN_LOGS).isFeatureDisabled());
    startAsManagedThread(port, new TcpIngester(createInitializer(channelHandler, port, proxyConfig.getPushListenerMaxReceivedLength(), proxyConfig.getPushListenerHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-relay-" + port);
}

Example 4 with HistogramKey

use of com.wavefront.agent.histogram.HistogramKey in project java by wavefrontHQ.

the class PushAgent method startHistogramListeners.

protected void startHistogramListeners(List<String> ports, ReportableEntityHandler<ReportPoint, String> pointHandler, SharedGraphiteHostAnnotator hostAnnotator, @Nullable Granularity granularity, int flushSecs, boolean memoryCacheEnabled, File baseDirectory, Long accumulatorSize, int avgKeyBytes, int avgDigestBytes, short compression, boolean persist, SpanSampler sampler) throws Exception {
    if (ports.size() == 0)
        return;
    String listenerBinType = HistogramUtils.granularityToString(granularity);
    // Accumulator
    if (persist) {
        // Check directory
        checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!");
        checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!");
    }
    MapLoader<HistogramKey, AgentDigest, HistogramKeyMarshaller, AgentDigestMarshaller> mapLoader = new MapLoader<>(HistogramKey.class, AgentDigest.class, accumulatorSize, avgKeyBytes, avgDigestBytes, HistogramKeyMarshaller.get(), AgentDigestMarshaller.get(), persist);
    File accumulationFile = new File(baseDirectory, "accumulator." + listenerBinType);
    ChronicleMap<HistogramKey, AgentDigest> accumulator = mapLoader.get(accumulationFile);
    histogramExecutor.scheduleWithFixedDelay(() -> {
        // as ChronicleMap starts losing efficiency
        if (accumulator.size() > accumulatorSize * 5) {
            logger.severe("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 5x higher than currently configured size (" + accumulatorSize + "), which may cause severe performance degradation issues " + "or data loss! If the data volume is expected to stay at this level, we strongly " + "recommend increasing the value for accumulator size in wavefront.conf and " + "restarting the proxy.");
        } else if (accumulator.size() > accumulatorSize * 2) {
            logger.warning("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 2x higher than currently configured size (" + accumulatorSize + "), which may cause performance issues. If the data volume is " + "expected to stay at this level, we strongly recommend increasing the value " + "for accumulator size in wavefront.conf and restarting the proxy.");
        }
    }, 10, 10, TimeUnit.SECONDS);
    AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> (short) Math.min(compression, entityProps.getGlobalProperties().getHistogramStorageAccuracy()), TimeUnit.SECONDS.toMillis(flushSecs), proxyConfig.getTimeProvider());
    Accumulator cachedAccumulator = new AccumulationCache(accumulator, agentDigestFactory, (memoryCacheEnabled ? accumulatorSize : 0), "histogram.accumulator." + HistogramUtils.granularityToString(granularity), null);
    // Schedule write-backs
    histogramExecutor.scheduleWithFixedDelay(cachedAccumulator::flush, proxyConfig.getHistogramAccumulatorResolveInterval(), proxyConfig.getHistogramAccumulatorResolveInterval(), TimeUnit.MILLISECONDS);
    histogramFlushRunnables.add(cachedAccumulator::flush);
    PointHandlerDispatcher dispatcher = new PointHandlerDispatcher(cachedAccumulator, pointHandler, proxyConfig.getTimeProvider(), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), proxyConfig.getHistogramAccumulatorFlushMaxBatchSize() < 0 ? null : proxyConfig.getHistogramAccumulatorFlushMaxBatchSize(), granularity);
    histogramExecutor.scheduleWithFixedDelay(dispatcher, proxyConfig.getHistogramAccumulatorFlushInterval(), proxyConfig.getHistogramAccumulatorFlushInterval(), TimeUnit.MILLISECONDS);
    histogramFlushRunnables.add(dispatcher);
    // gracefully shutdown persisted accumulator (ChronicleMap) on proxy exit
    shutdownTasks.add(() -> {
        try {
            logger.fine("Flushing in-flight histogram accumulator digests: " + listenerBinType);
            cachedAccumulator.flush();
            logger.fine("Shutting down histogram accumulator cache: " + listenerBinType);
            accumulator.close();
        } catch (Throwable t) {
            logger.log(Level.SEVERE, "Error flushing " + listenerBinType + " accumulator, possibly unclean shutdown: ", t);
        }
    });
    ReportableEntityHandlerFactory histogramHandlerFactory = new ReportableEntityHandlerFactory() {

        private final Map<HandlerKey, ReportableEntityHandler<?, ?>> handlers = new ConcurrentHashMap<>();

        @SuppressWarnings("unchecked")
        @Override
        public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
            return (ReportableEntityHandler<T, U>) handlers.computeIfAbsent(handlerKey, k -> new HistogramAccumulationHandlerImpl(handlerKey, cachedAccumulator, proxyConfig.getPushBlockedSamples(), granularity, validationConfiguration, granularity == null, null, blockedHistogramsLogger, VALID_HISTOGRAMS_LOGGER));
        }

        @Override
        public void shutdown(@Nonnull String handle) {
            handlers.values().forEach(ReportableEntityHandler::shutdown);
        }
    };
    ports.forEach(strPort -> {
        int port = Integer.parseInt(strPort);
        registerPrefixFilter(strPort);
        registerTimestampFilter(strPort);
        if (proxyConfig.isHttpHealthCheckAllPorts()) {
            healthCheckManager.enableHealthcheck(port);
        }
        WavefrontPortUnificationHandler wavefrontPortUnificationHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, decoderSupplier.get(), histogramHandlerFactory, hostAnnotator, preprocessors.get(strPort), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE_SPAN_LOGS).isFeatureDisabled(), sampler);
        startAsManagedThread(port, new TcpIngester(createInitializer(wavefrontPortUnificationHandler, port, proxyConfig.getHistogramMaxReceivedLength(), proxyConfig.getHistogramHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-histogram-" + port);
        logger.info("listening on port: " + port + " for histogram samples, accumulating to the " + listenerBinType);
    });
}