Search in sources :

Example 1 with GraphiteDecoder

use of com.wavefront.ingester.GraphiteDecoder in project java by wavefrontHQ.

the class PushAgent method startListeners.

@Override
protected void startListeners() {
    if (soLingerTime >= 0) {
        childChannelOptions.put(ChannelOption.SO_LINGER, soLingerTime);
    }
    if (pushListenerPorts != null) {
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts);
        for (String strPort : ports) {
            startGraphiteListener(strPort, false);
        }
    }
    {
        // Histogram bootstrap.
        Iterator<String> histMinPorts = Strings.isNullOrEmpty(histogramMinuteListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramMinuteListenerPorts).iterator();
        Iterator<String> histHourPorts = Strings.isNullOrEmpty(histogramHourListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramHourListenerPorts).iterator();
        Iterator<String> histDayPorts = Strings.isNullOrEmpty(histogramDayListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDayListenerPorts).iterator();
        Iterator<String> histDistPorts = Strings.isNullOrEmpty(histogramDistListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDistListenerPorts).iterator();
        int activeHistogramAggregationTypes = (histDayPorts.hasNext() ? 1 : 0) + (histHourPorts.hasNext() ? 1 : 0) + (histMinPorts.hasNext() ? 1 : 0) + (histDistPorts.hasNext() ? 1 : 0);
        if (activeHistogramAggregationTypes > 0) {
            /*Histograms enabled*/
            histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
            histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
            histogramScanExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-scan"));
            managedExecutors.add(histogramExecutor);
            managedExecutors.add(histogramFlushExecutor);
            managedExecutors.add(histogramScanExecutor);
            File baseDirectory = new File(histogramStateDirectory);
            if (persistMessages || persistAccumulator) {
                // Check directory
                checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!");
                checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!");
            }
            // Central dispatch
            PointHandler histogramHandler = new PointHandlerImpl("histogram ports", pushValidationLevel, pushBlockedSamples, prefix, getFlushTasks(Constants.PUSH_FORMAT_HISTOGRAM, "histogram ports"));
            // Input queue factory
            TapeDeck<List<String>> accumulatorDeck = new TapeDeck<>(persistMessagesCompression ? TapeStringListConverter.getCompressionEnabledInstance() : TapeStringListConverter.getDefaultInstance(), persistMessages);
            // Decoders
            Decoder<String> sampleDecoder = new GraphiteDecoder("unknown", customSourceTags);
            Decoder<String> distributionDecoder = new HistogramDecoder("unknown");
            if (histMinPorts.hasNext()) {
                startHistogramListeners(histMinPorts, sampleDecoder, histogramHandler, accumulatorDeck, "minute", histogramMinuteFlushSecs, histogramMinuteAccumulators, histogramMinuteMemoryCache, baseDirectory, histogramMinuteAccumulatorSize, histogramMinuteAvgKeyBytes, histogramMinuteAvgDigestBytes, histogramMinuteCompression);
            }
            if (histHourPorts.hasNext()) {
                startHistogramListeners(histHourPorts, sampleDecoder, histogramHandler, accumulatorDeck, "hour", histogramHourFlushSecs, histogramHourAccumulators, histogramHourMemoryCache, baseDirectory, histogramHourAccumulatorSize, histogramHourAvgKeyBytes, histogramHourAvgDigestBytes, histogramHourCompression);
            }
            if (histDayPorts.hasNext()) {
                startHistogramListeners(histDayPorts, sampleDecoder, histogramHandler, accumulatorDeck, "day", histogramDayFlushSecs, histogramDayAccumulators, histogramDayMemoryCache, baseDirectory, histogramDayAccumulatorSize, histogramDayAvgKeyBytes, histogramDayAvgDigestBytes, histogramDayCompression);
            }
            if (histDistPorts.hasNext()) {
                startHistogramListeners(histDistPorts, distributionDecoder, histogramHandler, accumulatorDeck, "distribution", histogramDistFlushSecs, histogramDistAccumulators, histogramDistMemoryCache, baseDirectory, histogramDistAccumulatorSize, histogramDistAvgKeyBytes, histogramDistAvgDigestBytes, histogramDistCompression);
            }
        }
    }
    GraphiteFormatter graphiteFormatter = null;
    if (graphitePorts != null || picklePorts != null) {
        Preconditions.checkNotNull(graphiteFormat, "graphiteFormat must be supplied to enable graphite support");
        Preconditions.checkNotNull(graphiteDelimiters, "graphiteDelimiters must be supplied to enable graphite support");
        graphiteFormatter = new GraphiteFormatter(graphiteFormat, graphiteDelimiters, graphiteFieldsToRemove);
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(graphitePorts);
        for (String strPort : ports) {
            preprocessors.forPort(strPort).forPointLine().addTransformer(0, graphiteFormatter);
            startGraphiteListener(strPort, true);
            logger.info("listening on port: " + strPort + " for graphite metrics");
        }
    }
    if (opentsdbPorts != null) {
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(opentsdbPorts);
        for (String strPort : ports) {
            startOpenTsdbListener(strPort);
            logger.info("listening on port: " + strPort + " for OpenTSDB metrics");
        }
    }
    if (picklePorts != null) {
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(picklePorts);
        for (String strPort : ports) {
            startPickleListener(strPort, graphiteFormatter);
            logger.info("listening on port: " + strPort + " for pickle protocol metrics");
        }
    }
    if (httpJsonPorts != null) {
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(httpJsonPorts);
        for (String strPort : ports) {
            preprocessors.forPort(strPort).forReportPoint().addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours));
            startAsManagedThread(() -> {
                activeListeners.inc();
                try {
                    org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort));
                    server.setHandler(new JsonMetricsEndpoint(strPort, hostname, prefix, pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort)));
                    server.start();
                    server.join();
                } catch (InterruptedException e) {
                    logger.warning("Http Json server interrupted.");
                } catch (Exception e) {
                    if (e instanceof BindException) {
                        logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!");
                    }
                } finally {
                    activeListeners.dec();
                }
            }, "listener-plaintext-json-" + strPort);
        }
    }
    if (writeHttpJsonPorts != null) {
        Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(writeHttpJsonPorts);
        for (String strPort : ports) {
            preprocessors.forPort(strPort).forReportPoint().addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours));
            startAsManagedThread(() -> {
                activeListeners.inc();
                try {
                    org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort));
                    server.setHandler(new WriteHttpJsonMetricsEndpoint(strPort, hostname, prefix, pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort)));
                    server.start();
                    server.join();
                } catch (InterruptedException e) {
                    logger.warning("WriteHttpJson server interrupted.");
                } catch (Exception e) {
                    if (e instanceof BindException) {
                        logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!");
                    }
                } finally {
                    activeListeners.dec();
                }
            }, "listener-plaintext-writehttpjson-" + strPort);
        }
    }
    // Logs ingestion.
    if (loadLogsIngestionConfig() != null) {
        logger.info("Loading logs ingestion.");
        try {
            final LogsIngester logsIngester = new LogsIngester(new PointHandlerImpl("logs-ingester", pushValidationLevel, pushBlockedSamples, getFlushTasks("logs-ingester")), this::loadLogsIngestionConfig, prefix, System::currentTimeMillis);
            logsIngester.start();
            if (filebeatPort > 0) {
                final Server filebeatServer = new Server(filebeatPort);
                filebeatServer.setMessageListener(new FilebeatIngester(logsIngester, System::currentTimeMillis));
                startAsManagedThread(() -> {
                    try {
                        activeListeners.inc();
                        filebeatServer.listen();
                    } catch (InterruptedException e) {
                        logger.log(Level.SEVERE, "Filebeat server interrupted.", e);
                    } catch (Exception e) {
                        // ChannelFuture throws undeclared checked exceptions, so we need to handle it
                        if (e instanceof BindException) {
                            logger.severe("Unable to start listener - port " + String.valueOf(rawLogsPort) + " is already in use!");
                        }
                    } finally {
                        activeListeners.dec();
                    }
                }, "listener-logs-filebeat-" + filebeatPort);
            }
            if (rawLogsPort > 0) {
                RawLogsIngester rawLogsIngester = new RawLogsIngester(logsIngester, rawLogsPort, System::currentTimeMillis);
                startAsManagedThread(() -> {
                    try {
                        activeListeners.inc();
                        rawLogsIngester.listen();
                    } catch (InterruptedException e) {
                        logger.log(Level.SEVERE, "Raw logs server interrupted.", e);
                    } catch (Exception e) {
                        // ChannelFuture throws undeclared checked exceptions, so we need to handle it
                        if (e instanceof BindException) {
                            logger.severe("Unable to start listener - port " + String.valueOf(rawLogsPort) + " is already in use!");
                        }
                    } finally {
                        activeListeners.dec();
                    }
                }, "listener-logs-raw-" + rawLogsPort);
            }
        } catch (ConfigurationException e) {
            logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
        }
    } else {
        logger.info("Not loading logs ingestion -- no config specified.");
    }
}
Also used : Server(org.logstash.beats.Server) GraphiteDecoder(com.wavefront.ingester.GraphiteDecoder) PickleProtocolDecoder(com.wavefront.ingester.PickleProtocolDecoder) Decoder(com.wavefront.ingester.Decoder) OpenTSDBDecoder(com.wavefront.ingester.OpenTSDBDecoder) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) LengthFieldBasedFrameDecoder(io.netty.handler.codec.LengthFieldBasedFrameDecoder) ConfigurationException(com.wavefront.agent.config.ConfigurationException) Iterator(java.util.Iterator) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) RawLogsIngester(com.wavefront.agent.logsharvesting.RawLogsIngester) RawLogsIngester(com.wavefront.agent.logsharvesting.RawLogsIngester) FilebeatIngester(com.wavefront.agent.logsharvesting.FilebeatIngester) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) TapeDeck(com.wavefront.agent.histogram.tape.TapeDeck) BindException(java.net.BindException) BindException(java.net.BindException) ConfigurationException(com.wavefront.agent.config.ConfigurationException) IOException(java.io.IOException) File(java.io.File) GraphiteDecoder(com.wavefront.ingester.GraphiteDecoder)

Example 2 with GraphiteDecoder

use of com.wavefront.ingester.GraphiteDecoder in project java by wavefrontHQ.

the class PushAgent method startGraphiteListener.

protected void startGraphiteListener(String strPort, boolean withCustomFormatter) {
    int port = Integer.parseInt(strPort);
    if (prefix != null && !prefix.isEmpty()) {
        preprocessors.forPort(strPort).forReportPoint().addTransformer(new ReportPointAddPrefixTransformer(prefix));
    }
    preprocessors.forPort(strPort).forReportPoint().addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours));
    // Add a metadatahandler, to handle @SourceTag, @SourceDescription, etc.
    SourceTagHandler metadataHandler = new SourceTagHandlerImpl(getSourceTagFlushTasks(port));
    // Set up a custom graphite handler, with no formatter
    ChannelHandler graphiteHandler = new ChannelStringHandler(new GraphiteDecoder("unknown", customSourceTags), new PointHandlerImpl(strPort, pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort)), preprocessors.forPort(strPort), metadataHandler);
    if (!withCustomFormatter) {
        List<Function<Channel, ChannelHandler>> handler = Lists.newArrayList(1);
        handler.add(input -> {
            SocketChannel ch = (SocketChannel) input;
            if (ch != null && ch.remoteAddress() != null) {
                return new GraphiteHostAnnotator(disableRdnsLookup ? ch.remoteAddress().getAddress().getHostAddress() : reverseDnsCache.get(ch.remoteAddress().getAddress()), customSourceTags);
            }
            return new GraphiteHostAnnotator("unknown", customSourceTags);
        });
        startAsManagedThread(new StringLineIngester(handler, graphiteHandler, port).withChildChannelOptions(childChannelOptions), "listener-plaintext-wavefront-" + port);
    } else {
        startAsManagedThread(new StringLineIngester(graphiteHandler, port).withChildChannelOptions(childChannelOptions), "Listener-plaintext-graphite-" + port);
    }
}
Also used : SocketChannel(io.netty.channel.socket.SocketChannel) GraphiteHostAnnotator(com.wavefront.ingester.GraphiteHostAnnotator) QueuingChannelHandler(com.wavefront.agent.histogram.QueuingChannelHandler) ChannelHandler(io.netty.channel.ChannelHandler) Function(com.google.common.base.Function) ReportPointAddPrefixTransformer(com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer) StringLineIngester(com.wavefront.ingester.StringLineIngester) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) GraphiteDecoder(com.wavefront.ingester.GraphiteDecoder)

Example 3 with GraphiteDecoder

use of com.wavefront.ingester.GraphiteDecoder in project java by wavefrontHQ.

the class AccumulationTaskTest method setUp.

@Before
public void setUp() throws Exception {
    AtomicInteger timeMillis = new AtomicInteger(0);
    in = new InMemoryObjectQueue<>();
    out = new ConcurrentHashMap<>();
    cache = new AccumulationCache(out, 0, timeMillis::get);
    badPointsOut = Lists.newArrayList();
    PointHandler pointHandler = new PointHandler() {

        @Override
        public void reportPoint(ReportPoint point, String debugLine) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void reportPoints(List<ReportPoint> points) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void handleBlockedPoint(String pointLine) {
            badPointsOut.add(pointLine);
        }
    };
    eventSubject = new AccumulationTask(in, cache, new GraphiteDecoder("unknown", ImmutableList.of()), pointHandler, Validation.Level.NUMERIC_ONLY, TTL, MINUTE, COMPRESSION);
    histoSubject = new AccumulationTask(in, cache, new HistogramDecoder(), pointHandler, Validation.Level.NUMERIC_ONLY, TTL, MINUTE, COMPRESSION);
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) PointHandler(com.wavefront.agent.PointHandler) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) ReportPoint(wavefront.report.ReportPoint) GraphiteDecoder(com.wavefront.ingester.GraphiteDecoder) Before(org.junit.Before)

Aggregations

GraphiteDecoder (com.wavefront.ingester.GraphiteDecoder)3 ReportPointTimestampInRangeFilter (com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter)2 HistogramDecoder (com.wavefront.ingester.HistogramDecoder)2 Function (com.google.common.base.Function)1 ImmutableList (com.google.common.collect.ImmutableList)1 PointHandler (com.wavefront.agent.PointHandler)1 ConfigurationException (com.wavefront.agent.config.ConfigurationException)1 GraphiteFormatter (com.wavefront.agent.formatter.GraphiteFormatter)1 QueuingChannelHandler (com.wavefront.agent.histogram.QueuingChannelHandler)1 TapeDeck (com.wavefront.agent.histogram.tape.TapeDeck)1 FilebeatIngester (com.wavefront.agent.logsharvesting.FilebeatIngester)1 LogsIngester (com.wavefront.agent.logsharvesting.LogsIngester)1 RawLogsIngester (com.wavefront.agent.logsharvesting.RawLogsIngester)1 ReportPointAddPrefixTransformer (com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer)1 Decoder (com.wavefront.ingester.Decoder)1 GraphiteHostAnnotator (com.wavefront.ingester.GraphiteHostAnnotator)1 OpenTSDBDecoder (com.wavefront.ingester.OpenTSDBDecoder)1 PickleProtocolDecoder (com.wavefront.ingester.PickleProtocolDecoder)1 StringLineIngester (com.wavefront.ingester.StringLineIngester)1 ChannelHandler (io.netty.channel.ChannelHandler)1