Search in sources :

Example 1 with DeltaCounterAccumulationHandlerImpl

use of com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl in project java by wavefrontHQ.

the class PushAgent method startDeltaCounterListener.

@VisibleForTesting
protected void startDeltaCounterListener(String strPort, SharedGraphiteHostAnnotator hostAnnotator, SenderTaskFactory senderTaskFactory, SpanSampler sampler) {
    final int port = Integer.parseInt(strPort);
    registerPrefixFilter(strPort);
    registerTimestampFilter(strPort);
    if (proxyConfig.isHttpHealthCheckAllPorts())
        healthCheckManager.enableHealthcheck(port);
    if (this.deltaCounterHandlerFactory == null) {
        this.deltaCounterHandlerFactory = new ReportableEntityHandlerFactory() {

            private final Map<String, ReportableEntityHandler<?, ?>> handlers = new ConcurrentHashMap<>();

            @Override
            public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
                // noinspection unchecked
                return (ReportableEntityHandler<T, U>) handlers.computeIfAbsent(handlerKey.getHandle(), k -> new DeltaCounterAccumulationHandlerImpl(handlerKey, proxyConfig.getPushBlockedSamples(), senderTaskFactory.createSenderTasks(handlerKey), validationConfiguration, proxyConfig.getDeltaCountersAggregationIntervalSeconds(), rate -> entityProps.get(ReportableEntityType.POINT).reportReceivedRate(handlerKey.getHandle(), rate), blockedPointsLogger, VALID_POINTS_LOGGER));
            }

            @Override
            public void shutdown(@Nonnull String handle) {
                if (handlers.containsKey(handle)) {
                    handlers.values().forEach(ReportableEntityHandler::shutdown);
                }
            }
        };
    }
    shutdownTasks.add(() -> deltaCounterHandlerFactory.shutdown(strPort));
    WavefrontPortUnificationHandler wavefrontPortUnificationHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, decoderSupplier.get(), deltaCounterHandlerFactory, hostAnnotator, preprocessors.get(strPort), () -> false, () -> false, () -> false, sampler);
    startAsManagedThread(port, new TcpIngester(createInitializer(wavefrontPortUnificationHandler, port, proxyConfig.getPushListenerMaxReceivedLength(), proxyConfig.getPushListenerHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-deltaCounter-" + port);
}
Also used : HandlerKey(com.wavefront.agent.handlers.HandlerKey) QueueingReason(com.wavefront.agent.data.QueueingReason) CustomTracingPortUnificationHandler(com.wavefront.agent.listeners.tracing.CustomTracingPortUnificationHandler) CompositeSampler(com.wavefront.sdk.entities.tracing.sampling.CompositeSampler) EntityProperties(com.wavefront.agent.data.EntityProperties) NettyServerBuilder(io.grpc.netty.NettyServerBuilder) RequestConfig(org.apache.http.client.config.RequestConfig) StringUtils(org.apache.commons.lang3.StringUtils) SpanSanitizeTransformer(com.wavefront.agent.preprocessor.SpanSanitizeTransformer) BooleanUtils(org.apache.commons.lang.BooleanUtils) InetAddress(java.net.InetAddress) SQSQueueFactoryImpl(com.wavefront.agent.queueing.SQSQueueFactoryImpl) PointHandlerDispatcher(com.wavefront.agent.histogram.PointHandlerDispatcher) Map(java.util.Map) HandlerKey(com.wavefront.agent.handlers.HandlerKey) ByteArrayDecoder(io.netty.handler.codec.bytes.ByteArrayDecoder) RecyclableRateLimiter(com.google.common.util.concurrent.RecyclableRateLimiter) HealthCheckManagerImpl(com.wavefront.agent.channel.HealthCheckManagerImpl) HistogramKey(com.wavefront.agent.histogram.HistogramKey) AdminPortUnificationHandler(com.wavefront.agent.listeners.AdminPortUnificationHandler) ChronicleMap(net.openhft.chronicle.map.ChronicleMap) ProxyUtil.createInitializer(com.wavefront.agent.ProxyUtil.createInitializer) Executors(java.util.concurrent.Executors) AgentDigestFactory(com.wavefront.agent.histogram.accumulator.AgentDigestFactory) ByteOrder(java.nio.ByteOrder) AgentConfiguration(com.wavefront.api.agent.AgentConfiguration) OpenTSDBDecoder(com.wavefront.ingester.OpenTSDBDecoder) ReportPointTimestampInRangeFilter(com.wavefront.agent.preprocessor.ReportPointTimestampInRangeFilter) AccumulationCache(com.wavefront.agent.histogram.accumulator.AccumulationCache) HealthCheckManager(com.wavefront.agent.channel.HealthCheckManager) ChannelOption(io.netty.channel.ChannelOption) SpanSampler(com.wavefront.agent.sampler.SpanSampler) TChannel(com.uber.tchannel.api.TChannel) Supplier(java.util.function.Supplier) RelayPortUnificationHandler(com.wavefront.agent.listeners.RelayPortUnificationHandler) TcpIngester(com.wavefront.ingester.TcpIngester) ArrayList(java.util.ArrayList) HttpClient(org.apache.http.client.HttpClient) SharedGraphiteHostAnnotator(com.wavefront.agent.channel.SharedGraphiteHostAnnotator) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConfigurationException(com.wavefront.agent.config.ConfigurationException) Utils.lazySupplier(com.wavefront.common.Utils.lazySupplier) Utils.csvToList(com.wavefront.common.Utils.csvToList) Server(org.logstash.beats.Server) TokenAuthenticatorBuilder(com.wavefront.agent.auth.TokenAuthenticatorBuilder) MapLoader(com.wavefront.agent.histogram.MapLoader) Nullable(javax.annotation.Nullable) HistogramUtils(com.wavefront.agent.histogram.HistogramUtils) DataDogPortUnificationHandler(com.wavefront.agent.listeners.DataDogPortUnificationHandler) SslContext(io.netty.handler.ssl.SslContext) WavefrontSender(com.wavefront.sdk.common.WavefrontSender) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) File(java.io.File) CachingHostnameLookupResolver(com.wavefront.agent.channel.CachingHostnameLookupResolver) NamedThreadFactory(com.wavefront.common.NamedThreadFactory) GraphiteFormatter(com.wavefront.agent.formatter.GraphiteFormatter) ReportSourceTagDecoder(com.wavefront.ingester.ReportSourceTagDecoder) ReportableEntityDecoder(com.wavefront.ingester.ReportableEntityDecoder) VALID_POINTS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_POINTS_LOGGER) Preconditions(com.google.common.base.Preconditions) WriteHttpJsonPortUnificationHandler(com.wavefront.agent.listeners.WriteHttpJsonPortUnificationHandler) Metrics(com.yammer.metrics.Metrics) HttpClientBuilder(org.apache.http.impl.client.HttpClientBuilder) QueueingFactory(com.wavefront.agent.queueing.QueueingFactory) SpanSamplerUtils(com.wavefront.agent.sampler.SpanSamplerUtils) ReportPointDecoder(com.wavefront.ingester.ReportPointDecoder) LogsIngester(com.wavefront.agent.logsharvesting.LogsIngester) SpanLogsDecoder(com.wavefront.ingester.SpanLogsDecoder) Sampler(com.wavefront.sdk.entities.tracing.sampling.Sampler) ZipkinPortUnificationHandler(com.wavefront.agent.listeners.tracing.ZipkinPortUnificationHandler) TaskQueueFactory(com.wavefront.agent.queueing.TaskQueueFactory) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TokenAuthenticator(com.wavefront.agent.auth.TokenAuthenticator) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) HistogramAccumulationHandlerImpl(com.wavefront.agent.handlers.HistogramAccumulationHandlerImpl) Granularity(com.wavefront.agent.histogram.Granularity) DefaultHttpRequestRetryHandler(org.apache.http.impl.client.DefaultHttpRequestRetryHandler) ReportPoint(wavefront.report.ReportPoint) WavefrontInternalReporter(com.wavefront.internal.reporter.WavefrontInternalReporter) HistogramRecompressor(com.wavefront.agent.histogram.HistogramRecompressor) CorsConfigBuilder(io.netty.handler.codec.http.cors.CorsConfigBuilder) FilebeatIngester(com.wavefront.agent.logsharvesting.FilebeatIngester) PickleProtocolDecoder(com.wavefront.ingester.PickleProtocolDecoder) JaegerPortUnificationHandler(com.wavefront.agent.listeners.tracing.JaegerPortUnificationHandler) IdentityHashMap(java.util.IdentityHashMap) TaggedMetricName(com.wavefront.common.TaggedMetricName) ImmutableMap(com.google.common.collect.ImmutableMap) HttpHealthCheckEndpointHandler(com.wavefront.agent.listeners.HttpHealthCheckEndpointHandler) Accumulator(com.wavefront.agent.histogram.accumulator.Accumulator) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) ReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl) Collectors(java.util.stream.Collectors) ReportableEntityType(com.wavefront.data.ReportableEntityType) List(java.util.List) ExpectedAgentMetric(com.wavefront.metrics.ExpectedAgentMetric) ChannelByteArrayHandler(com.wavefront.agent.listeners.ChannelByteArrayHandler) QueueingFactoryImpl(com.wavefront.agent.queueing.QueueingFactoryImpl) JaegerTChannelCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerTChannelCollectorHandler) PreprocessorRuleMetrics(com.wavefront.agent.preprocessor.PreprocessorRuleMetrics) HistogramDecoder(com.wavefront.ingester.HistogramDecoder) HashMap(java.util.HashMap) BindException(java.net.BindException) Function(java.util.function.Function) OpenTSDBPortUnificationHandler(com.wavefront.agent.listeners.OpenTSDBPortUnificationHandler) Level(java.util.logging.Level) JsonMetricsPortUnificationHandler(com.wavefront.agent.listeners.JsonMetricsPortUnificationHandler) TaskQueueFactoryImpl(com.wavefront.agent.queueing.TaskQueueFactoryImpl) ReportPointAddPrefixTransformer(com.wavefront.agent.preprocessor.ReportPointAddPrefixTransformer) ImmutableList(com.google.common.collect.ImmutableList) TrafficShapingRateLimitAdjuster(com.wavefront.agent.handlers.TrafficShapingRateLimitAdjuster) ObjectUtils(org.apache.commons.lang3.ObjectUtils) RawLogsIngesterPortUnificationHandler(com.wavefront.agent.listeners.RawLogsIngesterPortUnificationHandler) HistogramKeyMarshaller(com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller) EventDecoder(com.wavefront.ingester.EventDecoder) ReportPointDecoderWrapper(com.wavefront.ingester.ReportPointDecoderWrapper) Nonnull(javax.annotation.Nonnull) InternalProxyWavefrontClient(com.wavefront.agent.handlers.InternalProxyWavefrontClient) LengthFieldBasedFrameDecoder(io.netty.handler.codec.LengthFieldBasedFrameDecoder) Counter(com.yammer.metrics.core.Counter) SenderTaskFactory(com.wavefront.agent.handlers.SenderTaskFactory) AgentDigestMarshaller(com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller) DelegatingReportableEntityHandlerFactoryImpl(com.wavefront.agent.handlers.DelegatingReportableEntityHandlerFactoryImpl) SpanDecoder(com.wavefront.ingester.SpanDecoder) CorsConfig(io.netty.handler.codec.http.cors.CorsConfig) VALID_HISTOGRAMS_LOGGER(com.wavefront.agent.handlers.ReportableEntityHandlerFactoryImpl.VALID_HISTOGRAMS_LOGGER) HttpMethod(io.netty.handler.codec.http.HttpMethod) AgentDigest(com.tdunning.math.stats.AgentDigest) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) TracePortUnificationHandler(com.wavefront.agent.listeners.tracing.TracePortUnificationHandler) TimeUnit(java.util.concurrent.TimeUnit) Connection(com.uber.tchannel.channels.Connection) RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) SenderTaskFactoryImpl(com.wavefront.agent.handlers.SenderTaskFactoryImpl) JaegerGrpcCollectorHandler(com.wavefront.agent.listeners.tracing.JaegerGrpcCollectorHandler) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Histogram(wavefront.report.Histogram) WavefrontPortUnificationHandler(com.wavefront.agent.listeners.WavefrontPortUnificationHandler) ReportPoint(wavefront.report.ReportPoint) ReportableEntityHandler(com.wavefront.agent.handlers.ReportableEntityHandler) NO_RATE_LIMIT(com.wavefront.agent.data.EntityProperties.NO_RATE_LIMIT) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) ReportableEntityHandlerFactory(com.wavefront.agent.handlers.ReportableEntityHandlerFactory) TcpIngester(com.wavefront.ingester.TcpIngester) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 2 with DeltaCounterAccumulationHandlerImpl

use of com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl in project java by wavefrontHQ.

the class PushAgentTest method testDeltaCounterHandlerMixedData.

@Test
public void testDeltaCounterHandlerMixedData() throws Exception {
    deltaPort = findAvailablePort(5888);
    proxy.proxyConfig.deltaCountersAggregationListenerPorts = String.valueOf(deltaPort);
    proxy.proxyConfig.deltaCountersAggregationIntervalSeconds = 10;
    proxy.proxyConfig.pushFlushInterval = 100;
    proxy.startDeltaCounterListener(proxy.proxyConfig.getDeltaCountersAggregationListenerPorts(), null, mockSenderTaskFactory, new SpanSampler(new RateSampler(1.0D), () -> null));
    waitUntilListenerIsOnline(deltaPort);
    reset(mockSenderTask);
    Capture<String> capturedArgument = Capture.newInstance(CaptureType.ALL);
    mockSenderTask.add(EasyMock.capture(capturedArgument));
    expectLastCall().atLeastOnce();
    replay(mockSenderTask);
    String payloadStr1 = "∆test.mixed1 1.0 source=test1\n";
    String payloadStr2 = "∆test.mixed2 2.0 source=test1\n";
    String payloadStr3 = "test.mixed3 3.0 source=test1\n";
    String payloadStr4 = "∆test.mixed3 3.0 source=test1\n";
    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr1 + payloadStr2 + payloadStr2 + payloadStr3 + payloadStr4));
    ReportableEntityHandler<?, ?> handler = proxy.deltaCounterHandlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, String.valueOf(deltaPort)));
    if (handler instanceof DeltaCounterAccumulationHandlerImpl) {
        ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
    }
    verify(mockSenderTask);
    assertEquals(3, capturedArgument.getValues().size());
    assertTrue(capturedArgument.getValues().get(0).startsWith("\"∆test.mixed1\" 1.0"));
    assertTrue(capturedArgument.getValues().get(1).startsWith("\"∆test.mixed2\" 4.0"));
    assertTrue(capturedArgument.getValues().get(2).startsWith("\"∆test.mixed3\" 3.0"));
}
Also used : RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) SpanSampler(com.wavefront.agent.sampler.SpanSampler) EasyMock.anyString(org.easymock.EasyMock.anyString) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) Test(org.junit.Test)

Example 3 with DeltaCounterAccumulationHandlerImpl

use of com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl in project java by wavefrontHQ.

the class PushAgentTest method testDeltaCounterHandlerDataStream.

@Test
public void testDeltaCounterHandlerDataStream() throws Exception {
    deltaPort = findAvailablePort(5888);
    proxy.proxyConfig.deltaCountersAggregationListenerPorts = String.valueOf(deltaPort);
    proxy.proxyConfig.deltaCountersAggregationIntervalSeconds = 10;
    proxy.startDeltaCounterListener(proxy.proxyConfig.getDeltaCountersAggregationListenerPorts(), null, mockSenderTaskFactory, new SpanSampler(new RateSampler(1.0D), () -> null));
    waitUntilListenerIsOnline(deltaPort);
    reset(mockSenderTask);
    Capture<String> capturedArgument = Capture.newInstance(CaptureType.ALL);
    mockSenderTask.add(EasyMock.capture(capturedArgument));
    expectLastCall().atLeastOnce();
    replay(mockSenderTask);
    String payloadStr = "∆test.mixed 1.0 " + startTime + " source=test1\n";
    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr + payloadStr));
    ReportableEntityHandler<?, ?> handler = proxy.deltaCounterHandlerFactory.getHandler(HandlerKey.of(ReportableEntityType.POINT, String.valueOf(deltaPort)));
    if (!(handler instanceof DeltaCounterAccumulationHandlerImpl))
        fail();
    ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr));
    assertEquals(202, httpPost("http://localhost:" + deltaPort, payloadStr + payloadStr));
    ((DeltaCounterAccumulationHandlerImpl) handler).flushDeltaCounters();
    verify(mockSenderTask);
    assertEquals(2, capturedArgument.getValues().size());
    assertTrue(capturedArgument.getValues().get(0).startsWith("\"∆test.mixed\" 2.0"));
    assertTrue(capturedArgument.getValues().get(1).startsWith("\"∆test.mixed\" 3.0"));
}
Also used : RateSampler(com.wavefront.sdk.entities.tracing.sampling.RateSampler) SpanSampler(com.wavefront.agent.sampler.SpanSampler) EasyMock.anyString(org.easymock.EasyMock.anyString) DeltaCounterAccumulationHandlerImpl(com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl) Test(org.junit.Test)

Aggregations

DeltaCounterAccumulationHandlerImpl (com.wavefront.agent.handlers.DeltaCounterAccumulationHandlerImpl)3 SpanSampler (com.wavefront.agent.sampler.SpanSampler)3 RateSampler (com.wavefront.sdk.entities.tracing.sampling.RateSampler)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions (com.google.common.base.Preconditions)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 RecyclableRateLimiter (com.google.common.util.concurrent.RecyclableRateLimiter)1 AgentDigest (com.tdunning.math.stats.AgentDigest)1 AgentDigestMarshaller (com.tdunning.math.stats.AgentDigest.AgentDigestMarshaller)1 TChannel (com.uber.tchannel.api.TChannel)1 Connection (com.uber.tchannel.channels.Connection)1 ProxyUtil.createInitializer (com.wavefront.agent.ProxyUtil.createInitializer)1 TokenAuthenticator (com.wavefront.agent.auth.TokenAuthenticator)1 TokenAuthenticatorBuilder (com.wavefront.agent.auth.TokenAuthenticatorBuilder)1 CachingHostnameLookupResolver (com.wavefront.agent.channel.CachingHostnameLookupResolver)1 HealthCheckManager (com.wavefront.agent.channel.HealthCheckManager)1 HealthCheckManagerImpl (com.wavefront.agent.channel.HealthCheckManagerImpl)1 SharedGraphiteHostAnnotator (com.wavefront.agent.channel.SharedGraphiteHostAnnotator)1