Examples with QueueingFactoryImpl com.wavefront.agent.queueing.QueueingFactoryImpl used on opensource projects

Example 1 with QueueingFactoryImpl

use of com.wavefront.agent.queueing.QueueingFactoryImpl in project java by wavefrontHQ.

the class HttpEndToEndTest method testEndToEndSpans.

@Test
public void testEndToEndSpans() throws Exception {
    long time = Clock.now() / 1000;
    proxyPort = findAvailablePort(2898);
    proxyPort = findAvailablePort(2898);
    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
    proxy = new PushAgent();
    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
    proxy.proxyConfig.flushThreads = 1;
    proxy.proxyConfig.traceListenerPorts = String.valueOf(proxyPort);
    proxy.proxyConfig.pushFlushInterval = 50;
    proxy.proxyConfig.bufferFile = buffer;
    proxy.proxyConfig.trafficShaping = true;
    proxy.start(new String[] {});
    waitUntilListenerIsOnline(proxyPort);
    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl))
        fail();
    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl))
        fail();
    String traceId = UUID.randomUUID().toString();
    long timestamp1 = time * 1000000 + 12345;
    long timestamp2 = time * 1000000 + 23456;
    String payload = "testSpanName parent=parent1 source=testsource spanId=testspanid " + "traceId=\"" + traceId + "\" parent=parent2 " + time + " " + (time + 1) + "\n" + "{\"spanId\":\"testspanid\",\"traceId\":\"" + traceId + "\",\"logs\":[{\"timestamp\":" + timestamp1 + ",\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":" + timestamp2 + ",\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}]}\n";
    String expectedSpan = "\"testSpanName\" source=\"testsource\" spanId=\"testspanid\" " + "traceId=\"" + traceId + "\" \"parent\"=\"parent1\" \"parent\"=\"parent2\" " + (time * 1000) + " 1000";
    String expectedSpanLog = "{\"customer\":\"dummy\",\"traceId\":\"" + traceId + "\",\"spanId" + "\":\"testspanid\",\"spanSecondaryId\":null,\"logs\":[{\"timestamp\":" + timestamp1 + "," + "\"fields\":{\"key\":\"value\",\"key2\":\"value2\"}},{\"timestamp\":" + timestamp2 + "," + "\"fields\":{\"key3\":\"value3\",\"key4\":\"value4\"}}],\"span\":null}";
    AtomicBoolean gotSpan = new AtomicBoolean(false);
    AtomicBoolean gotSpanLog = new AtomicBoolean(false);
    server.update(req -> {
        String content = req.content().toString(CharsetUtil.UTF_8);
        logger.fine("Content received: " + content);
        if (content.equals(expectedSpan))
            gotSpan.set(true);
        if (content.equals(expectedSpanLog))
            gotSpanLog.set(true);
        return makeResponse(HttpResponseStatus.OK, "");
    });
    gzippedHttpPost("http://localhost:" + proxyPort + "/", payload);
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(HandlerKey.of(ReportableEntityType.TRACE, String.valueOf(proxyPort)));
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(HandlerKey.of(ReportableEntityType.TRACE_SPAN_LOGS, String.valueOf(proxyPort)));
    assertTrueWithTimeout(50, gotSpan::get);
    assertTrueWithTimeout(50, gotSpanLog::get);
}

Example 2 with QueueingFactoryImpl

use of com.wavefront.agent.queueing.QueueingFactoryImpl in project java by wavefrontHQ.

the class HttpEndToEndTest method testEndToEndEvents.

@Test
public void testEndToEndEvents() throws Exception {
    AtomicInteger successfulSteps = new AtomicInteger(0);
    AtomicInteger testCounter = new AtomicInteger(0);
    long time = Clock.now() / 1000;
    proxyPort = findAvailablePort(2898);
    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
    proxy = new PushAgent();
    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
    proxy.proxyConfig.flushThreads = 1;
    proxy.proxyConfig.flushThreadsEvents = 1;
    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
    proxy.proxyConfig.pushFlushInterval = 10000;
    proxy.proxyConfig.pushRateLimitEvents = 100;
    proxy.proxyConfig.bufferFile = buffer;
    proxy.start(new String[] {});
    waitUntilListenerIsOnline(proxyPort);
    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl))
        fail();
    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl))
        fail();
    String payloadEvents = "@Event " + time + " \"Event name for testing\" host=host1 host=host2 tag=tag1 " + "severity=INFO multi=bar multi=baz\n" + "@Event " + time + " \"Another test event\" host=host3";
    String expectedEvent1 = "{\"name\":\"Event name for testing\",\"startTime\":" + (time * 1000) + ",\"endTime\":" + (time * 1000 + 1) + ",\"annotations\":{\"severity\":\"INFO\"}," + "\"dimensions\":{\"multi\":[\"bar\",\"baz\"]},\"hosts\":[\"host1\",\"host2\"]," + "\"tags\":[\"tag1\"]}";
    String expectedEvent2 = "{\"name\":\"Another test event\",\"startTime\":" + (time * 1000) + ",\"endTime\":" + (time * 1000 + 1) + ",\"annotations\":{},\"dimensions\":null," + "\"hosts\":[\"host3\"],\"tags\":null}";
    server.update(req -> {
        String content = req.content().toString(CharsetUtil.UTF_8);
        URI uri;
        try {
            uri = new URI(req.uri());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        String path = uri.getPath();
        logger.fine("Content received: " + content);
        assertEquals(HttpMethod.POST, req.method());
        assertEquals("/api/v2/wfproxy/event", path);
        switch(testCounter.incrementAndGet()) {
            case 1:
                assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, "");
            case 2:
                assertEquals("[" + expectedEvent1 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 3:
                assertEquals("[" + expectedEvent2 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 4:
                assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.valueOf(407), "");
            case 5:
                assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, "");
            case 6:
                assertEquals("[" + expectedEvent1 + "," + expectedEvent2 + "]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
        }
        logger.warning("Too many requests");
        // this will force the assert to fail
        successfulSteps.incrementAndGet();
        return makeResponse(HttpResponseStatus.OK, "");
    });
    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadEvents);
    HandlerKey key = HandlerKey.of(ReportableEntityType.EVENT, String.valueOf(proxyPort));
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
    ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadEvents);
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
    for (int i = 0; i < 2; i++) ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
    assertEquals(6, successfulSteps.getAndSet(0));
}

Example 3 with QueueingFactoryImpl

use of com.wavefront.agent.queueing.QueueingFactoryImpl in project java by wavefrontHQ.

the class PushAgent method startListeners.

@Override
protected void startListeners() throws Exception {
    blockedPointsLogger = Logger.getLogger(proxyConfig.getBlockedPointsLoggerName());
    blockedHistogramsLogger = Logger.getLogger(proxyConfig.getBlockedHistogramsLoggerName());
    blockedSpansLogger = Logger.getLogger(proxyConfig.getBlockedSpansLoggerName());
    if (proxyConfig.getSoLingerTime() >= 0) {
        childChannelOptions.put(ChannelOption.SO_LINGER, proxyConfig.getSoLingerTime());
    }
    hostnameResolver = new CachingHostnameLookupResolver(proxyConfig.isDisableRdnsLookup(), ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName);
    if (proxyConfig.isSqsQueueBuffer()) {
        taskQueueFactory = new SQSQueueFactoryImpl(proxyConfig.getSqsQueueNameTemplate(), proxyConfig.getSqsQueueRegion(), proxyConfig.getSqsQueueIdentifier(), proxyConfig.isPurgeBuffer());
    } else {
        taskQueueFactory = new TaskQueueFactoryImpl(proxyConfig.getBufferFile(), proxyConfig.isPurgeBuffer(), proxyConfig.isDisableBufferSharding(), proxyConfig.getBufferShardSize());
    }
    remoteHostAnnotator = new SharedGraphiteHostAnnotator(proxyConfig.getCustomSourceTags(), hostnameResolver);
    queueingFactory = new QueueingFactoryImpl(apiContainer, agentId, taskQueueFactory, entityProps);
    senderTaskFactory = new SenderTaskFactoryImpl(apiContainer, agentId, taskQueueFactory, queueingFactory, entityProps);
    if (proxyConfig.isHistogramPassthroughRecompression()) {
        histogramRecompressor = new HistogramRecompressor(() -> entityProps.getGlobalProperties().getHistogramStorageAccuracy());
    }
    handlerFactory = new ReportableEntityHandlerFactoryImpl(senderTaskFactory, proxyConfig.getPushBlockedSamples(), validationConfiguration, blockedPointsLogger, blockedHistogramsLogger, blockedSpansLogger, histogramRecompressor, entityProps);
    if (proxyConfig.isTrafficShaping()) {
        new TrafficShapingRateLimitAdjuster(entityProps, proxyConfig.getTrafficShapingWindowSeconds(), proxyConfig.getTrafficShapingHeadroom()).start();
    }
    healthCheckManager = new HealthCheckManagerImpl(proxyConfig);
    tokenAuthenticator = configureTokenAuthenticator();
    shutdownTasks.add(() -> senderTaskFactory.shutdown());
    shutdownTasks.add(() -> senderTaskFactory.drainBuffersToQueue(null));
    // sampler for spans
    rateSampler.setSamplingRate(entityProps.getGlobalProperties().getTraceSamplingRate());
    Sampler durationSampler = SpanSamplerUtils.getDurationSampler(proxyConfig.getTraceSamplingDuration());
    List<Sampler> samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler);
    SpanSampler spanSampler = new SpanSampler(new CompositeSampler(samplers), () -> entityProps.getGlobalProperties().getActiveSpanSamplingPolicies());
    if (proxyConfig.getAdminApiListenerPort() > 0) {
        startAdminListener(proxyConfig.getAdminApiListenerPort());
    }
    csvToList(proxyConfig.getHttpHealthCheckPorts()).forEach(strPort -> startHealthCheckListener(Integer.parseInt(strPort)));
    csvToList(proxyConfig.getPushListenerPorts()).forEach(strPort -> {
        startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront metrics");
    });
    csvToList(proxyConfig.getDeltaCountersAggregationListenerPorts()).forEach(strPort -> {
        startDeltaCounterListener(strPort, remoteHostAnnotator, senderTaskFactory, spanSampler);
        logger.info("listening on port: " + strPort + " for Wavefront delta counter metrics");
    });
    {
        // Histogram bootstrap.
        List<String> histMinPorts = csvToList(proxyConfig.getHistogramMinuteListenerPorts());
        List<String> histHourPorts = csvToList(proxyConfig.getHistogramHourListenerPorts());
        List<String> histDayPorts = csvToList(proxyConfig.getHistogramDayListenerPorts());
        List<String> histDistPorts = csvToList(proxyConfig.getHistogramDistListenerPorts());
        int activeHistogramAggregationTypes = (histDayPorts.size() > 0 ? 1 : 0) + (histHourPorts.size() > 0 ? 1 : 0) + (histMinPorts.size() > 0 ? 1 : 0) + (histDistPorts.size() > 0 ? 1 : 0);
        if (activeHistogramAggregationTypes > 0) {
            /*Histograms enabled*/
            histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
            histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
            managedExecutors.add(histogramExecutor);
            managedExecutors.add(histogramFlushExecutor);
            File baseDirectory = new File(proxyConfig.getHistogramStateDirectory());
            // Central dispatch
            ReportableEntityHandler<ReportPoint, String> pointHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports"));
            startHistogramListeners(histMinPorts, pointHandler, remoteHostAnnotator, Granularity.MINUTE, proxyConfig.getHistogramMinuteFlushSecs(), proxyConfig.isHistogramMinuteMemoryCache(), baseDirectory, proxyConfig.getHistogramMinuteAccumulatorSize(), proxyConfig.getHistogramMinuteAvgKeyBytes(), proxyConfig.getHistogramMinuteAvgDigestBytes(), proxyConfig.getHistogramMinuteCompression(), proxyConfig.isHistogramMinuteAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histHourPorts, pointHandler, remoteHostAnnotator, Granularity.HOUR, proxyConfig.getHistogramHourFlushSecs(), proxyConfig.isHistogramHourMemoryCache(), baseDirectory, proxyConfig.getHistogramHourAccumulatorSize(), proxyConfig.getHistogramHourAvgKeyBytes(), proxyConfig.getHistogramHourAvgDigestBytes(), proxyConfig.getHistogramHourCompression(), proxyConfig.isHistogramHourAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDayPorts, pointHandler, remoteHostAnnotator, Granularity.DAY, proxyConfig.getHistogramDayFlushSecs(), proxyConfig.isHistogramDayMemoryCache(), baseDirectory, proxyConfig.getHistogramDayAccumulatorSize(), proxyConfig.getHistogramDayAvgKeyBytes(), proxyConfig.getHistogramDayAvgDigestBytes(), proxyConfig.getHistogramDayCompression(), proxyConfig.isHistogramDayAccumulatorPersisted(), spanSampler);
            startHistogramListeners(histDistPorts, pointHandler, remoteHostAnnotator, null, proxyConfig.getHistogramDistFlushSecs(), proxyConfig.isHistogramDistMemoryCache(), baseDirectory, proxyConfig.getHistogramDistAccumulatorSize(), proxyConfig.getHistogramDistAvgKeyBytes(), proxyConfig.getHistogramDistAvgDigestBytes(), proxyConfig.getHistogramDistCompression(), proxyConfig.isHistogramDistAccumulatorPersisted(), spanSampler);
        }
    }
    if (StringUtils.isNotBlank(proxyConfig.getGraphitePorts()) || StringUtils.isNotBlank(proxyConfig.getPicklePorts())) {
        if (tokenAuthenticator.authRequired()) {
            logger.warning("Graphite mode is not compatible with HTTP authentication, ignoring");
        } else {
            Preconditions.checkNotNull(proxyConfig.getGraphiteFormat(), "graphiteFormat must be supplied to enable graphite support");
            Preconditions.checkNotNull(proxyConfig.getGraphiteDelimiters(), "graphiteDelimiters must be supplied to enable graphite support");
            GraphiteFormatter graphiteFormatter = new GraphiteFormatter(proxyConfig.getGraphiteFormat(), proxyConfig.getGraphiteDelimiters(), proxyConfig.getGraphiteFieldsToRemove());
            csvToList(proxyConfig.getGraphitePorts()).forEach(strPort -> {
                preprocessors.getSystemPreprocessor(strPort).forPointLine().addTransformer(0, graphiteFormatter);
                startGraphiteListener(strPort, handlerFactory, null, spanSampler);
                logger.info("listening on port: " + strPort + " for graphite metrics");
            });
            csvToList(proxyConfig.getPicklePorts()).forEach(strPort -> startPickleListener(strPort, handlerFactory, graphiteFormatter));
        }
    }
    csvToList(proxyConfig.getOpentsdbPorts()).forEach(strPort -> startOpenTsdbListener(strPort, handlerFactory));
    if (proxyConfig.getDataDogJsonPorts() != null) {
        HttpClient httpClient = HttpClientBuilder.create().useSystemProperties().setUserAgent(proxyConfig.getHttpUserAgent()).setConnectionTimeToLive(1, TimeUnit.MINUTES).setMaxConnPerRoute(100).setMaxConnTotal(100).setRetryHandler(new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)).setDefaultRequestConfig(RequestConfig.custom().setContentCompressionEnabled(true).setRedirectsEnabled(true).setConnectTimeout(proxyConfig.getHttpConnectTimeout()).setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()).setSocketTimeout(proxyConfig.getHttpRequestTimeout()).build()).build();
        csvToList(proxyConfig.getDataDogJsonPorts()).forEach(strPort -> startDataDogListener(strPort, handlerFactory, httpClient));
    }
    csvToList(proxyConfig.getTraceListenerPorts()).forEach(strPort -> startTraceListener(strPort, handlerFactory, spanSampler));
    csvToList(proxyConfig.getCustomTracingListenerPorts()).forEach(strPort -> startCustomTracingListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler));
    csvToList(proxyConfig.getTraceJaegerListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerGrpcListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerGrpcListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceJaegerHttpListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceJaegerHttpListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getTraceZipkinListenerPorts()).forEach(strPort -> {
        PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
        preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
        startTraceZipkinListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
    });
    csvToList(proxyConfig.getPushRelayListenerPorts()).forEach(strPort -> startRelayListener(strPort, handlerFactory, remoteHostAnnotator));
    csvToList(proxyConfig.getJsonListenerPorts()).forEach(strPort -> startJsonListener(strPort, handlerFactory));
    csvToList(proxyConfig.getWriteHttpJsonListenerPorts()).forEach(strPort -> startWriteHttpJsonListener(strPort, handlerFactory));
    // Logs ingestion.
    if (proxyConfig.getFilebeatPort() > 0 || proxyConfig.getRawLogsPort() > 0) {
        if (loadLogsIngestionConfig() != null) {
            logger.info("Initializing logs ingestion");
            try {
                final LogsIngester logsIngester = new LogsIngester(handlerFactory, this::loadLogsIngestionConfig, proxyConfig.getPrefix());
                logsIngester.start();
                if (proxyConfig.getFilebeatPort() > 0) {
                    startLogsIngestionListener(proxyConfig.getFilebeatPort(), logsIngester);
                }
                if (proxyConfig.getRawLogsPort() > 0) {
                    startRawLogsIngestionListener(proxyConfig.getRawLogsPort(), logsIngester);
                }
            } catch (ConfigurationException e) {
                logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
            }
        } else {
            logger.warning("Cannot start logsIngestion: invalid configuration or no config specified");
        }
    }
    setupMemoryGuard();
}

Example 4 with QueueingFactoryImpl

use of com.wavefront.agent.queueing.QueueingFactoryImpl in project java by wavefrontHQ.

the class HttpEndToEndTest method testEndToEndHistograms.

@Test
public void testEndToEndHistograms() throws Exception {
    AtomicInteger successfulSteps = new AtomicInteger(0);
    AtomicInteger testCounter = new AtomicInteger(0);
    long time = (Clock.now() / 1000) / 60 * 60 + 30;
    AtomicLong digestTime = new AtomicLong(System.currentTimeMillis());
    proxyPort = findAvailablePort(2898);
    int histMinPort = findAvailablePort(40001);
    int histHourPort = findAvailablePort(40002);
    int histDayPort = findAvailablePort(40003);
    int histDistPort = findAvailablePort(40000);
    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
    proxy = new PushAgent();
    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
    proxy.proxyConfig.flushThreads = 1;
    proxy.proxyConfig.histogramMinuteListenerPorts = String.valueOf(histMinPort);
    proxy.proxyConfig.histogramHourListenerPorts = String.valueOf(histHourPort);
    proxy.proxyConfig.histogramDayListenerPorts = String.valueOf(histDayPort);
    proxy.proxyConfig.histogramDistListenerPorts = String.valueOf(histDistPort);
    proxy.proxyConfig.histogramMinuteAccumulatorPersisted = false;
    proxy.proxyConfig.histogramHourAccumulatorPersisted = false;
    proxy.proxyConfig.histogramDayAccumulatorPersisted = false;
    proxy.proxyConfig.histogramDistAccumulatorPersisted = false;
    proxy.proxyConfig.histogramMinuteMemoryCache = false;
    proxy.proxyConfig.histogramHourMemoryCache = false;
    proxy.proxyConfig.histogramDayMemoryCache = false;
    proxy.proxyConfig.histogramDistMemoryCache = false;
    proxy.proxyConfig.histogramMinuteFlushSecs = 1;
    proxy.proxyConfig.histogramHourFlushSecs = 1;
    proxy.proxyConfig.histogramDayFlushSecs = 1;
    proxy.proxyConfig.histogramDistFlushSecs = 1;
    proxy.proxyConfig.histogramMinuteAccumulatorSize = 10L;
    proxy.proxyConfig.histogramHourAccumulatorSize = 10L;
    proxy.proxyConfig.histogramDayAccumulatorSize = 10L;
    proxy.proxyConfig.histogramDistAccumulatorSize = 10L;
    proxy.proxyConfig.histogramAccumulatorFlushInterval = 10000L;
    proxy.proxyConfig.histogramAccumulatorResolveInterval = 10000L;
    proxy.proxyConfig.splitPushWhenRateLimited = true;
    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
    proxy.proxyConfig.pushFlushInterval = 10000;
    proxy.proxyConfig.bufferFile = buffer;
    proxy.proxyConfig.timeProvider = digestTime::get;
    proxy.start(new String[] {});
    waitUntilListenerIsOnline(histDistPort);
    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl))
        fail();
    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl))
        fail();
    String payloadHistograms = "metric.name 1 " + time + " source=metric.source tagk1=tagv1\n" + "metric.name 1 " + time + " source=metric.source tagk1=tagv1\n" + "metric.name 2 " + (time + 1) + " source=metric.source tagk1=tagv1\n" + "metric.name 2 " + (time + 2) + " source=metric.source tagk1=tagv1\n" + "metric.name 3 " + time + " source=metric.source tagk1=tagv2\n" + "metric.name 4 " + time + " source=metric.source tagk1=tagv2\n" + "metric.name 5 " + (time + 60) + " source=metric.source tagk1=tagv1\n" + "metric.name 6 " + (time + 60) + " source=metric.source tagk1=tagv1\n";
    long minuteBin = time / 60 * 60;
    long hourBin = time / 3600 * 3600;
    long dayBin = time / 86400 * 86400;
    Set<String> expectedHistograms = ImmutableSet.of("!M " + minuteBin + " #2 1.0 #2 2.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"", "!M " + minuteBin + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" " + "\"tagk1\"=\"tagv2\"", "!M " + (minuteBin + 60) + " #1 5.0 #1 6.0 \"metric.name\" source=\"metric.source\" " + "\"tagk1\"=\"tagv1\"", "!H " + hourBin + " #2 1.0 #2 2.0 #1 5.0 #1 6.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"", "!H " + hourBin + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" " + "\"tagk1\"=\"tagv2\"", "!D " + dayBin + " #1 3.0 #1 4.0 \"metric.name\" source=\"metric.source\" " + "\"tagk1\"=\"tagv2\"", "!D " + dayBin + " #2 1.0 #2 2.0 #1 5.0 #1 6.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"");
    String distPayload = "!M " + minuteBin + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n" + "!M " + minuteBin + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n" + "!H " + minuteBin + " #1 1.0 #1 1.0 #1 2.0 #1 2.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"\n";
    Set<String> expectedDists = ImmutableSet.of("!M " + minuteBin + " #4 1.0 #4 2.0 " + "\"metric.name\" source=\"metric.source\" \"tagk1\"=\"tagv1\"", "!H " + hourBin + " #2 1.0 #2 2.0 \"metric.name\" " + "source=\"metric.source\" \"tagk1\"=\"tagv1\"");
    server.update(req -> {
        String content = req.content().toString(CharsetUtil.UTF_8);
        URI uri;
        try {
            uri = new URI(req.uri());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        String path = uri.getPath();
        assertEquals(HttpMethod.POST, req.method());
        assertEquals("/api/v2/wfproxy/report", path);
        logger.fine("Content received: " + content);
        switch(testCounter.incrementAndGet()) {
            case 1:
                assertEquals(expectedHistograms, new HashSet<>(Arrays.asList(content.split("\n"))));
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 2:
                assertEquals(expectedDists, new HashSet<>(Arrays.asList(content.split("\n"))));
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
        }
        return makeResponse(HttpResponseStatus.OK, "");
    });
    digestTime.set(System.currentTimeMillis() - 1001);
    gzippedHttpPost("http://localhost:" + histMinPort + "/", payloadHistograms);
    gzippedHttpPost("http://localhost:" + histHourPort + "/", payloadHistograms);
    gzippedHttpPost("http://localhost:" + histDayPort + "/", payloadHistograms);
    // should reject
    gzippedHttpPost("http://localhost:" + histDistPort + "/", payloadHistograms);
    digestTime.set(System.currentTimeMillis());
    proxy.histogramFlushRunnables.forEach(Runnable::run);
    HandlerKey key = HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports");
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
    digestTime.set(System.currentTimeMillis() - 1001);
    gzippedHttpPost("http://localhost:" + histDistPort + "/", distPayload);
    digestTime.set(System.currentTimeMillis());
    proxy.histogramFlushRunnables.forEach(Runnable::run);
    ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
    assertEquals(2, successfulSteps.getAndSet(0));
}

Example 5 with QueueingFactoryImpl

use of com.wavefront.agent.queueing.QueueingFactoryImpl in project java by wavefrontHQ.

the class HttpEndToEndTest method testEndToEndSourceTags.

@Test
public void testEndToEndSourceTags() throws Exception {
    AtomicInteger successfulSteps = new AtomicInteger(0);
    AtomicInteger testCounter = new AtomicInteger(0);
    proxyPort = findAvailablePort(2898);
    String buffer = File.createTempFile("proxyTestBuffer", null).getPath();
    proxy = new PushAgent();
    proxy.proxyConfig.server = "http://localhost:" + backendPort + "/api/";
    proxy.proxyConfig.flushThreads = 1;
    proxy.proxyConfig.flushThreadsSourceTags = 1;
    proxy.proxyConfig.splitPushWhenRateLimited = true;
    proxy.proxyConfig.pushListenerPorts = String.valueOf(proxyPort);
    proxy.proxyConfig.pushFlushInterval = 10000;
    proxy.proxyConfig.pushRateLimitSourceTags = 100;
    proxy.proxyConfig.bufferFile = buffer;
    proxy.start(new String[] {});
    waitUntilListenerIsOnline(proxyPort);
    if (!(proxy.senderTaskFactory instanceof SenderTaskFactoryImpl))
        fail();
    if (!(proxy.queueingFactory instanceof QueueingFactoryImpl))
        fail();
    String payloadSourceTags = "@SourceTag action=add source=testSource addTag1 addTag2 addTag3\n" + "@SourceTag action=save source=testSource newtag1 newtag2\n" + "@SourceTag action=delete source=testSource deleteTag\n" + "@SourceDescription action=save source=testSource \"Long Description\"\n" + "@SourceDescription action=delete source=testSource";
    server.update(req -> {
        String content = req.content().toString(CharsetUtil.UTF_8);
        URI uri;
        try {
            uri = new URI(req.uri());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        String path = uri.getPath();
        logger.fine("Content received: " + content);
        switch(testCounter.incrementAndGet()) {
            case 1:
                assertEquals(HttpMethod.PUT, req.method());
                assertEquals("/api/v2/source/testSource/tag/addTag1", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 2:
                assertEquals(HttpMethod.PUT, req.method());
                assertEquals("/api/v2/source/testSource/tag/addTag2", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 3:
                assertEquals(HttpMethod.PUT, req.method());
                assertEquals("/api/v2/source/testSource/tag/addTag3", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 4:
                assertEquals(HttpMethod.POST, req.method());
                assertEquals("/api/v2/source/testSource/tag", path);
                assertEquals("[\"newtag1\",\"newtag2\"]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE, "");
            case 5:
                assertEquals(HttpMethod.DELETE, req.method());
                assertEquals("/api/v2/source/testSource/tag/deleteTag", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 6:
                assertEquals(HttpMethod.POST, req.method());
                assertEquals("/api/v2/source/testSource/description", path);
                assertEquals("Long Description", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.INTERNAL_SERVER_ERROR, "");
            case 7:
                assertEquals(HttpMethod.DELETE, req.method());
                assertEquals("/api/v2/source/testSource/description", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.valueOf(407), "");
            case 8:
                assertEquals(HttpMethod.POST, req.method());
                assertEquals("/api/v2/source/testSource/tag", path);
                assertEquals("[\"newtag1\",\"newtag2\"]", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 9:
                assertEquals(HttpMethod.POST, req.method());
                assertEquals("/api/v2/source/testSource/description", path);
                assertEquals("Long Description", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
            case 10:
                assertEquals(HttpMethod.DELETE, req.method());
                assertEquals("/api/v2/source/testSource/description", path);
                assertEquals("", content);
                successfulSteps.incrementAndGet();
                return makeResponse(HttpResponseStatus.OK, "");
        }
        logger.warning("Too many requests");
        // this will force the assert to fail
        successfulSteps.incrementAndGet();
        return makeResponse(HttpResponseStatus.OK, "");
    });
    gzippedHttpPost("http://localhost:" + proxyPort + "/", payloadSourceTags);
    HandlerKey key = HandlerKey.of(ReportableEntityType.SOURCE_TAG, String.valueOf(proxyPort));
    for (int i = 0; i < 2; i++) ((SenderTaskFactoryImpl) proxy.senderTaskFactory).flushNow(key);
    for (int i = 0; i < 4; i++) ((QueueingFactoryImpl) proxy.queueingFactory).flushNow(key);
    assertEquals(10, successfulSteps.getAndSet(0));
}