Examples with Histogram org.HdrHistogram.Histogram used on opensource projects

Example 26 with Histogram

use of org.HdrHistogram.Histogram in project hazelcast by hazelcast.

the class HyperLogLogImplTest method testEstimateErrorRateForBigCardinalities.

/**
     * - Add up-to runLength() random numbers on both a Set and a HyperLogLog encoder.
     * - Sample the actual count, and the estimate respectively every 100 operations.
     * - Compute the error rate, of the measurements and store it in a histogram.
     * - Assert that the 99th percentile of the histogram is less than the expected max error,
     * which is the result of std error (1.04 / sqrt(m)) + 3%.
     * (2% is the typical accuracy, but tests on the implementation showed up rare occurrences of 3%)
     */
@Test
public void testEstimateErrorRateForBigCardinalities() {
    double stdError = (1.04 / Math.sqrt(1 << precision)) * 100;
    double maxError = Math.ceil(stdError + 3.0);
    IntHashSet actualCount = new IntHashSet(runLength, -1);
    Random random = new Random();
    Histogram histogram = new Histogram(5);
    ByteBuffer bb = ByteBuffer.allocate(4);
    int sampleStep = 100;
    long expected;
    long actual;
    for (int i = 1; i <= runLength; i++) {
        int toCount = random.nextInt();
        actualCount.add(toCount);
        bb.clear();
        bb.putInt(toCount);
        hyperLogLog.add(HashUtil.MurmurHash3_x64_64(bb.array(), 0, bb.array().length));
        if (i % sampleStep == 0) {
            expected = actualCount.size();
            actual = hyperLogLog.estimate();
            double errorPct = ((actual * 100.0) / expected) - 100;
            histogram.recordValue(Math.abs((long) (errorPct * 100)));
        }
    }
    double errorPerc99 = histogram.getValueAtPercentile(99) / 100.0;
    if (errorPerc99 > maxError) {
        fail("For P=" + precision + ", Expected max error=" + maxError + "%." + " Actual error: " + errorPerc99 + "%.");
    }
}

Example 27 with Histogram

use of org.HdrHistogram.Histogram in project LatencyUtils by LatencyUtils.

the class LatencyStats method getIntervalHistogram.

// Interval Histogram access:
/**
     * Get a new interval histogram which will include the value counts accumulated since the last
     * interval histogram was taken.
     * <p>
     * Calling {@link #getIntervalHistogram}() will reset
     * the interval value counts, and start accumulating value counts for the next interval.
     *
     * @return a copy of the latest interval latency histogram
     */
public synchronized Histogram getIntervalHistogram() {
    Histogram intervalHistogram = new Histogram(lowestTrackableLatency, highestTrackableLatency, numberOfSignificantValueDigits);
    getIntervalHistogramInto(intervalHistogram);
    return intervalHistogram;
}

Example 28 with Histogram

use of org.HdrHistogram.Histogram in project LatencyUtils by LatencyUtils.

the class LatencyStats method swapPauseCorrectionHistograms.

private void swapPauseCorrectionHistograms() {
    final Histogram tempHistogram = inactivePauseCorrectionsHistogram;
    inactivePauseCorrectionsHistogram = activePauseCorrectionsHistogram;
    activePauseCorrectionsHistogram = tempHistogram;
}

Example 29 with Histogram

use of org.HdrHistogram.Histogram in project pulsar by yahoo.

the class PerformanceClient method runPerformanceTest.

public void runPerformanceTest(long messages, long limit, int numOfTopic, int sizeOfMessage, String baseUrl, String destination) throws InterruptedException, FileNotFoundException {
    ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec"));
    HashMap<String, Tuple> producersMap = new HashMap<>();
    String produceBaseEndPoint = baseUrl + destination;
    for (int i = 0; i < numOfTopic; i++) {
        String topic = produceBaseEndPoint + "1" + "/";
        URI produceUri = URI.create(topic);
        WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true));
        ClientUpgradeRequest produceRequest = new ClientUpgradeRequest();
        SimpleTestProducerSocket produceSocket = new SimpleTestProducerSocket();
        try {
            produceClient.start();
            produceClient.connect(produceSocket, produceUri, produceRequest);
        } catch (IOException e1) {
            log.error("Fail in connecting: [{}]", e1.getMessage());
            return;
        } catch (Exception e1) {
            log.error("Fail in starting client[{}]", e1.getMessage());
            return;
        }
        producersMap.put(produceUri.toString(), new Tuple(produceClient, produceRequest, produceSocket));
    }
    // connection to be established
    TimeUnit.SECONDS.sleep(5);
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(limit);
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (String topic : producersMap.keySet()) {
                    if (messages > 0) {
                        if (totalSent++ >= messages) {
                            log.trace("------------------- DONE -----------------------");
                            Thread.sleep(10000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    if (producersMap.get(topic).getSocket().getSession() == null) {
                        Thread.sleep(10000);
                        System.exit(0);
                    }
                    producersMap.get(topic).getSocket().sendMsg((String) String.valueOf(totalSent), sizeOfMessage);
                    messagesSent.increment();
                    bytesSent.add(1000);
                }
            }
        } catch (Throwable t) {
            log.error(t.getMessage());
            System.exit(0);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-websocket-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to %s \n", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = SimpleTestProducerSocket.recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    TimeUnit.SECONDS.sleep(100);
    executor.shutdown();
}

Example 30 with Histogram

use of org.HdrHistogram.Histogram in project fast-cast by RuedigerMoeller.

the class ShareMemPingPong method oneRun.

public static void oneRun(boolean odd, int runs, long start, File counters) throws IOException {
    Histogram histo = new Histogram(TimeUnit.SECONDS.toNanos(10), 3);
    byte[] msg = new byte[40];
    try (FileChannel fc = new RandomAccessFile(counters, "rw").getChannel()) {
        MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, 0, 1024);
        long address = ((DirectBuffer) mbb).address();
        for (int i = -1; i < runs; i++) {
            for (; ; ) {
                long tim = System.nanoTime();
                long value = UNSAFE.getLongVolatile(null, address);
                boolean isOdd = (value & 1) != 0;
                if (isOdd != odd)
                    // wait for the other side.
                    continue;
                // simulate reading message
                mbb.position(8);
                mbb.get(msg, 0, msg.length);
                // simulate writing msg
                msg[12] = (byte) i;
                mbb.position(8);
                mbb.put(msg, 0, msg.length);
                histo.recordValue(System.nanoTime() - tim);
                // make the change atomic, just in case there is more than one odd/even process
                if (UNSAFE.compareAndSwapLong(null, address, value, value + 1))
                    break;
            }
            if (i == 0) {
                System.out.println("Started");
                start = System.nanoTime();
            }
        }
    }
    System.out.printf("... Finished, average ping/pong took %,d ns%n", (System.nanoTime() - start) / runs);
    histo.outputPercentileDistribution(System.out, 1000.0);
}