Examples with Histogram org.HdrHistogram.Histogram used on opensource projects

Example 1 with Histogram

use of org.HdrHistogram.Histogram in project storm by apache.

the class ThroughputVsLatency method main.

public static void main(String[] args) throws Exception {
    long ratePerSecond = 500;
    if (args != null && args.length > 0) {
        ratePerSecond = Long.valueOf(args[0]);
    }
    int parallelism = 4;
    if (args != null && args.length > 1) {
        parallelism = Integer.valueOf(args[1]);
    }
    int numMins = 5;
    if (args != null && args.length > 2) {
        numMins = Integer.valueOf(args[2]);
    }
    String name = "wc-test";
    if (args != null && args.length > 3) {
        name = args[3];
    }
    Config conf = new Config();
    HttpForwardingMetricsServer metricServer = new HttpForwardingMetricsServer(conf) {

        @Override
        public void handle(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
            String worker = taskInfo.srcWorkerHost + ":" + taskInfo.srcWorkerPort;
            for (DataPoint dp : dataPoints) {
                if ("comp-lat-histo".equals(dp.name) && dp.value instanceof Histogram) {
                    synchronized (_histo) {
                        _histo.add((Histogram) dp.value);
                    }
                } else if ("CPU".equals(dp.name) && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object sys = m.get("sys-ms");
                    if (sys instanceof Number) {
                        _systemCPU.getAndAdd(((Number) sys).longValue());
                    }
                    Object user = m.get("user-ms");
                    if (user instanceof Number) {
                        _userCPU.getAndAdd(((Number) user).longValue());
                    }
                } else if (dp.name.startsWith("GC/") && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object count = m.get("count");
                    if (count instanceof Number) {
                        _gcCount.getAndAdd(((Number) count).longValue());
                    }
                    Object time = m.get("timeMs");
                    if (time instanceof Number) {
                        _gcMs.getAndAdd(((Number) time).longValue());
                    }
                } else if (dp.name.startsWith("memory/") && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object val = m.get("usedBytes");
                    if (val instanceof Number) {
                        MemMeasure mm = _memoryBytes.get(worker);
                        if (mm == null) {
                            mm = new MemMeasure();
                            MemMeasure tmp = _memoryBytes.putIfAbsent(worker, mm);
                            mm = tmp == null ? mm : tmp;
                        }
                        mm.update(((Number) val).longValue());
                    }
                }
            }
        }
    };
    metricServer.serve();
    String url = metricServer.getUrl();
    C cluster = new C(conf);
    conf.setNumWorkers(parallelism);
    conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
    conf.registerMetricsConsumer(org.apache.storm.metric.HttpForwardingMetricsConsumer.class, url, 1);
    Map<String, String> workerMetrics = new HashMap<String, String>();
    if (!cluster.isLocal()) {
        //sigar uses JNI and does not work in local mode
        workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric");
    }
    conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics);
    conf.put(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS, 10);
    conf.put(Config.TOPOLOGY_WORKER_GC_CHILDOPTS, "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewSize=128m -XX:CMSInitiatingOccupancyFraction=70 -XX:-CMSConcurrentMTEnabled");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    TopologyBuilder builder = new TopologyBuilder();
    int numEach = 4 * parallelism;
    builder.setSpout("spout", new FastRandomSentenceSpout(ratePerSecond / numEach), numEach);
    builder.setBolt("split", new SplitSentence(), numEach).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), numEach).fieldsGrouping("split", new Fields("word"));
    try {
        cluster.submitTopology(name, conf, builder.createTopology());
        for (int i = 0; i < numMins * 2; i++) {
            Thread.sleep(30 * 1000);
            printMetrics(cluster, name);
        }
    } finally {
        kill(cluster, name);
    }
    System.exit(0);
}

Example 2 with Histogram

use of org.HdrHistogram.Histogram in project storm by apache.

the class HistogramMetric method getValueAndReset.

@Override
public Object getValueAndReset() {
    Histogram copy = _histo.copy();
    _histo.reset();
    return copy;
}

Example 3 with Histogram

use of org.HdrHistogram.Histogram in project Aeron by real-logic.

the class WriteReceiveUdpPing method main.

public static void main(final String[] args) throws IOException {
    int numChannels = 1;
    if (1 == args.length) {
        numChannels = Integer.parseInt(args[0]);
    }
    final Histogram histogram = new Histogram(TimeUnit.SECONDS.toNanos(10), 3);
    final ByteBuffer buffer = ByteBuffer.allocateDirect(MTU_LENGTH_DEFAULT);
    final DatagramChannel[] receiveChannels = new DatagramChannel[numChannels];
    for (int i = 0; i < receiveChannels.length; i++) {
        receiveChannels[i] = DatagramChannel.open();
        init(receiveChannels[i]);
        receiveChannels[i].bind(new InetSocketAddress("localhost", PONG_PORT + i));
    }
    final InetSocketAddress writeAddress = new InetSocketAddress("localhost", PING_PORT);
    final DatagramChannel writeChannel = DatagramChannel.open();
    init(writeChannel, writeAddress);
    final AtomicBoolean running = new AtomicBoolean(true);
    SigInt.register(() -> running.set(false));
    while (running.get()) {
        measureRoundTrip(histogram, buffer, receiveChannels, writeChannel, running);
        histogram.reset();
        System.gc();
        LockSupport.parkNanos(1000_000_000);
    }
}

Example 4 with Histogram

use of org.HdrHistogram.Histogram in project Aeron by real-logic.

the class SendReceiveUdpPing method main.

public static void main(final String[] args) throws IOException {
    int numChannels = 1;
    if (1 == args.length) {
        numChannels = Integer.parseInt(args[0]);
    }
    final Histogram histogram = new Histogram(TimeUnit.SECONDS.toNanos(10), 3);
    final ByteBuffer buffer = ByteBuffer.allocateDirect(Configuration.MTU_LENGTH_DEFAULT);
    final DatagramChannel[] receiveChannels = new DatagramChannel[numChannels];
    for (int i = 0; i < receiveChannels.length; i++) {
        receiveChannels[i] = DatagramChannel.open();
        init(receiveChannels[i]);
        receiveChannels[i].bind(new InetSocketAddress("localhost", Common.PONG_PORT + i));
    }
    final InetSocketAddress sendAddress = new InetSocketAddress("localhost", Common.PING_PORT);
    final DatagramChannel sendChannel = DatagramChannel.open();
    init(sendChannel);
    final AtomicBoolean running = new AtomicBoolean(true);
    SigInt.register(() -> running.set(false));
    while (running.get()) {
        measureRoundTrip(histogram, sendAddress, buffer, receiveChannels, sendChannel, running);
        histogram.reset();
        System.gc();
        LockSupport.parkNanos(1000_000_000);
    }
}

Example 5 with Histogram

use of org.HdrHistogram.Histogram in project Aeron by real-logic.

the class SendSelectReceiveUdpPing method run.

private void run() throws IOException {
    final Histogram histogram = new Histogram(TimeUnit.SECONDS.toNanos(10), 3);
    final ByteBuffer buffer = ByteBuffer.allocateDirect(Configuration.MTU_LENGTH_DEFAULT);
    final DatagramChannel receiveChannel = DatagramChannel.open();
    Common.init(receiveChannel);
    receiveChannel.bind(new InetSocketAddress("localhost", Common.PONG_PORT));
    final DatagramChannel sendChannel = DatagramChannel.open();
    Common.init(sendChannel);
    final Selector selector = Selector.open();
    final IntSupplier handler = () -> {
        try {
            buffer.clear();
            receiveChannel.receive(buffer);
            final long receivedSequenceNumber = buffer.getLong(0);
            final long timestamp = buffer.getLong(SIZE_OF_LONG);
            if (receivedSequenceNumber != sequenceNumber) {
                throw new IllegalStateException("Data Loss:" + sequenceNumber + " to " + receivedSequenceNumber);
            }
            final long duration = System.nanoTime() - timestamp;
            histogram.recordValue(duration);
        } catch (final IOException ex) {
            ex.printStackTrace();
        }
        return 1;
    };
    receiveChannel.register(selector, OP_READ, handler);
    final AtomicBoolean running = new AtomicBoolean(true);
    SigInt.register(() -> running.set(false));
    while (running.get()) {
        measureRoundTrip(histogram, SEND_ADDRESS, buffer, sendChannel, selector, running);
        histogram.reset();
        System.gc();
        LockSupport.parkNanos(1000 * 1000 * 1000);
    }
}