Examples with RateLimiter com.google.common.util.concurrent.RateLimiter used on opensource projects

Example 31 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project web3sdk by FISCO-BCOS.

the class PerformanceDTTest method userAddTest.

public void userAddTest(BigInteger count, BigInteger qps) {
    try {
        ThreadPoolTaskExecutor threadPool = new ThreadPoolTaskExecutor();
        threadPool.setCorePoolSize(200);
        threadPool.setMaxPoolSize(500);
        threadPool.setQueueCapacity(count.intValue());
        threadPool.initialize();
        System.out.println("Deploying contract ");
        System.out.println("===================================================================");
        parallelok = ParallelOk.deploy(web3, credentials, new BigInteger("30000000"), new BigInteger("30000000")).send();
        // enable parallel transaction
        parallelok.enableParallel().send();
        parallelokAddr = parallelok.getContractAddress();
        System.out.println("ParallelOk address: " + parallelokAddr);
        RateLimiter limiter = RateLimiter.create(qps.intValue());
        Integer area = count.intValue() / 10;
        long seconds = System.currentTimeMillis() / 1000l;
        System.out.println("===================================================================");
        System.out.println("Start UserAdd test, count " + count);
        long startTime = System.currentTimeMillis();
        this.collector.setStartTimestamp(startTime);
        for (Integer i = 0; i < count.intValue(); ++i) {
            final int index = i;
            threadPool.execute(new Runnable() {

                @Override
                public void run() {
                    boolean success = false;
                    while (!success) {
                        limiter.acquire();
                        String user = Long.toHexString(seconds) + Integer.toHexString(index);
                        BigInteger amount = new BigInteger("1000000000");
                        DagTransferUser dtu = new DagTransferUser();
                        dtu.setUser(user);
                        dtu.setAmount(amount);
                        PerformanceDTCallback callback = new PerformanceDTCallback();
                        callback.setCollector(collector);
                        callback.setDagTransferUser(dtu);
                        callback.setDagUserMgr(getDagUserMgr());
                        callback.setCallBackType("set");
                        try {
                            callback.recordStartTime();
                            parallelok.set(user, amount, callback);
                            success = true;
                        } catch (Exception e) {
                            success = false;
                            continue;
                        }
                    }
                    int current = sended.incrementAndGet();
                    if (current >= area && ((current % area) == 0)) {
                        long elapsed = System.currentTimeMillis() - startTime;
                        double sendSpeed = current / ((double) elapsed / 1000);
                        System.out.println("Already sended: " + current + "/" + count + " transactions" + ",QPS=" + sendSpeed);
                    }
                }
            });
        }
        // end or not
        while (!collector.isEnd()) {
            Thread.sleep(3000);
            logger.info(" received: {}, total: {}", collector.getReceived().intValue(), collector.getTotal());
        }
        dagUserMgr.setContractAddr(parallelokAddr);
        dagUserMgr.writeDagTransferUser();
        System.exit(0);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(0);
    }
}

Example 32 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project pulsar by yahoo.

the class CmdConsume method run.

/**
     * Run the consume command.
     *
     * @return 0 for success, < 0 otherwise
     */
public int run() throws PulsarClientException, IOException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.serviceURL == null || this.serviceURL.isEmpty())
        throw (new ParameterException("Broker URL is not provided."));
    if (this.subscriptionName == null || this.subscriptionName.isEmpty())
        throw (new ParameterException("Subscription name is not provided."));
    if (this.numMessagesToConsume < 0)
        throw (new ParameterException("Number of messages should be zero or positive."));
    String topic = this.mainOptions.get(0);
    int numMessagesConsumed = 0;
    int returnCode = 0;
    try {
        ConsumerConfiguration consumerConf = new ConsumerConfiguration();
        consumerConf.setSubscriptionType(this.subscriptionType);
        PulsarClient client = PulsarClient.create(this.serviceURL, this.clientConfig);
        Consumer consumer = client.subscribe(topic, this.subscriptionName, consumerConf);
        RateLimiter limiter = (this.consumeRate > 0) ? RateLimiter.create(this.consumeRate) : null;
        while (this.numMessagesToConsume == 0 || numMessagesConsumed < this.numMessagesToConsume) {
            if (limiter != null) {
                limiter.acquire();
            }
            Message msg = consumer.receive(5, TimeUnit.SECONDS);
            if (msg == null) {
                LOG.warn("No message to consume after waiting for 20 seconds.");
            } else {
                numMessagesConsumed += 1;
                System.out.println(MESSAGE_BOUNDARY);
                String output = this.interpretMessage(msg, displayHex);
                System.out.println(output);
                consumer.acknowledge(msg);
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while consuming messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully consumed", numMessagesConsumed);
    }
    return returnCode;
}

Example 33 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project pulsar by yahoo.

the class CmdProduce method run.

/**
     * Run the producer.
     *
     * @return 0 for success, < 0 otherwise
     * @throws Exception
     */
public int run() throws PulsarClientException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.serviceURL == null || this.serviceURL.isEmpty())
        throw (new ParameterException("Broker URL is not provided."));
    if (this.numTimesProduce <= 0)
        throw (new ParameterException("Number of times need to be positive number."));
    if (messages.size() == 0 && messageFileNames.size() == 0)
        throw (new ParameterException("Please supply message content with either --messages or --files"));
    int totalMessages = (messages.size() + messageFileNames.size()) * numTimesProduce;
    if (totalMessages > MAX_MESSAGES) {
        String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages";
        throw new ParameterException(msg);
    }
    String topic = this.mainOptions.get(0);
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = PulsarClient.create(this.serviceURL, this.clientConfig);
        Producer producer = client.createProducer(topic);
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        for (int i = 0; i < this.numTimesProduce; i++) {
            List<Message> messages = generateMessages(messageBodies);
            for (Message msg : messages) {
                if (limiter != null)
                    limiter.acquire();
                producer.send(msg);
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}

Example 34 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project pulsar by yahoo.

the class PerformanceClient method runPerformanceTest.

public void runPerformanceTest(long messages, long limit, int numOfTopic, int sizeOfMessage, String baseUrl, String destination) throws InterruptedException, FileNotFoundException {
    ExecutorService executor = Executors.newCachedThreadPool(new DefaultThreadFactory("pulsar-perf-producer-exec"));
    HashMap<String, Tuple> producersMap = new HashMap<>();
    String produceBaseEndPoint = baseUrl + destination;
    for (int i = 0; i < numOfTopic; i++) {
        String topic = produceBaseEndPoint + "1" + "/";
        URI produceUri = URI.create(topic);
        WebSocketClient produceClient = new WebSocketClient(new SslContextFactory(true));
        ClientUpgradeRequest produceRequest = new ClientUpgradeRequest();
        SimpleTestProducerSocket produceSocket = new SimpleTestProducerSocket();
        try {
            produceClient.start();
            produceClient.connect(produceSocket, produceUri, produceRequest);
        } catch (IOException e1) {
            log.error("Fail in connecting: [{}]", e1.getMessage());
            return;
        } catch (Exception e1) {
            log.error("Fail in starting client[{}]", e1.getMessage());
            return;
        }
        producersMap.put(produceUri.toString(), new Tuple(produceClient, produceRequest, produceSocket));
    }
    // connection to be established
    TimeUnit.SECONDS.sleep(5);
    executor.submit(() -> {
        try {
            RateLimiter rateLimiter = RateLimiter.create(limit);
            // Send messages on all topics/producers
            long totalSent = 0;
            while (true) {
                for (String topic : producersMap.keySet()) {
                    if (messages > 0) {
                        if (totalSent++ >= messages) {
                            log.trace("------------------- DONE -----------------------");
                            Thread.sleep(10000);
                            System.exit(0);
                        }
                    }
                    rateLimiter.acquire();
                    if (producersMap.get(topic).getSocket().getSession() == null) {
                        Thread.sleep(10000);
                        System.exit(0);
                    }
                    producersMap.get(topic).getSocket().sendMsg((String) String.valueOf(totalSent), sizeOfMessage);
                    messagesSent.increment();
                    bytesSent.add(1000);
                }
            }
        } catch (Throwable t) {
            log.error(t.getMessage());
            System.exit(0);
        }
    });
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    String statsFileName = "perf-websocket-producer-" + System.currentTimeMillis() + ".hgrm";
    log.info("Dumping latency stats to %s \n", statsFileName);
    PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
    HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
    // Some log header bits
    histogramLogWriter.outputLogFormatVersion();
    histogramLogWriter.outputLegend();
    while (true) {
        try {
            Thread.sleep(5000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = SimpleTestProducerSocket.recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0));
        histogramLogWriter.outputIntervalHistogram(reportHistogram);
        reportHistogram.reset();
        oldTime = now;
    }
    TimeUnit.SECONDS.sleep(100);
    executor.shutdown();
}

Example 35 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project pulsar by yahoo.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one destination name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener listener = new MessageListener() {

        public void received(Consumer consumer, Message msg) {
            messagesReceived.increment();
            bytesReceived.add(msg.getData().length);
            if (limiter != null) {
                limiter.acquire();
            }
            consumer.acknowledgeAsync(msg);
        }
    };
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    List<Future<Consumer>> futures = Lists.newArrayList();
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setMessageListener(listener);
    consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize);
    for (int i = 0; i < arguments.numDestinations; i++) {
        final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i));
        log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig));
        }
    }
    for (Future<Consumer> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Throughput received: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}