Examples with RateLimiter com.google.common.util.concurrent.RateLimiter used on opensource projects

Example 11 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project bookkeeper by apache.

the class TestDisableEnsembleChange method disableEnsembleChangeTest.

void disableEnsembleChangeTest(boolean startNewBookie) throws Exception {
    ClientConfiguration conf = new ClientConfiguration();
    conf.setZkServers(zkUtil.getZooKeeperConnectString());
    conf.setDelayEnsembleChange(false).setDisableEnsembleChangeFeatureName(FEATURE_DISABLE_ENSEMBLE_CHANGE);
    SettableFeatureProvider featureProvider = new SettableFeatureProvider("test", 0);
    BookKeeper bkc = BookKeeper.forConfig(conf).featureProvider(featureProvider).build();
    SettableFeature disableEnsembleChangeFeature = featureProvider.getFeature(FEATURE_DISABLE_ENSEMBLE_CHANGE);
    disableEnsembleChangeFeature.set(true);
    final byte[] password = new byte[0];
    final LedgerHandle lh = bkc.createLedger(4, 3, 2, BookKeeper.DigestType.CRC32, password);
    final AtomicBoolean finished = new AtomicBoolean(false);
    final AtomicBoolean failTest = new AtomicBoolean(false);
    final byte[] entry = "test-disable-ensemble-change".getBytes(UTF_8);
    assertEquals(1, lh.getLedgerMetadata().getEnsembles().size());
    ArrayList<BookieSocketAddress> ensembleBeforeFailure = new ArrayList<>(lh.getLedgerMetadata().getEnsembles().entrySet().iterator().next().getValue());
    final RateLimiter rateLimiter = RateLimiter.create(10);
    Thread addThread = new Thread() {

        @Override
        public void run() {
            try {
                while (!finished.get()) {
                    rateLimiter.acquire();
                    lh.addEntry(entry);
                }
            } catch (Exception e) {
                logger.error("Exception on adding entry : ", e);
                failTest.set(true);
            }
        }
    };
    addThread.start();
    Thread.sleep(2000);
    killBookie(0);
    Thread.sleep(2000);
    finished.set(true);
    addThread.join();
    assertFalse("Should not fail adding entries facing one bookie failure when disable ensemble change", failTest.get());
    // check the ensemble after failure
    assertEquals("No new ensemble should be added when disable ensemble change.", 1, lh.getLedgerMetadata().getEnsembles().size());
    ArrayList<BookieSocketAddress> ensembleAfterFailure = new ArrayList<>(lh.getLedgerMetadata().getEnsembles().entrySet().iterator().next().getValue());
    assertArrayEquals(ensembleBeforeFailure.toArray(new BookieSocketAddress[ensembleBeforeFailure.size()]), ensembleAfterFailure.toArray(new BookieSocketAddress[ensembleAfterFailure.size()]));
    // enable ensemble change
    disableEnsembleChangeFeature.set(false);
    if (startNewBookie) {
        startNewBookie();
    }
    // reset add thread
    finished.set(false);
    final CountDownLatch failLatch = new CountDownLatch(1);
    addThread = new Thread() {

        @Override
        public void run() {
            try {
                while (!finished.get()) {
                    lh.addEntry(entry);
                }
            } catch (Exception e) {
                logger.error("Exception on adding entry : ", e);
                failLatch.countDown();
                failTest.set(true);
            }
        }
    };
    addThread.start();
    failLatch.await(4000, TimeUnit.MILLISECONDS);
    finished.set(true);
    addThread.join();
    if (startNewBookie) {
        assertFalse("Should not fail adding entries when enable ensemble change again.", failTest.get());
        assertFalse("Ledger should be closed when enable ensemble change again.", lh.getLedgerMetadata().isClosed());
        assertEquals("New ensemble should be added when enable ensemble change again.", 2, lh.getLedgerMetadata().getEnsembles().size());
    } else {
        assertTrue("Should fail adding entries when enable ensemble change again.", failTest.get());
        assertTrue("Ledger should be closed when enable ensemble change again.", lh.getLedgerMetadata().isClosed());
    }
}

Example 12 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project tutorials by eugenp.

the class RateLimiterLongRunningUnitTest method givenLimitedResource_whenTryAcquire_shouldNotBlockIndefinitely.

@Test
public void givenLimitedResource_whenTryAcquire_shouldNotBlockIndefinitely() {
    // given
    RateLimiter rateLimiter = RateLimiter.create(1);
    // when
    rateLimiter.acquire();
    boolean result = rateLimiter.tryAcquire(2, 10, TimeUnit.MILLISECONDS);
    // then
    assertThat(result).isFalse();
}

Example 13 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project incubator-pulsar by apache.

the class CmdProduce method run.

/**
 * Run the producer.
 *
 * @return 0 for success, < 0 otherwise
 * @throws Exception
 */
public int run() throws PulsarClientException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.numTimesProduce <= 0)
        throw (new ParameterException("Number of times need to be positive number."));
    if (messages.size() == 0 && messageFileNames.size() == 0)
        throw (new ParameterException("Please supply message content with either --messages or --files"));
    int totalMessages = (messages.size() + messageFileNames.size()) * numTimesProduce;
    if (totalMessages > MAX_MESSAGES) {
        String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages";
        throw new ParameterException(msg);
    }
    String topic = this.mainOptions.get(0);
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = clientBuilder.build();
        Producer<byte[]> producer = client.newProducer().topic(topic).create();
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        for (int i = 0; i < this.numTimesProduce; i++) {
            List<Message<byte[]>> messages = generateMessages(messageBodies);
            for (Message<byte[]> msg : messages) {
                if (limiter != null)
                    limiter.acquire();
                producer.send(msg);
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}

Example 14 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project incubator-pulsar by apache.

the class ManagedLedgerWriter method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments));
    byte[] payloadData = new byte[arguments.msgSize];
    ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize);
    payloadBuffer.writerIndex(arguments.msgSize);
    // Now processing command line arguments
    String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5);
    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setUseV2WireProtocol(true);
    bkConf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class);
    bkConf.setAddEntryTimeout(30);
    bkConf.setReadEntryTimeout(30);
    bkConf.setThrottleValue(0);
    bkConf.setNumChannelsPerBookie(arguments.maxConnections);
    bkConf.setZkServers(arguments.zookeeperServers);
    ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig();
    mlFactoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf);
    ManagedLedgerConfig mlConf = new ManagedLedgerConfig();
    mlConf.setEnsembleSize(arguments.ensembleSize);
    mlConf.setWriteQuorumSize(arguments.writeQuorum);
    mlConf.setAckQuorumSize(arguments.ackQuorum);
    mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES);
    mlConf.setMetadataEnsembleSize(arguments.ensembleSize);
    mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum);
    mlConf.setMetadataAckQuorumSize(arguments.ackQuorum);
    mlConf.setDigestType(arguments.digestType);
    mlConf.setMaxSizePerLedgerMb(2048);
    List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>();
    for (int i = 0; i < arguments.numManagedLedgers; i++) {
        String name = String.format("%s-%03d", managedLedgerPrefix, i);
        CompletableFuture<ManagedLedger> future = new CompletableFuture<>();
        futures.add(future);
        factory.asyncOpen(name, mlConf, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                future.complete(ledger);
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                future.completeExceptionally(exception);
            }
        }, null);
    }
    List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
    log.info("Created {} managed ledgers", managedLedgers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(managedLedgers);
    AtomicBoolean isDone = new AtomicBoolean();
    List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers, Math.max(1, managedLedgers.size() / arguments.numThreads));
    for (int i = 0; i < arguments.numThreads; i++) {
        List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i);
        int nunManagedLedgersForThisThread = managedLedgersForThisThread.size();
        long numMessagesForThisThread = arguments.numMessages / arguments.numThreads;
        int maxOutstandingForThisThread = arguments.maxOutstanding;
        executor.submit(() -> {
            try {
                final double msgRate = arguments.msgRate / (double) arguments.numThreads;
                final RateLimiter rateLimiter = RateLimiter.create(msgRate);
                // Acquire 1 sec worth of messages to have a slower ramp-up
                rateLimiter.acquire((int) msgRate);
                final long startTime = System.currentTimeMillis();
                final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread);
                final AddEntryCallback addEntryCallback = new AddEntryCallback() {

                    @Override
                    public void addComplete(Position position, Object ctx) {
                        long sendTime = (Long) (ctx);
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                        semaphore.release();
                    }

                    @Override
                    public void addFailed(ManagedLedgerException exception, Object ctx) {
                        log.warn("Write error on message", exception);
                        System.exit(-1);
                    }
                };
                // Send messages on all topics/producers
                long totalSent = 0;
                while (true) {
                    for (int j = 0; j < nunManagedLedgersForThisThread; j++) {
                        if (arguments.testTime > 0) {
                            if (System.currentTimeMillis() - startTime > arguments.testTime) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }
                        if (numMessagesForThisThread > 0) {
                            if (totalSent++ >= numMessagesForThisThread) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }
                        semaphore.acquire();
                        rateLimiter.acquire();
                        final long sendTime = System.nanoTime();
                        managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback, sendTime);
                    }
                }
            } catch (Throwable t) {
                log.error("Got error", t);
            }
        });
    }
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        reportHistogram.reset();
        oldTime = now;
    }
    factory.shutdown();
}

Example 15 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project incubator-pulsar by apache.

the class PerformanceReader method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-reader");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one topic name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
        if (arguments.useTls == false) {
            arguments.useTls = Boolean.parseBoolean(prop.getProperty("useTls"));
        }
        if (isBlank(arguments.tlsTrustCertsFilePath)) {
            arguments.tlsTrustCertsFilePath = prop.getProperty("tlsTrustCertsFilePath", "");
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(arguments));
    final TopicName prefixTopicName = TopicName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    ReaderListener<byte[]> listener = (reader, msg) -> {
        messagesReceived.increment();
        bytesReceived.add(msg.getData().length);
        if (limiter != null) {
            limiter.acquire();
        }
    };
    ClientBuilder clientBuilder = // 
    PulsarClient.builder().serviceUrl(// 
    arguments.serviceURL).connectionsPerBroker(// 
    arguments.maxConnections).statsInterval(arguments.statsIntervalSeconds, // 
    TimeUnit.SECONDS).ioThreads(// 
    Runtime.getRuntime().availableProcessors()).enableTls(// 
    arguments.useTls).tlsTrustCertsFilePath(arguments.tlsTrustCertsFilePath);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = clientBuilder.build();
    List<CompletableFuture<Reader<byte[]>>> futures = Lists.newArrayList();
    MessageId startMessageId;
    if ("earliest".equals(arguments.startMessageId)) {
        startMessageId = MessageId.earliest;
    } else if ("latest".equals(arguments.startMessageId)) {
        startMessageId = MessageId.latest;
    } else {
        String[] parts = arguments.startMessageId.split(":");
        startMessageId = new MessageIdImpl(Long.parseLong(parts[0]), Long.parseLong(parts[1]), -1);
    }
    ReaderBuilder<byte[]> readerBuilder = // 
    pulsarClient.newReader().readerListener(// 
    listener).receiverQueueSize(// 
    arguments.receiverQueueSize).startMessageId(startMessageId);
    for (int i = 0; i < arguments.numTopics; i++) {
        final TopicName topicName = (arguments.numTopics == 1) ? prefixTopicName : TopicName.get(String.format("%s-%d", prefixTopicName, i));
        futures.add(readerBuilder.clone().topic(topicName.toString()).createAsync());
    }
    FutureUtil.waitForAll(futures).get();
    log.info("Start reading from {} topics", arguments.numTopics);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Read throughput: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}