Examples with RateLimiter com.google.common.util.concurrent.RateLimiter used on opensource projects

Example 51 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project pulsar by yahoo.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one destination name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener listener = new MessageListener() {

        public void received(Consumer consumer, Message msg) {
            messagesReceived.increment();
            bytesReceived.add(msg.getData().length);
            if (limiter != null) {
                limiter.acquire();
            }
            consumer.acknowledgeAsync(msg);
        }
    };
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    List<Future<Consumer>> futures = Lists.newArrayList();
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setMessageListener(listener);
    consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize);
    for (int i = 0; i < arguments.numDestinations; i++) {
        final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i));
        log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig));
        }
    }
    for (Future<Consumer> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Throughput received: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}

Example 52 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project opennms by OpenNMS.

the class SyslogKafkaElasticsearchBufferingIT method testMinionSyslogsOverKafkaToEsRest.

@Test
public void testMinionSyslogsOverKafkaToEsRest() throws Exception {
    Date startOfTest = new Date();
    int numMessages = 1000;
    int packetsPerSecond = 50;
    InetSocketAddress minionSshAddr = testEnvironment.getServiceAddress(ContainerAlias.MINION, 8201);
    InetSocketAddress opennmsSshAddr = testEnvironment.getServiceAddress(ContainerAlias.OPENNMS, 8101);
    InetSocketAddress kafkaAddress = testEnvironment.getServiceAddress(ContainerAlias.KAFKA, 9092);
    InetSocketAddress zookeeperAddress = testEnvironment.getServiceAddress(ContainerAlias.KAFKA, 2181);
    // Install the Kafka syslog and trap handlers on the Minion system
    installFeaturesOnMinion(minionSshAddr, kafkaAddress);
    // Install the Kafka and Elasticsearch features on the OpenNMS system
    installFeaturesOnOpenNMS(opennmsSshAddr, kafkaAddress, zookeeperAddress);
    final String sender = testEnvironment.getContainerInfo(ContainerAlias.SNMPD).networkSettings().ipAddress();
    // Wait for the minion to show up
    await().atMost(90, SECONDS).pollInterval(5, SECONDS).until(DaoUtils.countMatchingCallable(getDaoFactory().getDao(MinionDaoHibernate.class), new CriteriaBuilder(OnmsMinion.class).gt("lastUpdated", startOfTest).eq("location", "MINION").toCriteria()), is(1));
    // Shut down OpenNMS. Syslog messages will accumulate in the Kafka
    // message queue while it is down.
    stopContainer(ContainerAlias.OPENNMS);
    LOG.info("Warming up syslog routes by sending 100 packets");
    // Warm up the routes
    sendMessage(ContainerAlias.MINION, sender, 100);
    for (int i = 0; i < 20; i++) {
        LOG.info("Slept for " + i + " seconds");
        Thread.sleep(1000);
    }
    // Make sure that this evenly divides into the numMessages
    final int chunk = 50;
    // Make sure that this is an even multiple of chunk
    final int logEvery = 100;
    int count = 0;
    long start = System.currentTimeMillis();
    // Send ${numMessages} syslog messages
    RateLimiter limiter = RateLimiter.create(packetsPerSecond);
    for (int i = 0; i < (numMessages / chunk); i++) {
        limiter.acquire(chunk);
        sendMessage(ContainerAlias.MINION, sender, chunk);
        count += chunk;
        if (count % logEvery == 0) {
            long mid = System.currentTimeMillis();
            LOG.info(String.format("Sent %d packets in %d milliseconds", logEvery, mid - start));
            start = System.currentTimeMillis();
        }
    }
    // Start OpenNMS. It should begin to consume syslog messages and forward
    // them to Elasticsearch without dropping messages.
    startContainer(ContainerAlias.OPENNMS);
    // 100 warm-up messages plus ${numMessages} messages
    pollForElasticsearchEventsUsingJest(this::getEs5Address, 100 + numMessages);
}

Example 53 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project tutorials by eugenp.

the class RateLimiterLongRunningUnitTest method givenLimitedResource_whenUseRateLimiter_thenShouldLimitPermits.

@Test
public void givenLimitedResource_whenUseRateLimiter_thenShouldLimitPermits() {
    // given
    RateLimiter rateLimiter = RateLimiter.create(100);
    // when
    long startTime = ZonedDateTime.now().getSecond();
    IntStream.range(0, 1000).forEach(i -> {
        rateLimiter.acquire();
        doSomeLimitedOperation();
    });
    long elapsedTimeSeconds = ZonedDateTime.now().getSecond() - startTime;
    // then
    assertThat(elapsedTimeSeconds >= 10);
}

Example 54 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project tutorials by eugenp.

the class RateLimiterLongRunningUnitTest method givenLimitedResource_whenRequestTwice_thenShouldPermitWithoutBlocking.

@Test
public void givenLimitedResource_whenRequestTwice_thenShouldPermitWithoutBlocking() {
    // given
    RateLimiter rateLimiter = RateLimiter.create(2);
    // when
    long startTime = ZonedDateTime.now().getSecond();
    rateLimiter.acquire(1);
    doSomeLimitedOperation();
    rateLimiter.acquire(1);
    doSomeLimitedOperation();
    long elapsedTimeSeconds = ZonedDateTime.now().getSecond() - startTime;
    // then
    assertThat(elapsedTimeSeconds <= 1);
}

Example 55 with RateLimiter

use of com.google.common.util.concurrent.RateLimiter in project tutorials by eugenp.

the class RateLimiterLongRunningUnitTest method givenLimitedResource_whenRequestOnce_thenShouldPermitWithoutBlocking.

@Test
public void givenLimitedResource_whenRequestOnce_thenShouldPermitWithoutBlocking() {
    // given
    RateLimiter rateLimiter = RateLimiter.create(100);
    // when
    long startTime = ZonedDateTime.now().getSecond();
    rateLimiter.acquire(100);
    doSomeLimitedOperation();
    long elapsedTimeSeconds = ZonedDateTime.now().getSecond() - startTime;
    // then
    assertThat(elapsedTimeSeconds <= 1);
}