Examples with CompletableFuture java.util.concurrent.CompletableFuture used on opensource projects

Example 81 with CompletableFuture

use of java.util.concurrent.CompletableFuture in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer.

@Test(dataProvider = "codec")
public void testSimpleBatchProducerConsumer(CompressionType compressionType) throws Exception {
    int numMsgs = 500;
    int numMsgsInBatch = numMsgs / 20;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer";
    final String subscriptionName = "pc-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        if (i % 2 == 0) {
            consumer.acknowledgeCumulative(msg);
        } else {
            lastunackedMsg = msg;
        }
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 82 with CompletableFuture

use of java.util.concurrent.CompletableFuture in project pulsar by yahoo.

the class ZookeeperCacheTest method testGlobalZooKeeperCache.

@Test
void testGlobalZooKeeperCache() throws Exception {
    OrderedSafeExecutor executor = new OrderedSafeExecutor(1, "test");
    ScheduledExecutorService scheduledExecutor = new ScheduledThreadPoolExecutor(1);
    MockZooKeeper zkc = MockZooKeeper.newInstance();
    ZooKeeperClientFactory zkClientfactory = new ZooKeeperClientFactory() {

        @Override
        public CompletableFuture<ZooKeeper> create(String serverList, SessionType sessionType, int zkSessionTimeoutMillis) {
            return CompletableFuture.completedFuture(zkc);
        }
    };
    GlobalZooKeeperCache zkCacheService = new GlobalZooKeeperCache(zkClientfactory, -1, "", executor, scheduledExecutor);
    zkCacheService.start();
    zkClient = (MockZooKeeper) zkCacheService.getZooKeeper();
    ZooKeeperDataCache<String> zkCache = new ZooKeeperDataCache<String>(zkCacheService) {

        @Override
        public String deserialize(String key, byte[] content) throws Exception {
            return new String(content);
        }
    };
    // Create callback counter
    AtomicInteger notificationCount = new AtomicInteger(0);
    ZooKeeperCacheListener<String> counter = (path, data, stat) -> {
        notificationCount.incrementAndGet();
    };
    // Register counter twice and unregister once, so callback should be counted correctly
    zkCache.registerListener(counter);
    zkCache.registerListener(counter);
    zkCache.unregisterListener(counter);
    String value = "test";
    zkClient.create("/my_test", value.getBytes(), null, null);
    assertEquals(zkCache.get("/my_test").get(), value);
    String newValue = "test2";
    // case 1: update and create znode directly and verify that the cache is retrieving the correct data
    assertEquals(notificationCount.get(), 0);
    zkClient.setData("/my_test", newValue.getBytes(), -1);
    zkClient.create("/my_test2", value.getBytes(), null, null);
    // Wait for the watch to be triggered
    while (notificationCount.get() < 1) {
        Thread.sleep(1);
    }
    // retrieve the data from the cache and verify it is the updated/new data
    assertEquals(zkCache.get("/my_test").get(), newValue);
    assertEquals(zkCache.get("/my_test2").get(), value);
    // The callback method should be called just only once
    assertEquals(notificationCount.get(), 1);
    // case 2: force the ZooKeeper session to be expired and verify that the data is still accessible
    zkCacheService.process(new WatchedEvent(Event.EventType.None, KeeperState.Expired, null));
    assertEquals(zkCache.get("/my_test").get(), newValue);
    assertEquals(zkCache.get("/my_test2").get(), value);
    // case 3: update the znode directly while the client session is marked as expired. Verify that the new updates
    // is not seen in the cache
    zkClient.create("/other", newValue.getBytes(), null, null);
    zkClient.failNow(Code.SESSIONEXPIRED);
    assertEquals(zkCache.get("/my_test").get(), newValue);
    assertEquals(zkCache.get("/my_test2").get(), value);
    try {
        zkCache.get("/other");
        fail("shuld have thrown exception");
    } catch (Exception e) {
    // Ok
    }
    // case 4: directly delete the znode while the session is not re-connected yet. Verify that the deletion is not
    // seen by the cache
    zkClient.failAfter(-1, Code.OK);
    zkClient.delete("/my_test2", -1);
    zkCacheService.process(new WatchedEvent(Event.EventType.None, KeeperState.SyncConnected, null));
    assertEquals(zkCache.get("/other").get(), newValue);
    // Make sure that the value is now directly from ZK and deleted
    assertFalse(zkCache.get("/my_test2").isPresent());
    // case 5: trigger a ZooKeeper disconnected event and make sure the cache content is not changed.
    zkCacheService.process(new WatchedEvent(Event.EventType.None, KeeperState.Disconnected, null));
    zkClient.create("/other2", newValue.getBytes(), null, null);
    // case 6: trigger a ZooKeeper SyncConnected event and make sure that the cache content is invalidated s.t. we
    // can see the updated content now
    zkCacheService.process(new WatchedEvent(Event.EventType.None, KeeperState.SyncConnected, null));
    // make sure that we get it
    assertEquals(zkCache.get("/other2").get(), newValue);
    zkCacheService.close();
    executor.shutdown();
    // Update shouldn't happen after the last check
    assertEquals(notificationCount.get(), 1);
}

Example 83 with CompletableFuture

use of java.util.concurrent.CompletableFuture in project pulsar by yahoo.

the class NamespaceService method searchForCandidateBroker.

private void searchForCandidateBroker(NamespaceBundle bundle, CompletableFuture<LookupResult> lookupFuture, boolean authoritative) {
    String candidateBroker = null;
    try {
        // check if this is Heartbeat or SLAMonitor namespace
        candidateBroker = checkHeartbeatNamespace(bundle);
        if (candidateBroker == null) {
            String broker = getSLAMonitorBrokerName(bundle);
            // checking if the broker is up and running
            if (broker != null && isBrokerActive(broker)) {
                candidateBroker = broker;
            }
        }
        if (candidateBroker == null) {
            if (!this.loadManager.isCentralized() || pulsar.getLeaderElectionService().isLeader()) {
                candidateBroker = getLeastLoadedFromLoadManager(bundle);
            } else {
                if (authoritative) {
                    // leader broker already assigned the current broker as owner
                    candidateBroker = pulsar.getWebServiceAddress();
                } else {
                    // forward to leader broker to make assignment
                    candidateBroker = pulsar.getLeaderElectionService().getCurrentLeader().getServiceUrl();
                }
            }
        }
    } catch (Exception e) {
        LOG.warn("Error when searching for candidate broker to acquire {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
        return;
    }
    try {
        checkNotNull(candidateBroker);
        if (pulsar.getWebServiceAddress().equals(candidateBroker)) {
            // Load manager decided that the local broker should try to become the owner
            ownershipCache.tryAcquiringOwnership(bundle).thenAccept(ownerInfo -> {
                if (ownerInfo.isDisabled()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Namespace bundle {} is currently being unloaded", bundle);
                    }
                    lookupFuture.completeExceptionally(new IllegalStateException(String.format("Namespace bundle %s is currently being unloaded", bundle)));
                } else {
                    pulsar.loadNamespaceDestinations(bundle);
                    lookupFuture.complete(new LookupResult(ownerInfo));
                }
            }).exceptionally(exception -> {
                LOG.warn("Failed to acquire ownership for namespace bundle {}: ", bundle, exception.getMessage(), exception);
                lookupFuture.completeExceptionally(new PulsarServerException("Failed to acquire ownership for namespace bundle " + bundle, exception));
                return null;
            });
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Redirecting to broker {} to acquire ownership of bundle {}", candidateBroker, bundle);
            }
            // Now setting the redirect url
            createLookupResult(candidateBroker).thenAccept(lookupResult -> lookupFuture.complete(lookupResult)).exceptionally(ex -> {
                lookupFuture.completeExceptionally(ex);
                return null;
            });
        }
    } catch (Exception e) {
        LOG.warn("Error in trying to acquire namespace bundle ownership for {}: {}", bundle, e.getMessage(), e);
        lookupFuture.completeExceptionally(e);
    }
}

Example 84 with CompletableFuture

use of java.util.concurrent.CompletableFuture in project pulsar by yahoo.

the class NamespaceService method splitAndOwnBundle.

/**
     * 1. split the given bundle into two bundles 2. assign ownership of both the bundles to current broker 3. update
     * policies with newly created bundles into LocalZK 4. disable original bundle and refresh the cache
     *
     * @param bundle
     * @return
     * @throws Exception
     */
public CompletableFuture<Void> splitAndOwnBundle(NamespaceBundle bundle) throws Exception {
    final CompletableFuture<Void> future = new CompletableFuture<>();
    Pair<NamespaceBundles, List<NamespaceBundle>> splittedBundles = bundleFactory.splitBundles(bundle, 2);
    if (splittedBundles != null) {
        checkNotNull(splittedBundles.getLeft());
        checkNotNull(splittedBundles.getRight());
        checkArgument(splittedBundles.getRight().size() == 2, "bundle has to be split in two bundles");
        NamespaceName nsname = bundle.getNamespaceObject();
        try {
            // take ownership of newly split bundles
            for (NamespaceBundle sBundle : splittedBundles.getRight()) {
                checkNotNull(ownershipCache.tryAcquiringOwnership(sBundle));
            }
            updateNamespaceBundles(nsname, splittedBundles.getLeft(), (rc, path, zkCtx, stat) -> pulsar.getOrderedExecutor().submit(safeRun(() -> {
                if (rc == KeeperException.Code.OK.intValue()) {
                    // disable old bundle
                    try {
                        ownershipCache.disableOwnership(bundle);
                        // invalidate cache as zookeeper has new split
                        // namespace bundle
                        bundleFactory.invalidateBundleCache(nsname);
                        // update bundled_topic cache for load-report-generation
                        pulsar.getBrokerService().refreshTopicToStatsMaps(bundle);
                        loadManager.setLoadReportForceUpdateFlag();
                        future.complete(null);
                    } catch (Exception e) {
                        String msg1 = format("failed to disable bundle %s under namespace [%s] with error %s", nsname.toString(), bundle.toString(), e.getMessage());
                        LOG.warn(msg1, e);
                        future.completeExceptionally(new ServiceUnitNotReadyException(msg1));
                    }
                } else {
                    String msg2 = format("failed to update namespace [%s] policies due to %s", nsname.toString(), KeeperException.create(KeeperException.Code.get(rc)).getMessage());
                    LOG.warn(msg2);
                    future.completeExceptionally(new ServiceUnitNotReadyException(msg2));
                }
            })));
        } catch (Exception e) {
            String msg = format("failed to aquire ownership of split bundle for namespace [%s], %s", nsname.toString(), e.getMessage());
            LOG.warn(msg, e);
            future.completeExceptionally(new ServiceUnitNotReadyException(msg));
        }
    } else {
        String msg = format("bundle %s not found under namespace", bundle.toString());
        future.completeExceptionally(new ServiceUnitNotReadyException(msg));
    }
    return future;
}

Example 85 with CompletableFuture

use of java.util.concurrent.CompletableFuture in project pulsar by yahoo.

the class BrokerService method unloadServiceUnit.

/**
     * Unload all the topic served by the broker service under the given service unit
     *
     * @param serviceUnit
     * @return
     */
public CompletableFuture<Integer> unloadServiceUnit(NamespaceBundle serviceUnit) {
    CompletableFuture<Integer> result = new CompletableFuture<Integer>();
    List<CompletableFuture<Void>> closeFutures = Lists.newArrayList();
    topics.forEach((name, topicFuture) -> {
        DestinationName topicName = DestinationName.get(name);
        if (serviceUnit.includes(topicName)) {
            // Topic needs to be unloaded
            log.info("[{}] Unloading topic", topicName);
            closeFutures.add(topicFuture.thenCompose(Topic::close));
        }
    });
    CompletableFuture<Void> aggregator = FutureUtil.waitForAll(closeFutures);
    aggregator.thenAccept(res -> result.complete(closeFutures.size())).exceptionally(ex -> {
        result.completeExceptionally(ex);
        return null;
    });
    return result;
}