Examples with NamespaceBundle org.apache.pulsar.common.naming.NamespaceBundle used on opensource projects

Example 51 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class BrokerService method unloadServiceUnit.

/**
 * Unload all the topic served by the broker service under the given service unit
 *
 * @param serviceUnit
 * @return
 */
public CompletableFuture<Integer> unloadServiceUnit(NamespaceBundle serviceUnit) {
    CompletableFuture<Integer> result = new CompletableFuture<Integer>();
    List<CompletableFuture<Void>> closeFutures = Lists.newArrayList();
    topics.forEach((name, topicFuture) -> {
        TopicName topicName = TopicName.get(name);
        if (serviceUnit.includes(topicName)) {
            // Topic needs to be unloaded
            log.info("[{}] Unloading topic", topicName);
            closeFutures.add(topicFuture.thenCompose(Topic::close));
        }
    });
    CompletableFuture<Void> aggregator = FutureUtil.waitForAll(closeFutures);
    aggregator.thenAccept(res -> result.complete(closeFutures.size())).exceptionally(ex -> {
        result.completeExceptionally(ex);
        return null;
    });
    return result;
}

Example 52 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class TopicLookup method getNamespaceBundle.

@GET
@Path("{topic-domain}/{property}/{cluster}/{namespace}/{topic}/bundle")
@Produces(MediaType.APPLICATION_JSON)
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 405, message = "Invalid topic domain type") })
public String getNamespaceBundle(@PathParam("topic-domain") String topicDomain, @PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String topicName) {
    topicName = Codec.decode(topicName);
    TopicDomain domain = null;
    try {
        domain = TopicDomain.getEnum(topicDomain);
    } catch (IllegalArgumentException e) {
        log.error("[{}] Invalid topic-domain {}", clientAppId(), topicDomain, e);
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Bundle lookup can not be done on topic domain " + topicDomain);
    }
    TopicName topic = TopicName.get(domain.value(), property, cluster, namespace, topicName);
    validateSuperUserAccess();
    try {
        NamespaceBundle bundle = pulsar().getNamespaceService().getBundle(topic);
        return bundle.getBundleRange();
    } catch (Exception e) {
        log.error("[{}] Failed to get namespace bundle for {}", clientAppId(), topic, e);
        throw new RestException(e);
    }
}

Example 53 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class NamespaceService method splitAndOwnBundleOnceAndRetry.

void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, boolean unload, AtomicInteger counter, CompletableFuture<Void> unloadFuture) {
    CompletableFuture<NamespaceBundles> updateFuture = new CompletableFuture<>();
    final Pair<NamespaceBundles, List<NamespaceBundle>> splittedBundles = bundleFactory.splitBundles(bundle, 2);
    // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper.
    if (splittedBundles != null) {
        checkNotNull(splittedBundles.getLeft());
        checkNotNull(splittedBundles.getRight());
        checkArgument(splittedBundles.getRight().size() == 2, "bundle has to be split in two bundles");
        NamespaceName nsname = bundle.getNamespaceObject();
        if (LOG.isDebugEnabled()) {
            LOG.debug("[{}] splitAndOwnBundleOnce: {}, counter: {},  2 bundles: {}, {}", nsname.toString(), bundle.getBundleRange(), counter.get(), splittedBundles != null ? splittedBundles.getRight().get(0).getBundleRange() : "null splittedBundles", splittedBundles != null ? splittedBundles.getRight().get(1).getBundleRange() : "null splittedBundles");
        }
        try {
            // take ownership of newly split bundles
            for (NamespaceBundle sBundle : splittedBundles.getRight()) {
                checkNotNull(ownershipCache.tryAcquiringOwnership(sBundle));
            }
            updateNamespaceBundles(nsname, splittedBundles.getLeft(), (rc, path, zkCtx, stat) -> {
                if (rc == Code.OK.intValue()) {
                    // invalidate cache as zookeeper has new split
                    // namespace bundle
                    bundleFactory.invalidateBundleCache(bundle.getNamespaceObject());
                    updateFuture.complete(splittedBundles.getLeft());
                } else if (rc == Code.BADVERSION.intValue()) {
                    KeeperException keeperException = KeeperException.create(KeeperException.Code.get(rc));
                    String msg = format("failed to update namespace policies [%s], NamespaceBundle: %s " + "due to %s, counter: %d", nsname.toString(), bundle.getBundleRange(), keeperException.getMessage(), counter.get());
                    LOG.warn(msg);
                    updateFuture.completeExceptionally(new ServerMetadataException(keeperException));
                } else {
                    String msg = format("failed to update namespace policies [%s], NamespaceBundle: %s due to %s", nsname.toString(), bundle.getBundleRange(), KeeperException.create(KeeperException.Code.get(rc)).getMessage());
                    LOG.warn(msg);
                    updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
                }
            });
        } catch (Exception e) {
            String msg = format("failed to acquire ownership of split bundle for namespace [%s], %s", nsname.toString(), e.getMessage());
            LOG.warn(msg, e);
            updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        }
    } else {
        String msg = format("bundle %s not found under namespace", bundle.toString());
        LOG.warn(msg);
        updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
    }
    // If success updateNamespaceBundles, then do invalidateBundleCache and unload.
    // Else retry splitAndOwnBundleOnceAndRetry.
    updateFuture.whenCompleteAsync((r, t) -> {
        if (t != null) {
            // retry several times on BadVersion
            if ((t instanceof ServerMetadataException) && (counter.decrementAndGet() >= 0)) {
                pulsar.getOrderedExecutor().submit(() -> splitAndOwnBundleOnceAndRetry(bundle, unload, counter, unloadFuture));
            } else {
                // Retry enough, or meet other exception
                String msg2 = format(" %s not success update nsBundles, counter %d, reason %s", bundle.toString(), counter.get(), t.getMessage());
                LOG.warn(msg2);
                unloadFuture.completeExceptionally(new ServiceUnitNotReadyException(msg2));
            }
            return;
        }
        // success updateNamespaceBundles
        try {
            // disable old bundle in memory
            getOwnershipCache().updateBundleState(bundle, false);
            // update bundled_topic cache for load-report-generation
            pulsar.getBrokerService().refreshTopicToStatsMaps(bundle);
            loadManager.get().setLoadReportForceUpdateFlag();
            if (unload) {
                // unload new split bundles
                r.getBundles().forEach(splitBundle -> {
                    try {
                        unloadNamespaceBundle(splitBundle);
                    } catch (Exception e) {
                        LOG.warn("Failed to unload split bundle {}", splitBundle, e);
                        throw new RuntimeException("Failed to unload split bundle " + splitBundle, e);
                    }
                });
            }
            unloadFuture.complete(null);
        } catch (Exception e) {
            String msg1 = format("failed to disable bundle %s under namespace [%s] with error %s", bundle.getNamespaceObject().toString(), bundle.toString(), e.getMessage());
            LOG.warn(msg1, e);
            unloadFuture.completeExceptionally(new ServiceUnitNotReadyException(msg1));
        }
        return;
    }, pulsar.getOrderedExecutor());
}

Example 54 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class LoadBalancerTestingUtils method makeBundles.

public static NamespaceBundle[] makeBundles(final NamespaceBundleFactory nsFactory, final String property, final String cluster, final String namespace, final int numBundles) {
    final NamespaceBundle[] result = new NamespaceBundle[numBundles];
    final NamespaceName namespaceName = NamespaceName.get(property, cluster, namespace);
    for (int i = 0; i < numBundles - 1; ++i) {
        final long lower = NamespaceBundles.FULL_UPPER_BOUND * i / numBundles;
        final long upper = NamespaceBundles.FULL_UPPER_BOUND * (i + 1) / numBundles;
        result[i] = nsFactory.getBundle(namespaceName, Range.range(lower, BoundType.CLOSED, upper, BoundType.OPEN));
    }
    result[numBundles - 1] = nsFactory.getBundle(namespaceName, Range.range(NamespaceBundles.FULL_UPPER_BOUND * (numBundles - 1) / numBundles, BoundType.CLOSED, NamespaceBundles.FULL_UPPER_BOUND, BoundType.CLOSED));
    return result;
}

Example 55 with NamespaceBundle

use of org.apache.pulsar.common.naming.NamespaceBundle in project incubator-pulsar by apache.

the class AdminApiTest method testNamespaceBundleUnload.

@Test(dataProvider = "numBundles")
public void testNamespaceBundleUnload(Integer numBundles) throws Exception {
    admin.namespaces().createNamespace("prop-xyz/use/ns1-bundles", numBundles);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1-bundles"), Lists.newArrayList());
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1-bundles/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1-bundles"), Lists.newArrayList("persistent://prop-xyz/use/ns1-bundles/ds2"));
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns1-bundles/ds2").subscriptionName("my-sub").subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1-bundles/ds2"), Lists.newArrayList("my-sub"));
    // Create producer
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://prop-xyz/use/ns1-bundles/ds2").create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(TopicName.get("persistent://prop-xyz/use/ns1-bundles/ds2"));
    consumer.close();
    producer.close();
    admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1-bundles", bundle.getBundleRange());
    // check that no one owns the namespace bundle
    assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle));
    assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle));
    LOG.info("--- RELOAD ---");
    // Force reload of namespace and wait for topic to be ready
    for (int i = 0; i < 30; i++) {
        try {
            admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1-bundles/ds2");
            break;
        } catch (PulsarAdminException e) {
            LOG.warn("Failed to get topic stats.. {}", e.getMessage());
            Thread.sleep(1000);
        }
    }
    admin.persistentTopics().deleteSubscription("persistent://prop-xyz/use/ns1-bundles/ds2", "my-sub");
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns1-bundles/ds2");
}