Examples with NamespaceBundles org.apache.pulsar.common.naming.NamespaceBundles used on opensource projects

Example 16 with NamespaceBundles

use of org.apache.pulsar.common.naming.NamespaceBundles in project incubator-pulsar by apache.

the class NamespaceBundlesTest method testsplitBundles.

@Test
public void testsplitBundles() throws Exception {
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = factory.getBundles(nsname);
    NamespaceBundle bundle = bundles.findBundle(topicName);
    final int numberSplitBundles = 4;
    // (1) split in 4
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = factory.splitBundles(bundle, numberSplitBundles);
    // existing_no_bundles(1) +
    // additional_new_split_bundle(4) -
    // parent_target_bundle(1)
    int totalExpectedSplitBundles = bundles.getBundles().size() + numberSplitBundles - 1;
    validateSplitBundlesRange(bundles.getFullBundle(), splitBundles.getRight());
    assertEquals(totalExpectedSplitBundles, splitBundles.getLeft().getBundles().size());
    // (2) split in 4: first bundle from above split bundles
    NamespaceBundleFactory utilityFactory = getNamespaceBundleFactory();
    NamespaceBundles bundles2 = splitBundles.getLeft();
    NamespaceBundle testChildBundle = bundles2.getBundles().get(0);
    Pair<NamespaceBundles, List<NamespaceBundle>> splitChildBundles = splitBundlesUtilFactory(utilityFactory, nsname, bundles2, testChildBundle, numberSplitBundles);
    // existing_no_bundles(4) +
    // additional_new_split_bundle(4) -
    // parent_target_bundle(1)
    totalExpectedSplitBundles = bundles2.getBundles().size() + numberSplitBundles - 1;
    validateSplitBundlesRange(testChildBundle, splitChildBundles.getRight());
    assertEquals(totalExpectedSplitBundles, splitChildBundles.getLeft().getBundles().size());
    // (3) split in 3: second bundle from above split bundles
    NamespaceBundle testChildBundl2 = bundles2.getBundles().get(1);
    Pair<NamespaceBundles, List<NamespaceBundle>> splitChildBundles2 = splitBundlesUtilFactory(utilityFactory, nsname, bundles2, testChildBundl2, 3);
    // existing_no_bundles(4) +
    // additional_new_split_bundle(3) -
    // parent_target_bundle(1)
    totalExpectedSplitBundles = bundles2.getBundles().size() + 3 - 1;
    validateSplitBundlesRange(testChildBundl2, splitChildBundles2.getRight());
    assertEquals(totalExpectedSplitBundles, splitChildBundles2.getLeft().getBundles().size());
}

Example 17 with NamespaceBundles

use of org.apache.pulsar.common.naming.NamespaceBundles in project incubator-pulsar by apache.

the class NamespaceBundlesTest method assertBundles.

private void assertBundles(NamespaceBundleFactory utilityFactory, NamespaceName nsname, NamespaceBundle bundle, Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles, int numBundles) throws Exception {
    NamespaceBundle bundle1 = splitBundles.getRight().get(0);
    NamespaceBundle bundle2 = splitBundles.getRight().get(1);
    NamespaceBundles nspaceBundles = splitBundles.getLeft();
    Pair<NamespaceBundles, List<NamespaceBundle>> bundle1Split = splitBundlesUtilFactory(utilityFactory, nsname, nspaceBundles, bundle1, numBundles);
    assertBundleDivideInTwo(bundle1, bundle1Split.getRight(), numBundles);
    Pair<NamespaceBundles, List<NamespaceBundle>> bundle2Split = splitBundlesUtilFactory(utilityFactory, nsname, nspaceBundles, bundle2, numBundles);
    assertBundleDivideInTwo(bundle2, bundle2Split.getRight(), numBundles);
}

Example 18 with NamespaceBundles

use of org.apache.pulsar.common.naming.NamespaceBundles in project incubator-pulsar by apache.

the class NamespaceService method splitAndOwnBundleOnceAndRetry.

void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, boolean unload, AtomicInteger counter, CompletableFuture<Void> unloadFuture) {
    CompletableFuture<NamespaceBundles> updateFuture = new CompletableFuture<>();
    final Pair<NamespaceBundles, List<NamespaceBundle>> splittedBundles = bundleFactory.splitBundles(bundle, 2);
    // Split and updateNamespaceBundles. Update may fail because of concurrent write to Zookeeper.
    if (splittedBundles != null) {
        checkNotNull(splittedBundles.getLeft());
        checkNotNull(splittedBundles.getRight());
        checkArgument(splittedBundles.getRight().size() == 2, "bundle has to be split in two bundles");
        NamespaceName nsname = bundle.getNamespaceObject();
        if (LOG.isDebugEnabled()) {
            LOG.debug("[{}] splitAndOwnBundleOnce: {}, counter: {},  2 bundles: {}, {}", nsname.toString(), bundle.getBundleRange(), counter.get(), splittedBundles != null ? splittedBundles.getRight().get(0).getBundleRange() : "null splittedBundles", splittedBundles != null ? splittedBundles.getRight().get(1).getBundleRange() : "null splittedBundles");
        }
        try {
            // take ownership of newly split bundles
            for (NamespaceBundle sBundle : splittedBundles.getRight()) {
                checkNotNull(ownershipCache.tryAcquiringOwnership(sBundle));
            }
            updateNamespaceBundles(nsname, splittedBundles.getLeft(), (rc, path, zkCtx, stat) -> {
                if (rc == Code.OK.intValue()) {
                    // invalidate cache as zookeeper has new split
                    // namespace bundle
                    bundleFactory.invalidateBundleCache(bundle.getNamespaceObject());
                    updateFuture.complete(splittedBundles.getLeft());
                } else if (rc == Code.BADVERSION.intValue()) {
                    KeeperException keeperException = KeeperException.create(KeeperException.Code.get(rc));
                    String msg = format("failed to update namespace policies [%s], NamespaceBundle: %s " + "due to %s, counter: %d", nsname.toString(), bundle.getBundleRange(), keeperException.getMessage(), counter.get());
                    LOG.warn(msg);
                    updateFuture.completeExceptionally(new ServerMetadataException(keeperException));
                } else {
                    String msg = format("failed to update namespace policies [%s], NamespaceBundle: %s due to %s", nsname.toString(), bundle.getBundleRange(), KeeperException.create(KeeperException.Code.get(rc)).getMessage());
                    LOG.warn(msg);
                    updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
                }
            });
        } catch (Exception e) {
            String msg = format("failed to acquire ownership of split bundle for namespace [%s], %s", nsname.toString(), e.getMessage());
            LOG.warn(msg, e);
            updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        }
    } else {
        String msg = format("bundle %s not found under namespace", bundle.toString());
        LOG.warn(msg);
        updateFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
    }
    // If success updateNamespaceBundles, then do invalidateBundleCache and unload.
    // Else retry splitAndOwnBundleOnceAndRetry.
    updateFuture.whenCompleteAsync((r, t) -> {
        if (t != null) {
            // retry several times on BadVersion
            if ((t instanceof ServerMetadataException) && (counter.decrementAndGet() >= 0)) {
                pulsar.getOrderedExecutor().submit(() -> splitAndOwnBundleOnceAndRetry(bundle, unload, counter, unloadFuture));
            } else {
                // Retry enough, or meet other exception
                String msg2 = format(" %s not success update nsBundles, counter %d, reason %s", bundle.toString(), counter.get(), t.getMessage());
                LOG.warn(msg2);
                unloadFuture.completeExceptionally(new ServiceUnitNotReadyException(msg2));
            }
            return;
        }
        // success updateNamespaceBundles
        try {
            // disable old bundle in memory
            getOwnershipCache().updateBundleState(bundle, false);
            // update bundled_topic cache for load-report-generation
            pulsar.getBrokerService().refreshTopicToStatsMaps(bundle);
            loadManager.get().setLoadReportForceUpdateFlag();
            if (unload) {
                // unload new split bundles
                r.getBundles().forEach(splitBundle -> {
                    try {
                        unloadNamespaceBundle(splitBundle);
                    } catch (Exception e) {
                        LOG.warn("Failed to unload split bundle {}", splitBundle, e);
                        throw new RuntimeException("Failed to unload split bundle " + splitBundle, e);
                    }
                });
            }
            unloadFuture.complete(null);
        } catch (Exception e) {
            String msg1 = format("failed to disable bundle %s under namespace [%s] with error %s", bundle.getNamespaceObject().toString(), bundle.toString(), e.getMessage());
            LOG.warn(msg1, e);
            unloadFuture.completeExceptionally(new ServiceUnitNotReadyException(msg1));
        }
        return;
    }, pulsar.getOrderedExecutor());
}

Example 19 with NamespaceBundles

use of org.apache.pulsar.common.naming.NamespaceBundles in project incubator-pulsar by apache.

the class NamespaceServiceTest method testSplitMapWithRefreshedStatMap.

@Test
public void testSplitMapWithRefreshedStatMap() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
    TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(topicName);
    PersistentTopic topic = new PersistentTopic(topicName.toString(), ledger, pulsar.getBrokerService());
    Method method = pulsar.getBrokerService().getClass().getDeclaredMethod("addTopicToStatsMaps", TopicName.class, Topic.class);
    method.setAccessible(true);
    method.invoke(pulsar.getBrokerService(), topicName, topic);
    String nspace = originalBundle.getNamespaceObject().toString();
    List<Topic> list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertNotNull(list);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle, false);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    // old bundle should be removed from status-map
    list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertTrue(list.isEmpty());
    // status-map should be updated with new split bundles
    NamespaceBundle splitBundle = pulsar.getNamespaceService().getBundle(topicName);
    assertTrue(!CollectionUtils.isEmpty(this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, splitBundle.toString())));
}

Example 20 with NamespaceBundles

use of org.apache.pulsar.common.naming.NamespaceBundles in project incubator-pulsar by apache.

the class NamespacesTest method testUnloadNamespaceWithBundles.

@Test
public void testUnloadNamespaceWithBundles() throws Exception {
    URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
    String bundledNsLocal = "test-bundled-namespace-1";
    BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"));
    createBundledTestNamespaces(this.testProperty, this.testLocalCluster, bundledNsLocal, bundleData);
    final NamespaceName testNs = NamespaceName.get(this.testProperty, this.testLocalCluster, bundledNsLocal);
    doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {

        @Override
        public void describeTo(Description description) {
        // TODO Auto-generated method stub
        }

        @Override
        public boolean matches(Object item) {
            if (item instanceof NamespaceBundle) {
                NamespaceBundle bundle = (NamespaceBundle) item;
                return bundle.getNamespaceObject().equals(testNs);
            }
            return false;
        }

        @Override
        public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
        // TODO Auto-generated method stub
        }
    }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
    doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {

        @Override
        public void describeTo(Description description) {
        // TODO Auto-generated method stub
        }

        @Override
        public boolean matches(Object item) {
            if (item instanceof NamespaceBundle) {
                NamespaceBundle bundle = (NamespaceBundle) item;
                return bundle.getNamespaceObject().equals(testNs);
            }
            return false;
        }

        @Override
        public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
        // TODO Auto-generated method stub
        }
    }));
    NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData);
    NamespaceBundle testBundle = nsBundles.getBundles().get(0);
    // make one bundle owned
    doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testBundle, false, true, false);
    doReturn(true).when(nsSvc).isServiceUnitOwned(testBundle);
    doNothing().when(nsSvc).unloadNamespaceBundle(testBundle);
    namespaces.unloadNamespaceBundle(testProperty, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000", false);
    verify(nsSvc, times(1)).unloadNamespaceBundle(testBundle);
    try {
        namespaces.unloadNamespaceBundle(testProperty, testLocalCluster, bundledNsLocal, "0x00000000_0x88000000", false);
        fail("should have failed");
    } catch (RestException re) {
    // ok
    }
}