Examples with NamespaceBundles com.yahoo.pulsar.common.naming.NamespaceBundles used on opensource projects

Example 6 with NamespaceBundles

use of com.yahoo.pulsar.common.naming.NamespaceBundles in project pulsar by yahoo.

the class Namespaces method clearNamespaceBacklogForSubscription.

@POST
@Path("/{property}/{cluster}/{namespace}/clearBacklog/{subscription}")
@ApiOperation(value = "Clear backlog for a given subscription on all destinations on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") })
public void clearNamespaceBacklogForSubscription(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("subscription") String subscription, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateAdminAccessOnProperty(property);
    NamespaceName nsName = new NamespaceName(property, cluster, namespace);
    try {
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(nsName);
        Exception exception = null;
        for (NamespaceBundle nsBundle : bundles.getBundles()) {
            try {
                // clear
                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
                    // TODO: make this admin call asynchronous
                    pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklogForSubscription(nsName.toString(), nsBundle.getBundleRange(), subscription);
                }
            } catch (Exception e) {
                if (exception == null) {
                    exception = e;
                }
            }
        }
        if (exception != null) {
            if (exception instanceof PulsarAdminException) {
                throw new RestException((PulsarAdminException) exception);
            } else {
                throw new RestException(exception.getCause());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    log.info("[{}] Successfully cleared backlog for subscription {} on all the bundles for namespace {}", clientAppId(), subscription, nsName.toString());
}

Example 7 with NamespaceBundles

use of com.yahoo.pulsar.common.naming.NamespaceBundles in project pulsar by yahoo.

the class Namespaces method deleteNamespace.

@DELETE
@Path("/{property}/{cluster}/{namespace}")
@ApiOperation(value = "Delete a namespace and all the destinations under it.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property or cluster or namespace doesn't exist"), @ApiResponse(code = 409, message = "Namespace is not empty") })
public void deleteNamespace(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    NamespaceName nsName = new NamespaceName(property, cluster, namespace);
    validateAdminAccessOnProperty(property);
    validatePoliciesReadOnlyAccess();
    // ensure that non-global namespace is directed to the correct cluster
    validateClusterOwnership(cluster);
    Entry<Policies, Stat> policiesNode = null;
    Policies policies = null;
    // ensure the local cluster is the only cluster for the global namespace configuration
    try {
        policiesNode = policiesCache().getWithStat(path("policies", property, cluster, namespace)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Namespace " + nsName + " does not exist."));
        policies = policiesNode.getKey();
        if (cluster.equals(Namespaces.GLOBAL_CLUSTER)) {
            if (policies.replication_clusters.size() > 1) {
                // There are still more than one clusters configured for the global namespace
                throw new RestException(Status.PRECONDITION_FAILED, "Cannot delete the global namespace " + nsName + ". There are still more than one replication clusters configured.");
            }
            if (policies.replication_clusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) {
                // the only replication cluster is other cluster, redirect
                String replCluster = policies.replication_clusters.get(0);
                ClusterData replClusterData = clustersCache().get(AdminResource.path("clusters", replCluster)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Cluser " + replCluster + " does not exist"));
                URL replClusterUrl;
                if (!config().isTlsEnabled()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrl());
                } else if (!replClusterData.getServiceUrlTls().isEmpty()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrlTls());
                } else {
                    throw new RestException(Status.PRECONDITION_FAILED, "The replication cluster does not provide TLS encrypted service");
                }
                URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(replClusterUrl.getHost()).port(replClusterUrl.getPort()).replaceQueryParam("authoritative", false).build();
                log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, cluster);
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    List<String> destinations = getDestinations(property, cluster, namespace);
    if (!destinations.isEmpty()) {
        log.info("Found destinations: {}", destinations);
        throw new RestException(Status.CONFLICT, "Cannot delete non empty namespace");
    }
    // set the policies to deleted so that somebody else cannot acquire this namespace
    try {
        policies.deleted = true;
        globalZk().setData(path("policies", property, cluster, namespace), jsonMapper().writeValueAsBytes(policies), policiesNode.getValue().getVersion());
        policiesCache().invalidate(path("policies", property, cluster, namespace));
    } catch (Exception e) {
        log.error("[{}] Failed to delete namespace on global ZK {}/{}/{}", clientAppId(), property, cluster, namespace, e);
        throw new RestException(e);
    }
    // remove from owned namespace map and ephemeral node from ZK
    try {
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(nsName);
        for (NamespaceBundle bundle : bundles.getBundles()) {
            // check if the bundle is owned by any broker, if not then we do not need to delete the bundle
            if (pulsar().getNamespaceService().getOwner(bundle).isPresent()) {
                pulsar().getAdminClient().namespaces().deleteNamespaceBundle(nsName.toString(), bundle.getBundleRange());
            }
        }
        // we have successfully removed all the ownership for the namespace, the policies znode can be deleted now
        globalZk().delete(path("policies", property, cluster, namespace), -1);
        policiesCache().invalidate(path("policies", property, cluster, namespace));
    } catch (PulsarAdminException cae) {
        throw new RestException(cae);
    } catch (Exception e) {
        log.error(String.format("[%s] Failed to remove owned namespace %s/%s/%s", clientAppId(), property, cluster, namespace), e);
    // avoid throwing exception in case of the second failure
    }
}

Example 8 with NamespaceBundles

use of com.yahoo.pulsar.common.naming.NamespaceBundles in project pulsar by yahoo.

the class Namespaces method clearNamespaceBacklog.

@POST
@Path("/{property}/{cluster}/{namespace}/clearBacklog")
@ApiOperation(value = "Clear backlog for all destinations on a namespace.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") })
public void clearNamespaceBacklog(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    validateAdminAccessOnProperty(property);
    NamespaceName nsName = new NamespaceName(property, cluster, namespace);
    try {
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(nsName);
        Exception exception = null;
        for (NamespaceBundle nsBundle : bundles.getBundles()) {
            try {
                // clear
                if (pulsar().getNamespaceService().getOwner(nsBundle).isPresent()) {
                    // TODO: make this admin call asynchronous
                    pulsar().getAdminClient().namespaces().clearNamespaceBundleBacklog(nsName.toString(), nsBundle.getBundleRange());
                }
            } catch (Exception e) {
                if (exception == null) {
                    exception = e;
                }
            }
        }
        if (exception != null) {
            if (exception instanceof PulsarAdminException) {
                throw new RestException((PulsarAdminException) exception);
            } else {
                throw new RestException(exception.getCause());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    log.info("[{}] Successfully cleared backlog on all the bundles for namespace {}", clientAppId(), nsName.toString());
}

Example 9 with NamespaceBundles

use of com.yahoo.pulsar.common.naming.NamespaceBundles in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitMapWithRefreshedStatMap.

@Test
public void testSplitMapWithRefreshedStatMap() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    PersistentTopic topic = new PersistentTopic(dn.toString(), ledger, pulsar.getBrokerService());
    Method method = pulsar.getBrokerService().getClass().getDeclaredMethod("addTopicToStatsMaps", DestinationName.class, PersistentTopic.class);
    method.setAccessible(true);
    method.invoke(pulsar.getBrokerService(), dn, topic);
    String nspace = originalBundle.getNamespaceObject().toString();
    List<PersistentTopic> list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertNotNull(list);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    try {
        // old bundle should be removed from status-map
        list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
        fail();
    } catch (NullPointerException ne) {
    // OK
    }
    // status-map should be updated with new split bundles
    NamespaceBundle splitBundle = pulsar.getNamespaceService().getBundle(dn);
    assertTrue(!CollectionUtils.isEmpty(this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, splitBundle.toString())));
}

Example 10 with NamespaceBundles

use of com.yahoo.pulsar.common.naming.NamespaceBundles in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitAndOwnBundles.

@Test
public void testSplitAndOwnBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}