Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 61 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitMapWithRefreshedStatMap.

@Test
public void testSplitMapWithRefreshedStatMap() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    ManagedLedger ledger = mock(ManagedLedger.class);
    when(ledger.getCursors()).thenReturn(Lists.newArrayList());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    PersistentTopic topic = new PersistentTopic(dn.toString(), ledger, pulsar.getBrokerService());
    Method method = pulsar.getBrokerService().getClass().getDeclaredMethod("addTopicToStatsMaps", DestinationName.class, PersistentTopic.class);
    method.setAccessible(true);
    method.invoke(pulsar.getBrokerService(), dn, topic);
    String nspace = originalBundle.getNamespaceObject().toString();
    List<PersistentTopic> list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
    assertNotNull(list);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    try {
        // old bundle should be removed from status-map
        list = this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, originalBundle.toString());
        fail();
    } catch (NullPointerException ne) {
    // OK
    }
    // status-map should be updated with new split bundles
    NamespaceBundle splitBundle = pulsar.getNamespaceService().getBundle(dn);
    assertTrue(!CollectionUtils.isEmpty(this.pulsar.getBrokerService().getAllTopicsFromNamespaceBundle(nspace, splitBundle.toString())));
}

Example 62 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitAndOwnBundles.

@Test
public void testSplitAndOwnBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}

Example 63 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PersistentTopic method checkReplication.

@Override
public CompletableFuture<Void> checkReplication() {
    DestinationName name = DestinationName.get(topic);
    if (!name.isGlobal()) {
        return CompletableFuture.completedFuture(null);
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Checking replication status", name);
    }
    Policies policies = null;
    try {
        policies = brokerService.pulsar().getConfigurationCache().policiesCache().get(AdminResource.path("policies", name.getNamespace())).orElseThrow(() -> new KeeperException.NoNodeException());
    } catch (Exception e) {
        CompletableFuture<Void> future = new CompletableFuture<>();
        future.completeExceptionally(new ServerMetadataException(e));
        return future;
    }
    final int newMessageTTLinSeconds = policies.message_ttl_in_seconds;
    Set<String> configuredClusters;
    if (policies.replication_clusters != null) {
        configuredClusters = Sets.newTreeSet(policies.replication_clusters);
    } else {
        configuredClusters = Collections.emptySet();
    }
    String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
    List<CompletableFuture<Void>> futures = Lists.newArrayList();
    // Check for missing replicators
    for (String cluster : configuredClusters) {
        if (cluster.equals(localCluster)) {
            continue;
        }
        if (!replicators.containsKey(cluster)) {
            futures.add(startReplicator(cluster));
        }
    }
    // Check for replicators to be stopped
    replicators.forEach((cluster, replicator) -> {
        // Update message TTL
        replicator.updateMessageTTL(newMessageTTLinSeconds);
        if (!cluster.equals(localCluster)) {
            if (!configuredClusters.contains(cluster)) {
                futures.add(removeReplicator(cluster));
            }
        }
    });
    return FutureUtil.waitForAll(futures);
}

Example 64 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class DestinationLookup method lookupDestinationAsync.

@GET
@Path("persistent/{property}/{cluster}/{namespace}/{dest}")
@Produces(MediaType.APPLICATION_JSON)
public void lookupDestinationAsync(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("dest") @Encoded String dest, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative, @Suspended AsyncResponse asyncResponse) {
    dest = Codec.decode(dest);
    DestinationName topic = DestinationName.get("persistent", property, cluster, namespace, dest);
    if (!pulsar().getBrokerService().getLookupRequestSemaphore().tryAcquire()) {
        log.warn("No broker was found available for topic {}", topic);
        asyncResponse.resume(new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE));
        return;
    }
    try {
        validateClusterOwnership(topic.getCluster());
        checkConnect(topic);
        validateReplicationSettingsOnNamespace(pulsar(), topic.getNamespaceObject());
    } catch (WebApplicationException we) {
        // Validation checks failed
        log.error("Validation check failed: {}", we.getMessage());
        completeLookupResponseExceptionally(asyncResponse, we);
        return;
    } catch (Throwable t) {
        // Validation checks failed with unknown error
        log.error("Validation check failed: {}", t.getMessage(), t);
        completeLookupResponseExceptionally(asyncResponse, new RestException(t));
        return;
    }
    CompletableFuture<LookupResult> lookupFuture = pulsar().getNamespaceService().getBrokerServiceUrlAsync(topic, authoritative);
    lookupFuture.thenAccept(result -> {
        if (result == null) {
            log.warn("No broker was found available for topic {}", topic);
            completeLookupResponseExceptionally(asyncResponse, new WebApplicationException(Response.Status.SERVICE_UNAVAILABLE));
            return;
        }
        if (result.isRedirect()) {
            boolean newAuthoritative = this.isLeaderBroker();
            URI redirect;
            try {
                String redirectUrl = isRequestHttps() ? result.getLookupData().getHttpUrlTls() : result.getLookupData().getHttpUrl();
                redirect = new URI(String.format("%s%s%s?authoritative=%s", redirectUrl, "/lookup/v2/destination/", topic.getLookupName(), newAuthoritative));
            } catch (URISyntaxException e) {
                log.error("Error in preparing redirect url for {}: {}", topic, e.getMessage(), e);
                completeLookupResponseExceptionally(asyncResponse, e);
                return;
            }
            if (log.isDebugEnabled()) {
                log.debug("Redirect lookup for topic {} to {}", topic, redirect);
            }
            completeLookupResponseExceptionally(asyncResponse, new WebApplicationException(Response.temporaryRedirect(redirect).build()));
        } else {
            if (log.isDebugEnabled()) {
                log.debug("Lookup succeeded for topic {} -- broker: {}", topic, result.getLookupData());
            }
            completeLookupResponseSuccessfully(asyncResponse, result.getLookupData());
        }
    }).exceptionally(exception -> {
        log.warn("Failed to lookup broker for topic {}: {}", topic, exception.getMessage(), exception);
        completeLookupResponseExceptionally(asyncResponse, exception);
        return null;
    });
}

Example 65 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class DestinationLookup method lookupDestinationAsync.

/**
     * 
     * Lookup broker-service address for a given namespace-bundle which contains given topic.
     * 
     * a. Returns broker-address if namespace-bundle is already owned by any broker
     * b. If current-broker receives lookup-request and if it's not a leader
     * then current broker redirects request to leader by returning leader-service address. 
     * c. If current-broker is leader then it finds out least-loaded broker to own namespace bundle and 
     * redirects request by returning least-loaded broker.
     * d. If current-broker receives request to own the namespace-bundle then it owns a bundle and returns 
     * success(connect) response to client.
     * 
     * @param pulsarService
     * @param fqdn
     * @param authoritative
     * @param clientAppId
     * @param requestId
     * @return
     */
public static CompletableFuture<ByteBuf> lookupDestinationAsync(PulsarService pulsarService, DestinationName fqdn, boolean authoritative, String clientAppId, long requestId) {
    final CompletableFuture<ByteBuf> validationFuture = new CompletableFuture<>();
    final CompletableFuture<ByteBuf> lookupfuture = new CompletableFuture<>();
    final String cluster = fqdn.getCluster();
    // (1) validate cluster
    getClusterDataIfDifferentCluster(pulsarService, cluster, clientAppId).thenAccept(differentClusterData -> {
        if (differentClusterData != null) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Redirecting the lookup call to {}/{} cluster={}", clientAppId, differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), cluster);
            }
            validationFuture.complete(newLookupResponse(differentClusterData.getBrokerServiceUrl(), differentClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, requestId));
        } else {
            try {
                checkAuthorization(pulsarService, fqdn, clientAppId);
            } catch (RestException authException) {
                log.warn("Failed to authorized {} on cluster {}", clientAppId, fqdn.toString());
                validationFuture.complete(newLookupResponse(ServerError.AuthorizationError, authException.getMessage(), requestId));
                return;
            } catch (Exception e) {
                log.warn("Unknown error while authorizing {} on cluster {}", clientAppId, fqdn.toString());
                validationFuture.completeExceptionally(e);
                return;
            }
            validateReplicationSettingsOnNamespaceAsync(pulsarService, fqdn.getNamespaceObject()).thenAccept(success -> {
                validationFuture.complete(null);
            }).exceptionally(ex -> {
                validationFuture.complete(newLookupResponse(ServerError.MetadataError, ex.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        validationFuture.completeExceptionally(ex);
        return null;
    });
    // Initiate lookup once validation completes
    validationFuture.thenAccept(validaitonFailureResponse -> {
        if (validaitonFailureResponse != null) {
            lookupfuture.complete(validaitonFailureResponse);
        } else {
            pulsarService.getNamespaceService().getBrokerServiceUrlAsync(fqdn, authoritative).thenAccept(lookupResult -> {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Lookup result {}", fqdn.toString(), lookupResult);
                }
                LookupData lookupData = lookupResult.getLookupData();
                if (lookupResult.isRedirect()) {
                    boolean newAuthoritative = isLeaderBroker(pulsarService);
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), newAuthoritative, LookupType.Redirect, requestId));
                } else {
                    lookupfuture.complete(newLookupResponse(lookupData.getBrokerUrl(), lookupData.getBrokerUrlTls(), true, LookupType.Connect, requestId));
                }
            }).exceptionally(e -> {
                log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), e.getMessage(), e);
                lookupfuture.complete(newLookupResponse(ServerError.ServiceNotReady, e.getMessage(), requestId));
                return null;
            });
        }
    }).exceptionally(ex -> {
        log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, fqdn.toString(), ex.getMessage(), ex);
        lookupfuture.complete(newLookupResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId));
        return null;
    });
    return lookupfuture;
}