Examples with ClusterData org.apache.pulsar.common.policies.data.ClusterData used on opensource projects

Example 26 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ProxyAuthenticatedProducerConsumerTest method testTlsSyncProducerAndConsumer.

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String proxyServiceUrl = "pulsar://localhost:" + proxyConfig.getServicePortTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);
    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1").subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 27 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ClustersBase method setPeerClusterNames.

@POST
@Path("/{cluster}/peers")
@ApiOperation(value = "Update peer-cluster-list for a cluster.", notes = "This operation requires Pulsar super-user privileges.")
@ApiResponses(value = { @ApiResponse(code = 204, message = "Cluster has been updated"), @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 412, message = "Peer cluster doesn't exist"), @ApiResponse(code = 404, message = "Cluster doesn't exist") })
public void setPeerClusterNames(@PathParam("cluster") String cluster, LinkedHashSet<String> peerClusterNames) {
    validateSuperUserAccess();
    validatePoliciesReadOnlyAccess();
    // validate if peer-cluster exist
    if (peerClusterNames != null && !peerClusterNames.isEmpty()) {
        for (String peerCluster : peerClusterNames) {
            try {
                if (cluster.equalsIgnoreCase(peerCluster)) {
                    throw new RestException(Status.PRECONDITION_FAILED, cluster + " itself can't be part of peer-list");
                }
                clustersCache().get(path("clusters", peerCluster)).orElseThrow(() -> new RestException(Status.PRECONDITION_FAILED, "Peer cluster " + peerCluster + " does not exist"));
            } catch (RestException e) {
                log.warn("[{}] Peer cluster doesn't exist from {}, {}", clientAppId(), peerClusterNames, e.getMessage());
                throw e;
            } catch (Exception e) {
                log.warn("[{}] Failed to validate peer-cluster list {}, {}", clientAppId(), peerClusterNames, e.getMessage());
                throw new RestException(e);
            }
        }
    }
    try {
        String clusterPath = path("clusters", cluster);
        Stat nodeStat = new Stat();
        byte[] content = globalZk().getData(clusterPath, null, nodeStat);
        ClusterData currentClusterData = jsonMapper().readValue(content, ClusterData.class);
        currentClusterData.setPeerClusterNames(peerClusterNames);
        // Write back the new updated ClusterData into zookeeper
        globalZk().setData(clusterPath, jsonMapper().writeValueAsBytes(currentClusterData), nodeStat.getVersion());
        globalZkCache().invalidate(clusterPath);
        log.info("[{}] Successfully added peer-cluster {} for {}", clientAppId(), peerClusterNames, cluster);
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update cluster {}: Does not exist", clientAppId(), cluster);
        throw new RestException(Status.NOT_FOUND, "Cluster does not exist");
    } catch (Exception e) {
        log.error("[{}] Failed to update cluster {}", clientAppId(), cluster, e);
        throw new RestException(e);
    }
}

Example 28 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class NamespacesBase method validatePeerClusterConflict.

/**
 * It validates that peer-clusters can't coexist in replication-clusters
 *
 * @param clusterName:
 *            given cluster whose peer-clusters can't be present into replication-cluster list
 * @param clusters:
 *            replication-cluster list
 */
private void validatePeerClusterConflict(String clusterName, Set<String> replicationClusters) {
    try {
        ClusterData clusterData = clustersCache().get(path("clusters", clusterName)).orElseThrow(() -> new RestException(Status.PRECONDITION_FAILED, "Invalid replication cluster " + clusterName));
        Set<String> peerClusters = clusterData.getPeerClusterNames();
        if (peerClusters != null && !peerClusters.isEmpty()) {
            SetView<String> conflictPeerClusters = Sets.intersection(peerClusters, replicationClusters);
            if (!conflictPeerClusters.isEmpty()) {
                log.warn("[{}] {}'s peer cluster can't be part of replication clusters {}", clientAppId(), clusterName, conflictPeerClusters);
                throw new RestException(Status.CONFLICT, String.format("%s's peer-clusters %s can't be part of replication-clusters %s", clusterName, conflictPeerClusters, replicationClusters));
            }
        }
    } catch (RestException re) {
        throw re;
    } catch (Exception e) {
        log.warn("[{}] Failed to get cluster-data for {}", clientAppId(), clusterName, e);
    }
}

Example 29 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class NamespacesBase method internalDeleteNamespaceBundle.

@SuppressWarnings("deprecation")
protected void internalDeleteNamespaceBundle(String bundleRange, boolean authoritative) {
    validateAdminAccessOnProperty(namespaceName.getProperty());
    validatePoliciesReadOnlyAccess();
    // ensure that non-global namespace is directed to the correct cluster
    if (!namespaceName.isGlobal()) {
        validateClusterOwnership(namespaceName.getCluster());
    }
    Policies policies = getNamespacePolicies(namespaceName);
    // ensure the local cluster is the only cluster for the global namespace configuration
    try {
        if (namespaceName.isGlobal()) {
            if (policies.replication_clusters.size() > 1) {
                // There are still more than one clusters configured for the global namespace
                throw new RestException(Status.PRECONDITION_FAILED, "Cannot delete the global namespace " + namespaceName + ". There are still more than one replication clusters configured.");
            }
            if (policies.replication_clusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) {
                // the only replication cluster is other cluster, redirect
                String replCluster = policies.replication_clusters.get(0);
                ClusterData replClusterData = clustersCache().get(AdminResource.path("clusters", replCluster)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Cluser " + replCluster + " does not exist"));
                URL replClusterUrl;
                if (!config().isTlsEnabled() || !isRequestHttps()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrl());
                } else if (StringUtils.isNotBlank(replClusterData.getServiceUrlTls())) {
                    replClusterUrl = new URL(replClusterData.getServiceUrlTls());
                } else {
                    throw new RestException(Status.PRECONDITION_FAILED, "The replication cluster does not provide TLS encrypted service");
                }
                URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(replClusterUrl.getHost()).port(replClusterUrl.getPort()).replaceQueryParam("authoritative", false).build();
                log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, replCluster);
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    NamespaceBundle bundle = validateNamespaceBundleOwnership(namespaceName, policies.bundles, bundleRange, authoritative, true);
    try {
        List<String> topics = pulsar().getNamespaceService().getListOfTopics(namespaceName);
        for (String topic : topics) {
            NamespaceBundle topicBundle = (NamespaceBundle) pulsar().getNamespaceService().getBundle(TopicName.get(topic));
            if (bundle.equals(topicBundle)) {
                throw new RestException(Status.CONFLICT, "Cannot delete non empty bundle");
            }
        }
        // remove from owned namespace map and ephemeral node from ZK
        pulsar().getNamespaceService().removeOwnedServiceUnit(bundle);
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        log.error("[{}] Failed to remove namespace bundle {}/{}", clientAppId(), namespaceName.toString(), bundleRange, e);
        throw new RestException(e);
    }
}

Example 30 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class NamespacesBase method internalDeleteNamespace.

@SuppressWarnings("deprecation")
protected void internalDeleteNamespace(boolean authoritative) {
    validateAdminAccessOnProperty(namespaceName.getProperty());
    validatePoliciesReadOnlyAccess();
    // ensure that non-global namespace is directed to the correct cluster
    if (!namespaceName.isGlobal()) {
        validateClusterOwnership(namespaceName.getCluster());
    }
    Entry<Policies, Stat> policiesNode = null;
    Policies policies = null;
    // ensure the local cluster is the only cluster for the global namespace configuration
    try {
        policiesNode = policiesCache().getWithStat(path(POLICIES, namespaceName.toString())).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Namespace " + namespaceName + " does not exist."));
        policies = policiesNode.getKey();
        if (namespaceName.isGlobal()) {
            if (policies.replication_clusters.size() > 1) {
                // There are still more than one clusters configured for the global namespace
                throw new RestException(Status.PRECONDITION_FAILED, "Cannot delete the global namespace " + namespaceName + ". There are still more than one replication clusters configured.");
            }
            if (policies.replication_clusters.size() == 1 && !policies.replication_clusters.contains(config().getClusterName())) {
                // the only replication cluster is other cluster, redirect
                String replCluster = policies.replication_clusters.get(0);
                ClusterData replClusterData = clustersCache().get(AdminResource.path("clusters", replCluster)).orElseThrow(() -> new RestException(Status.NOT_FOUND, "Cluster " + replCluster + " does not exist"));
                URL replClusterUrl;
                if (!config().isTlsEnabled() || !isRequestHttps()) {
                    replClusterUrl = new URL(replClusterData.getServiceUrl());
                } else if (StringUtils.isNotBlank(replClusterData.getServiceUrlTls())) {
                    replClusterUrl = new URL(replClusterData.getServiceUrlTls());
                } else {
                    throw new RestException(Status.PRECONDITION_FAILED, "The replication cluster does not provide TLS encrypted service");
                }
                URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(replClusterUrl.getHost()).port(replClusterUrl.getPort()).replaceQueryParam("authoritative", false).build();
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Redirecting the rest call to {}: cluster={}", clientAppId(), redirect, replCluster);
                }
                throw new WebApplicationException(Response.temporaryRedirect(redirect).build());
            }
        }
    } catch (WebApplicationException wae) {
        throw wae;
    } catch (Exception e) {
        throw new RestException(e);
    }
    boolean isEmpty;
    try {
        isEmpty = pulsar().getNamespaceService().getListOfTopics(namespaceName).isEmpty();
    } catch (Exception e) {
        throw new RestException(e);
    }
    if (!isEmpty) {
        log.debug("Found topics on namespace {}", namespaceName);
        throw new RestException(Status.CONFLICT, "Cannot delete non empty namespace");
    }
    // set the policies to deleted so that somebody else cannot acquire this namespace
    try {
        policies.deleted = true;
        globalZk().setData(path(POLICIES, namespaceName.toString()), jsonMapper().writeValueAsBytes(policies), policiesNode.getValue().getVersion());
        policiesCache().invalidate(path(POLICIES, namespaceName.toString()));
    } catch (Exception e) {
        log.error("[{}] Failed to delete namespace on global ZK {}", clientAppId(), namespaceName, e);
        throw new RestException(e);
    }
    // remove from owned namespace map and ephemeral node from ZK
    try {
        NamespaceBundles bundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(namespaceName);
        for (NamespaceBundle bundle : bundles.getBundles()) {
            // check if the bundle is owned by any broker, if not then we do not need to delete the bundle
            if (pulsar().getNamespaceService().getOwner(bundle).isPresent()) {
                pulsar().getAdminClient().namespaces().deleteNamespaceBundle(namespaceName.toString(), bundle.getBundleRange());
            }
        }
        // we have successfully removed all the ownership for the namespace, the policies znode can be deleted now
        final String globalZkPolicyPath = path(POLICIES, namespaceName.toString());
        final String lcaolZkPolicyPath = joinPath(LOCAL_POLICIES_ROOT, namespaceName.toString());
        globalZk().delete(globalZkPolicyPath, -1);
        localZk().delete(lcaolZkPolicyPath, -1);
        policiesCache().invalidate(globalZkPolicyPath);
        localCacheService().policiesCache().invalidate(lcaolZkPolicyPath);
    } catch (PulsarAdminException cae) {
        throw new RestException(cae);
    } catch (Exception e) {
        log.error("[{}] Failed to remove owned namespace {}", clientAppId(), namespaceName, e);
    // avoid throwing exception in case of the second failure
    }
}