Examples with ClusterData com.yahoo.pulsar.common.policies.data.ClusterData used on opensource projects

Example 16 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class AuthenticatedProducerConsumerTest method testInternalServerExceptionOnLookup.

/**
     * verifies that topicLookup/PartitionMetadataLookup gives InternalServerError(500) instead 401(auth_failed) on
     * unknown-exception failure
     * 
     * @throws Exception
     */
@Test
public void testInternalServerExceptionOnLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    String namespace = "my-property/use/my-ns";
    admin.namespaces().createNamespace(namespace);
    String destination = "persistent://" + namespace + "1/topic1";
    // this will cause NPE and it should throw 500
    mockZookKeeper.shutdown();
    pulsar.getConfiguration().setSuperUserRoles(Sets.newHashSet());
    try {
        admin.persistentTopics().getPartitionedTopicMetadata(destination);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
    try {
        admin.lookups().lookupDestination(destination);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
}

Example 17 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class ProducerConsumerBase method producerBaseSetup.

public void producerBaseSetup() throws Exception {
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}

Example 18 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class PulsarClusterMetadataSetup method main.

public static void main(String[] args) throws Exception {
    Arguments arguments = new Arguments();
    JCommander jcommander = new JCommander();
    try {
        jcommander.addObject(arguments);
        jcommander.parse(args);
        if (arguments.help) {
            jcommander.usage();
            return;
        }
    } catch (Exception e) {
        jcommander.usage();
        return;
    }
    log.info("Setting up cluster {} with zk={} global-zk={}", arguments.cluster, arguments.zookeeper, arguments.globalZookeeper);
    // Format BookKeeper metadata
    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class);
    bkConf.setZkServers(arguments.zookeeper);
    if (!BookKeeperAdmin.format(bkConf, false, /* interactive */
    false)) {
        throw new IOException("Failed to initialize BookKeeper metadata");
    }
    ZooKeeperClientFactory zkfactory = new ZookeeperClientFactoryImpl();
    ZooKeeper localZk = zkfactory.create(arguments.zookeeper, SessionType.ReadWrite, 30000).get();
    ZooKeeper globalZk = zkfactory.create(arguments.globalZookeeper, SessionType.ReadWrite, 30000).get();
    localZk.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    localZk.create("/namespace", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    try {
        ZkUtils.createFullPathOptimistic(globalZk, "/admin/policies", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    try {
        ZkUtils.createFullPathOptimistic(globalZk, "/admin/clusters", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    ClusterData clusterData = new ClusterData(arguments.clusterWebServiceUrl, arguments.clusterWebServiceUrlTls, arguments.clusterBrokerServiceUrl, arguments.clusterBrokerServiceUrlTls);
    byte[] clusterDataJson = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(clusterData);
    globalZk.create("/admin/clusters/" + arguments.cluster, clusterDataJson, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    // Create marker for "global" cluster
    ClusterData globalClusterData = new ClusterData(null, null);
    byte[] globalClusterDataJson = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(globalClusterData);
    try {
        globalZk.create("/admin/clusters/global", globalClusterDataJson, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    } catch (NodeExistsException e) {
    // Ignore
    }
    log.info("Cluster metadata for '{}' setup correctly", arguments.cluster);
}

Example 19 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class PulsarStandaloneStarter method start.

void start() throws Exception {
    if (config == null) {
        System.exit(1);
    }
    log.debug("--- setup PulsarStandaloneStarter ---");
    if (!onlyBroker) {
        // Start LocalBookKeeper
        bkEnsemble = new LocalBookkeeperEnsemble(numOfBk, zkPort, bkPort, zkDir, bkDir, wipeData);
        bkEnsemble.start();
    }
    if (noBroker) {
        return;
    }
    // Start Broker
    broker = new PulsarService(config);
    broker.start();
    // Create a sample namespace
    URL webServiceUrl = new URL(String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort()));
    String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(), config.getBrokerServicePort());
    admin = new PulsarAdmin(webServiceUrl, config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters());
    String property = "sample";
    String cluster = config.getClusterName();
    String namespace = property + "/" + cluster + "/ns1";
    try {
        ClusterData clusterData = new ClusterData(webServiceUrl.toString(), null, /* serviceUrlTls */
        brokerServiceUrl, null);
        if (!admin.clusters().getClusters().contains(cluster)) {
            admin.clusters().createCluster(cluster, clusterData);
        } else {
            admin.clusters().updateCluster(cluster, clusterData);
        }
        if (!admin.properties().getProperties().contains(property)) {
            admin.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList(config.getSuperUserRoles()), Sets.newHashSet(cluster)));
        }
        if (!admin.namespaces().getNamespaces(property).contains(namespace)) {
            admin.namespaces().createNamespace(namespace);
        }
    } catch (PulsarAdminException e) {
        log.info(e.getMessage());
    }
    log.debug("--- setup completed ---");
}

Example 20 with ClusterData

use of com.yahoo.pulsar.common.policies.data.ClusterData in project pulsar by yahoo.

the class AdminApiTest method namespaces.

@Test(invocationCount = 1)
public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
    admin.clusters().createCluster("usw", new ClusterData());
    PropertyAdmin propertyAdmin = new PropertyAdmin(Lists.newArrayList("role1", "role2"), Sets.newHashSet("use", "usw"));
    admin.properties().updateProperty("prop-xyz", propertyAdmin);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1").bundles, Policies.defaultBundle());
    admin.namespaces().createNamespace("prop-xyz/use/ns2");
    admin.namespaces().createNamespace("prop-xyz/use/ns3", 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.numBundles, 4);
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.boundaries.size(), 5);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns3");
    try {
        admin.namespaces().createNamespace("non-existing/usw/ns1");
        fail("Should not have passed");
    } catch (NotFoundException e) {
    // Ok
    }
    assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
    try {
        admin.namespaces().createNamespace("prop-xyz/usc/ns1");
        fail("Should not have passed");
    } catch (NotAuthorizedException e) {
    // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
    }
    admin.namespaces().grantPermissionOnNamespace("prop-xyz/use/ns1", "my-role", EnumSet.allOf(AuthAction.class));
    Policies policies = new Policies();
    policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.namespace_auth);
    assertEquals(admin.namespaces().getDestinations("prop-xyz/use/ns1"), Lists.newArrayList());
    admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role");
    policies.auth_policies.namespace_auth.remove("my-role");
    assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
    admin.namespaces().setPersistence("prop-xyz/use/ns1", new PersistencePolicies(3, 2, 1, 10.0));
    assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
    // Force topic creation and namespace being loaded
    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns1/my-topic");
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns1/my-topic");
    admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1", "0x00000000_0xffffffff");
    NamespaceName ns = new NamespaceName("prop-xyz/use/ns1");
    // Now, w/ bundle policies, we will use default bundle
    NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
    int i = 0;
    for (; i < 10; i++) {
        Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache().getOwnerAsync(defaultBundle).get();
        if (!data1.isPresent()) {
            // Already unloaded
            break;
        }
        LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle, data1.get().isDisabled());
        Thread.sleep(1000);
    }
    assertTrue(i < 10);
    admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
    assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2"));
    try {
        admin.namespaces().unload("prop-xyz/use/ns1");
        fail("should have raised exception");
    } catch (Exception e) {
    // OK excepted
    }
    // Force topic creation and namespace being loaded
    producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns2/my-topic");
    producer.close();
    admin.persistentTopics().delete("persistent://prop-xyz/use/ns2/my-topic");
// both unload and delete should succeed for ns2 on other broker with a redirect
// otheradmin.namespaces().unload("prop-xyz/use/ns2");
}