Examples with PropertyAdmin com.yahoo.pulsar.common.policies.data.PropertyAdmin used on opensource projects

Example 11 with PropertyAdmin

use of com.yahoo.pulsar.common.policies.data.PropertyAdmin in project pulsar by yahoo.

the class Properties method updateProperty.

@POST
@Path("/{property}")
@ApiOperation(value = "Update the admins for a property.", notes = "This operation requires Pulsar super-user privileges.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Property does not exist"), @ApiResponse(code = 409, message = "Property already exist") })
public void updateProperty(@PathParam("property") String property, PropertyAdmin newPropertyAdmin) {
    validateSuperUserAccess();
    validatePoliciesReadOnlyAccess();
    Stat nodeStat = new Stat();
    try {
        byte[] content = globalZk().getData(path("policies", property), null, nodeStat);
        PropertyAdmin oldPropertyAdmin = jsonMapper().readValue(content, PropertyAdmin.class);
        List<String> clustersWithActiveNamespaces = Lists.newArrayList();
        if (oldPropertyAdmin.getAllowedClusters().size() > newPropertyAdmin.getAllowedClusters().size()) {
            // Get the colo(s) being removed from the list
            oldPropertyAdmin.getAllowedClusters().removeAll(newPropertyAdmin.getAllowedClusters());
            log.debug("Following clusters are being removed : [{}]", oldPropertyAdmin.getAllowedClusters());
            for (String cluster : oldPropertyAdmin.getAllowedClusters()) {
                List<String> activeNamespaces = Lists.newArrayList();
                try {
                    activeNamespaces = globalZk().getChildren(path("policies", property, cluster), false);
                    if (activeNamespaces.size() != 0) {
                        // There are active namespaces in this cluster
                        clustersWithActiveNamespaces.add(cluster);
                    }
                } catch (KeeperException.NoNodeException nne) {
                // Fine, some cluster does not have active namespace. Move on!
                }
            }
            if (!clustersWithActiveNamespaces.isEmpty()) {
                // Throw an exception because colos being removed are having active namespaces
                String msg = String.format("Failed to update the property because active namespaces are present in colos %s. Please delete those namespaces first", clustersWithActiveNamespaces);
                throw new RestException(Status.CONFLICT, msg);
            }
        }
        String propertyPath = path("policies", property);
        globalZk().setData(propertyPath, jsonMapper().writeValueAsBytes(newPropertyAdmin), -1);
        globalZkCache().invalidate(propertyPath);
        log.info("[{}] updated property {}", clientAppId(), property);
    } catch (RestException re) {
        throw re;
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to update property {}: does not exist", clientAppId(), property);
        throw new RestException(Status.NOT_FOUND, "Property does not exist");
    } catch (Exception e) {
        log.error("[{}] Failed to update property {}", clientAppId(), property, e);
        throw new RestException(e);
    }
}

Example 12 with PropertyAdmin

use of com.yahoo.pulsar.common.policies.data.PropertyAdmin in project pulsar by yahoo.

the class ReplicatorTestBase method setup.

void setup() throws Exception {
    log.info("--- Starting ReplicatorTestBase::setup ---");
    int globalZKPort = PortManager.nextFreePort();
    globalZkS = new ZookeeperServerTest(globalZKPort);
    globalZkS.start();
    // Start region 1
    int zkPort1 = PortManager.nextFreePort();
    bkEnsemble1 = new LocalBookkeeperEnsemble(3, zkPort1, PortManager.nextFreePort());
    bkEnsemble1.start();
    int webServicePort1 = PortManager.nextFreePort();
    // NOTE: we have to instantiate a new copy of System.getProperties() to make sure pulsar1 and pulsar2 have
    // completely
    // independent config objects instead of referring to the same properties object
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setClusterName("r1");
    config1.setWebServicePort(webServicePort1);
    config1.setZookeeperServers("127.0.0.1:" + zkPort1);
    config1.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config1.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config1.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config1.setBrokerServicePort(PortManager.nextFreePort());
    config1.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();
    ns1 = pulsar1.getBrokerService();
    url1 = new URL("http://127.0.0.1:" + webServicePort1);
    admin1 = new PulsarAdmin(url1, (Authentication) null);
    // Start region 2
    // Start zk & bks
    int zkPort2 = PortManager.nextFreePort();
    bkEnsemble2 = new LocalBookkeeperEnsemble(3, zkPort2, PortManager.nextFreePort());
    bkEnsemble2.start();
    int webServicePort2 = PortManager.nextFreePort();
    config2 = new ServiceConfiguration();
    config2.setClusterName("r2");
    config2.setWebServicePort(webServicePort2);
    config2.setZookeeperServers("127.0.0.1:" + zkPort2);
    config2.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config2.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config2.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config2.setBrokerServicePort(PortManager.nextFreePort());
    config2.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA);
    pulsar2 = new PulsarService(config2);
    pulsar2.start();
    ns2 = pulsar2.getBrokerService();
    url2 = new URL("http://127.0.0.1:" + webServicePort2);
    admin2 = new PulsarAdmin(url2, (Authentication) null);
    // Start region 3
    // Start zk & bks
    int zkPort3 = PortManager.nextFreePort();
    bkEnsemble3 = new LocalBookkeeperEnsemble(3, zkPort3, PortManager.nextFreePort());
    bkEnsemble3.start();
    int webServicePort3 = PortManager.nextFreePort();
    config3 = new ServiceConfiguration();
    config3.setClusterName("r3");
    config3.setWebServicePort(webServicePort3);
    config3.setZookeeperServers("127.0.0.1:" + zkPort3);
    config3.setGlobalZookeeperServers("127.0.0.1:" + globalZKPort + "/foo");
    config3.setBrokerDeleteInactiveTopicsEnabled(isBrokerServicePurgeInactiveDestination());
    config3.setBrokerServicePurgeInactiveFrequencyInSeconds(inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS));
    config3.setBrokerServicePort(PortManager.nextFreePort());
    pulsar3 = new PulsarService(config3);
    pulsar3.start();
    ns3 = pulsar3.getBrokerService();
    url3 = new URL("http://127.0.0.1:" + webServicePort3);
    admin3 = new PulsarAdmin(url3, (Authentication) null);
    // Provision the global namespace
    admin1.clusters().createCluster("r1", new ClusterData(url1.toString(), null, pulsar1.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r2", new ClusterData(url2.toString(), null, pulsar2.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("r3", new ClusterData(url3.toString(), null, pulsar3.getBrokerServiceUrl(), pulsar1.getBrokerServiceUrlTls()));
    admin1.clusters().createCluster("global", new ClusterData("http://global:8080"));
    admin1.properties().createProperty("pulsar", new PropertyAdmin(Lists.newArrayList("appid1", "appid2", "appid3"), Sets.newHashSet("r1", "r2", "r3")));
    admin1.namespaces().createNamespace("pulsar/global/ns");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
    admin1.namespaces().createNamespace("pulsar/global/ns1");
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns1", Lists.newArrayList("r1", "r2"));
    assertEquals(admin2.clusters().getCluster("r1").getServiceUrl(), url1.toString());
    assertEquals(admin2.clusters().getCluster("r2").getServiceUrl(), url2.toString());
    assertEquals(admin2.clusters().getCluster("r3").getServiceUrl(), url3.toString());
    assertEquals(admin2.clusters().getCluster("r1").getBrokerServiceUrl(), pulsar1.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r2").getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl());
    assertEquals(admin2.clusters().getCluster("r3").getBrokerServiceUrl(), pulsar3.getBrokerServiceUrl());
    /*
         * assertEquals(admin2.clusters().getCluster("global").getServiceUrl(), "http://global:8080");
         * assertEquals(admin2.properties().getPropertyAdmin("pulsar").getAdminRoles(), Lists.newArrayList("appid1",
         * "appid2")); assertEquals(admin2.namespaces().getPolicies("pulsar/global/ns").replication_clusters,
         * Lists.newArrayList("r1", "r2", "r3"));
         *
         * admin1.namespaces().createNamespace("pulsar/global/ns2");
         * admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns2", Lists.newArrayList("r1", "r2",
         * "r3"));
         */
    Thread.sleep(100);
    log.info("--- ReplicatorTestBase::setup completed ---");
}

Example 13 with PropertyAdmin

use of com.yahoo.pulsar.common.policies.data.PropertyAdmin in project pulsar by yahoo.

the class AuthenticatedProducerConsumerTest method testTlsSyncProducerAndConsumer.

@Test(dataProvider = "batch")
public void testTlsSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 14 with PropertyAdmin

use of com.yahoo.pulsar.common.policies.data.PropertyAdmin in project pulsar by yahoo.

the class AuthenticatedProducerConsumerTest method testInternalServerExceptionOnLookup.

/**
     * verifies that topicLookup/PartitionMetadataLookup gives InternalServerError(500) instead 401(auth_failed) on
     * unknown-exception failure
     * 
     * @throws Exception
     */
@Test
public void testInternalServerExceptionOnLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    Map<String, String> authParams = new HashMap<>();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    internalSetup(authTls);
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(), "pulsar://localhost:" + BROKER_PORT, "pulsar+ssl://localhost:" + BROKER_PORT_TLS));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    String namespace = "my-property/use/my-ns";
    admin.namespaces().createNamespace(namespace);
    String destination = "persistent://" + namespace + "1/topic1";
    // this will cause NPE and it should throw 500
    mockZookKeeper.shutdown();
    pulsar.getConfiguration().setSuperUserRoles(Sets.newHashSet());
    try {
        admin.persistentTopics().getPartitionedTopicMetadata(destination);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
    try {
        admin.lookups().lookupDestination(destination);
    } catch (PulsarAdminException e) {
        Assert.assertTrue(e.getCause() instanceof InternalServerErrorException);
    }
}

Example 15 with PropertyAdmin

use of com.yahoo.pulsar.common.policies.data.PropertyAdmin in project pulsar by yahoo.

the class ProducerConsumerBase method producerBaseSetup.

public void producerBaseSetup() throws Exception {
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}