Examples with PropertyAdmin org.apache.pulsar.common.policies.data.PropertyAdmin used on opensource projects

Example 11 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testGetConsumersAndGetTopics.

@Test(timeOut = testTimeout)
public void testGetConsumersAndGetTopics() throws Exception {
    String key = "TopicsConsumerGet";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).topic(topicName3).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    List<String> topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
    List<ConsumerImpl<byte[]>> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
    topics.forEach(topic -> log.info("topic: {}", topic));
    consumers.forEach(c -> log.info("consumer: {}", c.getTopic()));
    IntStream.range(0, 6).forEach(index -> assertTrue(topics.get(index).equals(consumers.get(index).getTopic())));
    assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 3);
    consumer.unsubscribe();
    consumer.close();
}

Example 12 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class UnAcknowledgedMessagesTimeoutTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 20;
    final int numberOfPartitions = 3;
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(100).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(100).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).consumerName("Consumer-2").subscribe();
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        MessageId msgId = producer.send(message.getBytes());
        log.info("Message produced: {} -- msgId: {}", message, msgId);
    }
    // 4. Receive messages
    int messageCount1 = receiveAllMessage(consumer1, false);
    int messageCount2 = receiveAllMessage(consumer2, true);
    int ackCount1 = 0;
    int ackCount2 = messageCount2;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Check if Messages redelivered again
    // Since receive is a blocking call hoping that timeout will kick in
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    log.info(key + " Timeout should be triggered now");
    messageCount1 = receiveAllMessage(consumer1, true);
    messageCount2 += receiveAllMessage(consumer2, false);
    ackCount1 = messageCount1;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    assertEquals(ackCount1 + messageCount2, totalMessages);
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    // Since receive is a blocking call hoping that timeout will kick in
    log.info(key + " Timeout should be triggered again");
    ackCount1 += receiveAllMessage(consumer1, true);
    ackCount2 += receiveAllMessage(consumer2, true);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(ackCount1 + ackCount2, totalMessages);
}

Example 13 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class KafkaConsumerTest method testPartitions.

@Test
public void testPartitions() throws Exception {
    String topic = "persistent://sample/standalone/ns/testPartitions";
    // Create 8 partitions in topic
    admin.properties().createProperty("sample", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topic, 8);
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "true");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // Create 2 Kakfa consumer and verify each gets half of the messages
    List<Consumer<String, String>> consumers = new ArrayList<>();
    for (int c = 0; c < 2; c++) {
        Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(topic));
        consumers.add(consumer);
    }
    int N = 8 * 3;
    for (int i = 0; i < N; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    consumers.forEach(consumer -> {
        int expectedMessaged = N / consumers.size();
        for (int i = 0; i < expectedMessaged; ) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            i += records.count();
        }
        // No more messages for this consumer
        ConsumerRecords<String, String> records = consumer.poll(100);
        assertEquals(records.count(), 0);
    });
    consumers.forEach(Consumer::close);
}

Example 14 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class PulsarClientToolTest method testInitialzation.

@Test(timeOut = 10000)
public void testInitialzation() throws MalformedURLException, InterruptedException, ExecutionException, PulsarAdminException {
    Properties properties = new Properties();
    properties.setProperty("serviceUrl", brokerUrl.toString());
    properties.setProperty("useTls", "false");
    admin.properties().createProperty("property", new PropertyAdmin());
    String topicName = "persistent://property/ns/topic-scale-ns-0/topic";
    int numberOfMessages = 10;
    ExecutorService executor = Executors.newSingleThreadExecutor();
    CompletableFuture<Void> future = new CompletableFuture<Void>();
    executor.execute(() -> {
        PulsarClientTool pulsarClientToolConsumer;
        try {
            pulsarClientToolConsumer = new PulsarClientTool(properties);
            String[] args = { "consume", "-t", "Exclusive", "-s", "sub-name", "-n", Integer.toString(numberOfMessages), "--hex", "-r", "30", topicName };
            Assert.assertEquals(pulsarClientToolConsumer.run(args), 0);
            future.complete(null);
        } catch (Throwable t) {
            future.completeExceptionally(t);
        }
    });
    // Make sure subscription has been created
    while (true) {
        try {
            List<String> subscriptions = admin.persistentTopics().getSubscriptions(topicName);
            if (subscriptions.size() == 1) {
                break;
            }
        } catch (Exception e) {
        }
        Thread.sleep(200);
    }
    PulsarClientTool pulsarClientToolProducer = new PulsarClientTool(properties);
    String[] args = { "produce", "--messages", "Have a nice day", "-n", Integer.toString(numberOfMessages), "-r", "20", topicName };
    Assert.assertEquals(pulsarClientToolProducer.run(args), 0);
    future.get();
    executor.shutdown();
}

Example 15 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class ProxyForwardAuthDataTest method testForwardAuthData.

@Test
void testForwardAuthData() throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Step 1: Create Admin Client
    createAdminClient();
    final String proxyServiceUrl = "pulsar://localhost:" + servicePort;
    // create a client which connects to proxy and pass authData
    String namespaceName = "my-property/use/my-ns";
    String topicName = "persistent://my-property/use/my-ns/my-topic1";
    String subscriptionName = "my-subscriber-name";
    String clientAuthParams = "authParam:client";
    String proxyAuthParams = "authParam:proxy";
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace(namespaceName);
    admin.namespaces().grantPermissionOnNamespace(namespaceName, "proxy", Sets.newHashSet(AuthAction.consume, AuthAction.produce));
    admin.namespaces().grantPermissionOnNamespace(namespaceName, "client", Sets.newHashSet(AuthAction.consume, AuthAction.produce));
    // Step 2: Run Pulsar Proxy without forwarding authData - expect Exception
    ProxyConfiguration proxyConfig = new ProxyConfiguration();
    proxyConfig.setAuthenticationEnabled(true);
    proxyConfig.setServicePort(servicePort);
    proxyConfig.setWebServicePort(webServicePort);
    proxyConfig.setBrokerServiceURL("pulsar://localhost:" + BROKER_PORT);
    proxyConfig.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName());
    proxyConfig.setBrokerClientAuthenticationParameters(proxyAuthParams);
    Set<String> providers = new HashSet<>();
    providers.add(BasicAuthenticationProvider.class.getName());
    proxyConfig.setAuthenticationProviders(providers);
    try (ProxyService proxyService = new ProxyService(proxyConfig);
        PulsarClient proxyClient = createPulsarClient(proxyServiceUrl, clientAuthParams)) {
        proxyService.start();
        proxyClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe();
        Assert.fail("Shouldn't be able to subscribe, auth required");
    } catch (PulsarClientException.AuthorizationException e) {
    // expected behaviour
    }
    // Step 3: Create proxy with forwardAuthData enabled
    proxyConfig.setForwardAuthorizationCredentials(true);
    try (ProxyService proxyService = new ProxyService(proxyConfig);
        PulsarClient proxyClient = createPulsarClient(proxyServiceUrl, clientAuthParams)) {
        proxyService.start();
        proxyClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscribe().close();
    }
}