Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 56 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class BrokerService method checkTopicNsOwnership.

void checkTopicNsOwnership(final String topic) throws RuntimeException {
    DestinationName destination = DestinationName.get(topic);
    boolean ownedByThisInstance;
    try {
        ownedByThisInstance = pulsar.getNamespaceService().isServiceUnitOwned(destination);
    } catch (Exception e) {
        log.debug(String.format("Failed to check the ownership of the destination: %s", destination), e);
        throw new RuntimeException(new ServerMetadataException(e));
    }
    if (!ownedByThisInstance) {
        String msg = String.format("Namespace not served by this instance. Please redo the lookup. " + "Request is denied: namespace=%s", destination.getNamespace());
        log.warn(msg);
        throw new RuntimeException(new ServiceUnitNotReadyException(msg));
    }
}

Example 57 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class Consumer method checkPermissions.

public void checkPermissions() {
    DestinationName destination = DestinationName.get(subscription.getDestination());
    if (cnx.getBrokerService().getAuthorizationManager() != null) {
        try {
            if (cnx.getBrokerService().getAuthorizationManager().canConsume(destination, appId)) {
                return;
            }
        } catch (Exception e) {
            log.warn("[{}] Get unexpected error while autorizing [{}]  {}", appId, subscription.getDestination(), e.getMessage(), e);
        }
        log.info("[{}] is not allowed to consume from Destination" + " [{}] anymore", appId, subscription.getDestination());
        disconnect();
    }
}

Example 58 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithPartition.

@Test(enabled = false)
public void testSimpleConsumerEventsWithPartition() throws Exception {
    int numPartitions = 4;
    final String topicName = "persistent://prop/use/ns-abc/failover-topic2";
    final DestinationName destName = DestinationName.get(topicName);
    final String subName = "sub1";
    final int numMsgs = 100;
    Set<String> uniqueMessages = new HashSet<>();
    admin.persistentTopics().createPartitionedTopic(topicName, numPartitions);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    ConsumerConfiguration consumerConf1 = new ConsumerConfiguration();
    consumerConf1.setSubscriptionType(SubscriptionType.Failover);
    consumerConf1.setConsumerName("1");
    ConsumerConfiguration consumerConf2 = new ConsumerConfiguration();
    consumerConf2.setSubscriptionType(SubscriptionType.Failover);
    consumerConf2.setConsumerName("2");
    // 1. two consumers on the same subscription
    Consumer consumer1 = pulsarClient.subscribe(topicName, subName, consumerConf1);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subName, consumerConf2);
    PersistentTopic topicRef;
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString());
    PersistentDispatcherSingleActiveConsumer disp0 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(1).toString());
    PersistentDispatcherSingleActiveConsumer disp1 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(2).toString());
    PersistentDispatcherSingleActiveConsumer disp2 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(3).toString());
    PersistentDispatcherSingleActiveConsumer disp3 = (PersistentDispatcherSingleActiveConsumer) topicRef.getPersistentSubscription(subName).getDispatcher();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // equal distribution between both consumers
    int totalMessages = 0;
    Message msg = null;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer1.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs / 2);
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer2.acknowledge(msg);
    }
    Assert.assertEquals(totalMessages, numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    totalMessages = 0;
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // add a consumer
    ConsumerConfiguration consumerConf3 = new ConsumerConfiguration();
    consumerConf3.setSubscriptionType(SubscriptionType.Failover);
    consumerConf3.setConsumerName("3");
    for (int i = 0; i < 20; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    Consumer consumer3 = pulsarClient.subscribe(topicName, subName, consumerConf3);
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    int consumer1Messages = 0;
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer1Messages, 55);
            break;
        }
        consumer1Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    int consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 50);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    int consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 15, 10);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf1.getConsumerName());
    uniqueMessages.clear();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // remove a consumer
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2Messages = 0;
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer2Messages, 70, 5);
            break;
        }
        consumer2Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    consumer3Messages = 0;
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            Assert.assertEquals(consumer3Messages, 70, 5);
            break;
        }
        consumer3Messages++;
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), consumerConf2.getConsumerName());
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), consumerConf3.getConsumerName());
    producer.close();
    consumer2.close();
    consumer3.unsubscribe();
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 59 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class LoadBalancerTest method testUpdateLoadReportAndCheckUpdatedRanking.

/*
     * tests rankings get updated when we write write the new load reports to the zookeeper on loadbalance root node
     * tests writing pre-configured load report on the zookeeper translates the pre-calculated rankings
     */
@Test
public void testUpdateLoadReportAndCheckUpdatedRanking() throws Exception {
    for (int i = 0; i < BROKER_COUNT; i++) {
        LoadReport lr = new LoadReport();
        lr.setName(lookupAddresses[i]);
        SystemResourceUsage sru = new SystemResourceUsage();
        sru.setBandwidthIn(new ResourceUsage(256, 1024000));
        sru.setBandwidthOut(new ResourceUsage(250, 1024000));
        sru.setMemory(new ResourceUsage(1024, 8192));
        sru.setCpu(new ResourceUsage(5, 400));
        lr.setSystemResourceUsage(sru);
        String znodePath = String.format("%s/%s", SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT, lookupAddresses[i]);
        String loadReportJson = objectMapper.writeValueAsString(lr);
        bkEnsemble.getZkClient().setData(znodePath, loadReportJson.getBytes(Charsets.UTF_8), -1);
    }
    // sleep to wait the load ranking be triggered
    Thread.sleep(5000);
    // do lookup for bunch of bundles
    int totalNamespaces = 200;
    Map<String, Integer> namespaceOwner = new HashMap<>();
    for (int i = 0; i < totalNamespaces; i++) {
        DestinationName fqdn = DestinationName.get("persistent://pulsar/use/primary-ns-" + i + "/test-topic");
        ResourceUnit found = pulsarServices[0].getLoadManager().getLeastLoaded(pulsarServices[0].getNamespaceService().getBundle(fqdn));
        if (namespaceOwner.containsKey(found.getResourceId())) {
            namespaceOwner.put(found.getResourceId(), namespaceOwner.get(found.getResourceId()) + 1);
        } else {
            namespaceOwner.put(found.getResourceId(), 1);
        }
    }
    // assert that distribution variation is not more than 10%
    int averageNamespaces = totalNamespaces / BROKER_COUNT;
    int tenPercentOfAverageNamespaces = averageNamespaces / 10;
    int lowerBound = averageNamespaces - tenPercentOfAverageNamespaces;
    int upperBound = averageNamespaces + tenPercentOfAverageNamespaces;
    // assert each broker received ownership of fair amount of namespaces 90%+
    for (Map.Entry<String, Integer> broker : namespaceOwner.entrySet()) {
        log.info("Count of bundles assigned: {}, {}", broker.getKey(), broker.getValue());
        assertTrue(broker.getValue() >= lowerBound && broker.getValue() <= upperBound);
    }
}

Example 60 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PersistentTopicE2ETest method testUnloadNamespace.

@Test(enabled = false)
public void testUnloadNamespace() throws Exception {
    String topicName = "persistent://prop/use/ns-abc/topic-9";
    DestinationName destinationName = DestinationName.get(topicName);
    pulsarClient.createProducer(topicName);
    pulsarClient.close();
    assertTrue(pulsar.getBrokerService().getTopicReference(topicName) != null);
    assertTrue(((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().containsKey(destinationName.getPersistenceNamingEncoding()));
    admin.namespaces().unload("prop/use/ns-abc");
    int i = 0;
    for (i = 0; i < 30; i++) {
        if (pulsar.getBrokerService().getTopicReference(topicName) == null) {
            break;
        }
        Thread.sleep(1000);
    }
    if (i == 30) {
        fail("The topic reference should be null");
    }
    // ML should have been closed as well
    assertFalse(((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().containsKey(destinationName.getPersistenceNamingEncoding()));
}