Examples with PropertyAdmin org.apache.pulsar.common.policies.data.PropertyAdmin used on opensource projects

Example 71 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testSubscribeUnsubscribeSingleTopic.

@Test
public void testSubscribeUnsubscribeSingleTopic() throws Exception {
    String key = "TopicsConsumerSubscribeUnsubscribeSingleTopicTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }
    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);
    // 4, unsubscribe topic3
    CompletableFuture<Void> unsubFuture = ((TopicsConsumerImpl<byte[]>) consumer).unsubscribeAsync(topicName3);
    unsubFuture.get();
    // 5. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-round2" + i).getBytes());
        producer2.send((messagePredicate + "producer2-round2" + i).getBytes());
        producer3.send((messagePredicate + "producer3-round2" + i).getBytes());
    }
    // 6. should not receive messages from topic3, verify get 2/3 of all messages
    messageSet = 0;
    message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages * 2 / 3);
    // 7. use getter to verify internal topics number after un-subscribe topic3
    List<String> topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
    List<ConsumerImpl<byte[]>> consumers = ((TopicsConsumerImpl) consumer).getConsumers();
    assertEquals(topics.size(), 3);
    assertEquals(consumers.size(), 3);
    assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 2);
    // 8. re-subscribe topic3
    CompletableFuture<Void> subFuture = ((TopicsConsumerImpl<byte[]>) consumer).subscribeAsync(topicName3);
    subFuture.get();
    // 9. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-round3" + i).getBytes());
        producer2.send((messagePredicate + "producer2-round3" + i).getBytes());
        producer3.send((messagePredicate + "producer3-round3" + i).getBytes());
    }
    // 10. should receive messages from all 3 topics
    messageSet = 0;
    message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);
    // 11. use getter to verify internal topics number after subscribe topic3
    topics = ((TopicsConsumerImpl<byte[]>) consumer).getPartitionedTopics();
    consumers = ((TopicsConsumerImpl) consumer).getConsumers();
    assertEquals(topics.size(), 6);
    assertEquals(consumers.size(), 6);
    assertTrue(((TopicsConsumerImpl<byte[]>) consumer).getTopics().size() == 3);
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 72 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class AntiAffinityNamespaceGroupTest method testBrokerSelectionForAntiAffinityGroup.

/**
 * It verifies anti-affinity with failure domain enabled with 2 brokers.
 *
 * <pre>
 * 1. Register brokers to domain: domain-1: broker1 & domain-2: broker2
 * 2. Load-Manager receives a watch and updates brokerToDomain cache with new domain data
 * 3. Create two namespace with anti-affinity
 * 4. Load-manager selects broker for each namespace such that from different domains
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testBrokerSelectionForAntiAffinityGroup() throws Exception {
    final String broker1 = primaryHost;
    final String broker2 = secondaryHost;
    final String cluster = pulsar1.getConfiguration().getClusterName();
    final String property = "prop";
    final String namespace1 = property + "/" + cluster + "/ns1";
    final String namespace2 = property + "/" + cluster + "/ns2";
    final String namespaceAntiAffinityGroup = "group";
    FailureDomain domain = new FailureDomain();
    domain.brokers = Sets.newHashSet(broker1);
    admin1.clusters().createFailureDomain(cluster, "domain1", domain);
    domain.brokers = Sets.newHashSet(broker2);
    admin1.clusters().createFailureDomain(cluster, "domain1", domain);
    admin1.properties().createProperty(property, new PropertyAdmin(null, Sets.newHashSet(cluster)));
    admin1.namespaces().createNamespace(namespace1);
    admin1.namespaces().createNamespace(namespace2);
    admin1.namespaces().setNamespaceAntiAffinityGroup(namespace1, namespaceAntiAffinityGroup);
    admin1.namespaces().setNamespaceAntiAffinityGroup(namespace2, namespaceAntiAffinityGroup);
    // validate strategically if brokerToDomainCache updated
    for (int i = 0; i < 5; i++) {
        if (!isLoadManagerUpdatedDomainCache(primaryLoadManager) || !isLoadManagerUpdatedDomainCache(secondaryLoadManager) || i != 4) {
            Thread.sleep(200);
        }
    }
    assertTrue(isLoadManagerUpdatedDomainCache(primaryLoadManager));
    assertTrue(isLoadManagerUpdatedDomainCache(secondaryLoadManager));
    ServiceUnitId serviceUnit = makeBundle(property, cluster, "ns1");
    String selectedBroker1 = primaryLoadManager.selectBrokerForAssignment(serviceUnit).get();
    serviceUnit = makeBundle(property, cluster, "ns2");
    String selectedBroker2 = primaryLoadManager.selectBrokerForAssignment(serviceUnit).get();
    assertNotEquals(selectedBroker1, selectedBroker2);
}

Example 73 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class AntiAffinityNamespaceGroupTest method testAntiAffinityNamespaceFilteringWithDomain.

/**
 * It verifies anti-affinity-namespace assignment with failure-domain
 *
 * <pre>
 * Domain     Brokers-count
 * ________  ____________
 * domain-0   broker-0,broker-1
 * domain-1   broker-2,broker-3
 *
 * Anti-affinity-namespace assignment
 *
 * (1) ns0 -> candidate-brokers: b0, b1, b2, b3 => selected b0
 * (2) ns1 -> candidate-brokers: b2, b3         => selected b2
 * (3) ns2 -> candidate-brokers: b1, b3         => selected b1
 * (4) ns3 -> candidate-brokers: b3             => selected b3
 * (5) ns4 -> candidate-brokers: b0, b1, b2, b3 => selected b0
 *
 * "candidate" broker to own anti-affinity-namespace = b2 or b4
 *
 * </pre>
 */
@Test
public void testAntiAffinityNamespaceFilteringWithDomain() throws Exception {
    final String namespace = "my-property/use/my-ns";
    final int totalNamespaces = 5;
    final String namespaceAntiAffinityGroup = "my-antiaffinity";
    final String bundle = "/0x00000000_0xffffffff";
    final int totalBrokers = 4;
    pulsar1.getConfiguration().setFailureDomainsEnabled(true);
    admin1.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    for (int i = 0; i < totalNamespaces; i++) {
        final String ns = namespace + i;
        admin1.namespaces().createNamespace(ns);
        admin1.namespaces().setNamespaceAntiAffinityGroup(ns, namespaceAntiAffinityGroup);
    }
    Set<String> brokers = Sets.newHashSet();
    Map<String, String> brokerToDomainMap = Maps.newHashMap();
    brokers.add("brokerName-0");
    brokerToDomainMap.put("brokerName-0", "domain-0");
    brokers.add("brokerName-1");
    brokerToDomainMap.put("brokerName-1", "domain-0");
    brokers.add("brokerName-2");
    brokerToDomainMap.put("brokerName-2", "domain-1");
    brokers.add("brokerName-3");
    brokerToDomainMap.put("brokerName-3", "domain-1");
    Set<String> candidate = Sets.newHashSet();
    Map<String, Map<String, Set<String>>> brokerToNamespaceToBundleRange = Maps.newHashMap();
    assertEquals(brokers.size(), totalBrokers);
    String assignedNamespace = namespace + "0" + bundle;
    candidate.addAll(brokers);
    // for namespace-0 all brokers available
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers, brokerToNamespaceToBundleRange, brokerToDomainMap);
    assertEquals(brokers.size(), totalBrokers);
    // add namespace-0 to broker-0 of domain-0 => state: n0->b0
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "brokerName-0", namespace + "0", assignedNamespace);
    candidate.addAll(brokers);
    // for namespace-1 only domain-1 brokers are available as broker-0 already owns namespace-0
    assignedNamespace = namespace + "1" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, brokerToDomainMap);
    assertEquals(candidate.size(), 2);
    candidate.forEach(broker -> assertEquals(brokerToDomainMap.get(broker), "domain-1"));
    // add namespace-1 to broker-2 of domain-1 => state: n0->b0, n1->b2
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "brokerName-2", namespace + "1", assignedNamespace);
    candidate.addAll(brokers);
    // for namespace-2 only brokers available are : broker-1 and broker-3
    assignedNamespace = namespace + "2" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, brokerToDomainMap);
    assertEquals(candidate.size(), 2);
    assertTrue(candidate.contains("brokerName-1"));
    assertTrue(candidate.contains("brokerName-3"));
    // add namespace-2 to broker-1 of domain-0 => state: n0->b0, n1->b2, n2->b1
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "brokerName-1", namespace + "2", assignedNamespace);
    candidate.addAll(brokers);
    // for namespace-3 only brokers available are : broker-3
    assignedNamespace = namespace + "3" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, brokerToDomainMap);
    assertEquals(candidate.size(), 1);
    assertTrue(candidate.contains("brokerName-3"));
    // add namespace-3 to broker-3 of domain-1 => state: n0->b0, n1->b2, n2->b1, n3->b3
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "brokerName-3", namespace + "3", assignedNamespace);
    candidate.addAll(brokers);
    // for namespace-4 only brokers available are : all 4 brokers
    assignedNamespace = namespace + "4" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, brokerToDomainMap);
    assertEquals(candidate.size(), 4);
}

Example 74 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class AntiAffinityNamespaceGroupTest method testAntiAffinityNamespaceFilteringWithoutDomain.

/**
 * It verifies anti-affinity-namespace assignment without failure-domain enabled
 *
 * <pre>
 *  Anti-affinity-namespace assignment
 *
 * (1) ns0 -> candidate-brokers: b0, b1, b2     => selected b0
 * (2) ns1 -> candidate-brokers: b1, b2         => selected b1
 * (3) ns2 -> candidate-brokers: b2             => selected b2
 * (5) ns3 -> candidate-brokers: b0, b1, b2     => selected b0
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testAntiAffinityNamespaceFilteringWithoutDomain() throws Exception {
    final String namespace = "my-property/use/my-ns";
    final int totalNamespaces = 5;
    final String namespaceAntiAffinityGroup = "my-antiaffinity";
    final String bundle = "/0x00000000_0xffffffff";
    admin1.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    for (int i = 0; i < totalNamespaces; i++) {
        final String ns = namespace + i;
        admin1.namespaces().createNamespace(ns);
        admin1.namespaces().setNamespaceAntiAffinityGroup(ns, namespaceAntiAffinityGroup);
    }
    Set<String> brokers = Sets.newHashSet();
    Set<String> candidate = Sets.newHashSet();
    Map<String, Map<String, Set<String>>> brokerToNamespaceToBundleRange = Maps.newHashMap();
    brokers.add("broker-0");
    brokers.add("broker-1");
    brokers.add("broker-2");
    String assignedNamespace = namespace + "0" + bundle;
    // all brokers available so, candidate will be all 3 brokers
    candidate.addAll(brokers);
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, brokers, brokerToNamespaceToBundleRange, null);
    assertEquals(brokers.size(), 3);
    // add ns-0 to broker-0
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-0", namespace + "0", assignedNamespace);
    candidate.addAll(brokers);
    assignedNamespace = namespace + "1" + bundle;
    // available brokers for ns-1 => broker-1, broker-2
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, null);
    assertEquals(candidate.size(), 2);
    assertTrue(candidate.contains("broker-1"));
    assertTrue(candidate.contains("broker-2"));
    // add ns-1 to broker-1
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-1", namespace + "1", assignedNamespace);
    candidate.addAll(brokers);
    // available brokers for ns-2 => broker-2
    assignedNamespace = namespace + "2" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, null);
    assertEquals(candidate.size(), 1);
    assertTrue(candidate.contains("broker-2"));
    // add ns-2 to broker-2
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-2", namespace + "2", assignedNamespace);
    candidate.addAll(brokers);
    // available brokers for ns-3 => broker-0, broker-1, broker-2
    assignedNamespace = namespace + "3" + bundle;
    LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar1, assignedNamespace, candidate, brokerToNamespaceToBundleRange, null);
    assertEquals(candidate.size(), 3);
}

Example 75 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class AntiAffinityNamespaceGroupTest method testLoadSheddingUtilWithAntiAffinityNamespace.

/**
 * It verifies that load-shedding task should unload namespace only if there is a broker available which doesn't
 * cause uneven anti-affinitiy namespace distribution.
 *
 * <pre>
 * 1. broker1 owns ns-0 => broker1 can unload ns-0
 * 1. broker2 owns ns-1 => broker1 can unload ns-0
 * 1. broker3 owns ns-2 => broker1 can't unload ns-0 as all brokers have same no NS
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLoadSheddingUtilWithAntiAffinityNamespace() throws Exception {
    final String namespace = "my-property/use/my-ns";
    final int totalNamespaces = 5;
    final String namespaceAntiAffinityGroup = "my-antiaffinity";
    final String bundle = "/0x00000000_0xffffffff";
    admin1.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    for (int i = 0; i < totalNamespaces; i++) {
        final String ns = namespace + i;
        admin1.namespaces().createNamespace(ns);
        admin1.namespaces().setNamespaceAntiAffinityGroup(ns, namespaceAntiAffinityGroup);
    }
    Set<String> brokers = Sets.newHashSet();
    Set<String> candidate = Sets.newHashSet();
    Map<String, Map<String, Set<String>>> brokerToNamespaceToBundleRange = Maps.newHashMap();
    brokers.add("broker-0");
    brokers.add("broker-1");
    brokers.add("broker-2");
    String assignedNamespace = namespace + "0" + bundle;
    // all brokers available so, candidate will be all 3 brokers
    candidate.addAll(brokers);
    // add ns-0 to broker-0
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-0", namespace + "0", assignedNamespace);
    String currentBroker = "broker-0";
    boolean shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker, pulsar1, brokerToNamespaceToBundleRange, candidate);
    assertTrue(shouldUnload);
    // add ns-1 to broker-1
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-1", namespace + "1", assignedNamespace);
    shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker, pulsar1, brokerToNamespaceToBundleRange, candidate);
    assertTrue(shouldUnload);
    // add ns-2 to broker-2
    selectBrokerForNamespace(brokerToNamespaceToBundleRange, "broker-2", namespace + "2", assignedNamespace);
    shouldUnload = LoadManagerShared.shouldAntiAffinityNamespaceUnload(namespace + "0", bundle, currentBroker, pulsar1, brokerToNamespaceToBundleRange, candidate);
    assertFalse(shouldUnload);
}