Examples with ClusterData org.apache.pulsar.common.policies.data.ClusterData used on opensource projects

Example 41 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class CompactorTest method setup.

@BeforeMethod
@Override
public void setup() throws Exception {
    super.internalSetup();
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
    compactionScheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("compactor").setDaemon(true).build());
}

Example 42 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testGlobalNamespaceThrottling.

/**
 * <pre>
 * Verifies setting dispatch-rate on global namespace.
 * 1. It sets dispatch-rate for a local cluster into global-zk.policies
 * 2. Topic fetches dispatch-rate for the local cluster from policies
 * 3. applies dispatch rate
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testGlobalNamespaceThrottling() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/global/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 5;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.clusters().createCluster("global", new ClusterData("http://global:8080"));
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("use"));
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 43 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ProducerConsumerBase method producerBaseSetup.

public void producerBaseSetup() throws Exception {
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}

Example 44 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ConsumerConfigurationTest method setup.

@BeforeMethod
@Override
public void setup() throws Exception {
    super.internalSetup();
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}

Example 45 with ClusterData

use of org.apache.pulsar.common.policies.data.ClusterData in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers.

/**
 * It verifies namespace-isolation policies with primary and secondary brokers.
 *
 * usecase:
 *
 * <pre>
 *  1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1
 *     a. available-brokers: broker1, broker2, broker3 => result: broker1
 *     b. available-brokers: broker2, broker3          => result: broker2
 *     c. available-brokers: broker3                   => result: NULL
 *  2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2
 *     a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2
 *     b. available-brokers: broker2, broker3          => result: broker2
 *     c. available-brokers: broker3                   => result: NULL
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws Exception {
    final String property = "my-property";
    final String cluster = "use";
    final String namespace = "my-ns";
    final String broker1Address = pulsar1.getAdvertisedAddress() + "0";
    final String broker2Address = pulsar2.getAdvertisedAddress() + "1";
    final String sharedBroker = "broker3";
    admin1.clusters().createCluster(cluster, new ClusterData("http://" + pulsar1.getAdvertisedAddress()));
    admin1.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(cluster)));
    admin1.namespaces().createNamespace(property + "/" + cluster + "/" + namespace);
    // set a new policy
    String newPolicyJsonTemplate = "{\"namespaces\":[\"%s/%s/%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}";
    String newPolicyJson = String.format(newPolicyJsonTemplate, property, cluster, namespace, broker1Address, broker2Address, 1);
    String newPolicyName = "my-ns-isolation-policies";
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    NamespaceIsolationData nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationData.class);
    admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData);
    SimpleResourceAllocationPolicies simpleResourceAllocationPolicies = new SimpleResourceAllocationPolicies(pulsar1);
    ServiceUnitId serviceUnit = LoadBalancerTestingUtils.makeBundles(nsFactory, property, cluster, namespace, 1)[0];
    BrokerTopicLoadingPredicate brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() {

        @Override
        public boolean isEnablePersistentTopics(String brokerUrl) {
            return true;
        }

        @Override
        public boolean isEnableNonPersistentTopics(String brokerUrl) {
            return true;
        }
    };
    // (1) now we have isolation policy : primary=broker1, secondary=broker2, minLimit=1
    // test1: shared=1, primary=1, secondary=1 => It should return 1 primary broker only
    Set<String> brokerCandidateCache = Sets.newHashSet();
    Set<String> availableBrokers = Sets.newHashSet(sharedBroker, broker1Address, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker1Address));
    // test2: shared=1, primary=0, secondary=1 => It should return 1 secondary broker only
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test3: shared=1, primary=0, secondary=0 => It should return 0 broker
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 0);
    // (2) now we will have isolation policy : primary=broker1, secondary=broker2, minLimit=2
    newPolicyJson = String.format(newPolicyJsonTemplate, property, cluster, namespace, broker1Address, broker2Address, 2);
    nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationData.class);
    admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData);
    // test1: shared=1, primary=1, secondary=1 => It should return primary + secondary
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker1Address, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 2);
    assertTrue(brokerCandidateCache.contains(broker1Address));
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test2: shared=1, secondary=1 => It should return secondary
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test3: shared=1, => It should return 0 broker
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 0);
}