Examples with NamespaceIsolationData org.apache.pulsar.common.policies.data.NamespaceIsolationData used on opensource projects

Example 6 with NamespaceIsolationData

use of org.apache.pulsar.common.policies.data.NamespaceIsolationData in project incubator-pulsar by apache.

the class NamespaceIsolationPoliciesTest method testSetPolicy.

@Test
public void testSetPolicy() throws Exception {
    NamespaceIsolationPolicies policies = this.getDefaultTestPolicies();
    // set a new policy
    String newPolicyJson = "{\"namespaces\":[\"pulsar/use/TESTNS.*\"],\"primary\":[\"prod1-broker[45].messaging.use.example.com\"],\"secondary\":[\"prod1-broker.*.use.example.com\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":2,\"usage_threshold\":80}}}";
    String newPolicyName = "policy2";
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    NamespaceIsolationData nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationData.class);
    policies.setPolicy(newPolicyName, nsPolicyData);
    assertEquals(policies.getPolicies().size(), 2);
    assertEquals(policies.getPolicyByName(newPolicyName), new NamespaceIsolationPolicyImpl(nsPolicyData));
    assertTrue(!policies.getPolicyByName(newPolicyName).equals(policies.getPolicyByName("policy1")));
    assertEquals(policies.getPolicyByNamespace(NamespaceName.get("pulsar/use/TESTNS.1")), new NamespaceIsolationPolicyImpl(nsPolicyData));
}

Example 7 with NamespaceIsolationData

use of org.apache.pulsar.common.policies.data.NamespaceIsolationData in project incubator-pulsar by apache.

the class LoadBalancerTest method createNamespacePolicies.

private void createNamespacePolicies(PulsarService pulsar) throws Exception {
    // // prepare three policies for the namespace isolation
    NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies();
    // set up policy that use this broker as primary
    NamespaceIsolationData policyData = new NamespaceIsolationData();
    policyData.namespaces = new ArrayList<String>();
    policyData.namespaces.add("pulsar/use/primary-ns.*");
    policyData.primary = new ArrayList<String>();
    for (int i = 0; i < BROKER_COUNT; i++) {
        policyData.primary.add(pulsarServices[i].getAdvertisedAddress());
    }
    policyData.secondary = new ArrayList<String>();
    policyData.auto_failover_policy = new AutoFailoverPolicyData();
    policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
    policyData.auto_failover_policy.parameters = new HashMap<String, String>();
    policyData.auto_failover_policy.parameters.put("min_limit", "1");
    policyData.auto_failover_policy.parameters.put("usage_threshold", "100");
    policies.setPolicy("primaryBrokerPolicy", policyData);
    // set up policy that use this broker as secondary
    policyData = new NamespaceIsolationData();
    policyData.namespaces = new ArrayList<String>();
    policyData.namespaces.add("pulsar/use/secondary-ns.*");
    policyData.primary = new ArrayList<String>();
    policyData.primary.add(pulsarServices[0].getAdvertisedAddress());
    policyData.secondary = new ArrayList<String>();
    for (int i = 1; i < BROKER_COUNT; i++) {
        policyData.secondary.add(pulsarServices[i].getAdvertisedAddress());
    }
    policyData.auto_failover_policy = new AutoFailoverPolicyData();
    policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
    policyData.auto_failover_policy.parameters = new HashMap<String, String>();
    policyData.auto_failover_policy.parameters.put("min_limit", "1");
    policyData.auto_failover_policy.parameters.put("usage_threshold", "100");
    policies.setPolicy("secondaryBrokerPolicy", policyData);
    // set up policy that do not use this broker (neither primary nor secondary)
    policyData = new NamespaceIsolationData();
    policyData.namespaces = new ArrayList<String>();
    policyData.namespaces.add("pulsar/use/shared-ns.*");
    policyData.primary = new ArrayList<String>();
    policyData.primary.add(pulsarServices[0].getAdvertisedAddress());
    policyData.secondary = new ArrayList<String>();
    for (int i = 1; i < BROKER_COUNT; i++) {
        policyData.secondary.add(pulsarServices[i].getAdvertisedAddress());
    }
    policyData.auto_failover_policy = new AutoFailoverPolicyData();
    policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
    policyData.auto_failover_policy.parameters = new HashMap<String, String>();
    policyData.auto_failover_policy.parameters.put("min_limit", "1");
    policyData.auto_failover_policy.parameters.put("usage_threshold", "100");
    policies.setPolicy("otherBrokerPolicy", policyData);
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    ZooKeeper globalZk = pulsar.getGlobalZkCache().getZooKeeper();
    ZkUtils.createFullPathOptimistic(globalZk, AdminResource.path("clusters", "use", "namespaceIsolationPolicies"), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    byte[] content = jsonMapper.writeValueAsBytes(policies.getPolicies());
    globalZk.setData(AdminResource.path("clusters", "use", "namespaceIsolationPolicies"), content, -1);
}

Example 8 with NamespaceIsolationData

use of org.apache.pulsar.common.policies.data.NamespaceIsolationData in project incubator-pulsar by apache.

the class ModularLoadManagerImplTest method testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers.

/**
 * It verifies namespace-isolation policies with primary and secondary brokers.
 *
 * usecase:
 *
 * <pre>
 *  1. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 1
 *     a. available-brokers: broker1, broker2, broker3 => result: broker1
 *     b. available-brokers: broker2, broker3          => result: broker2
 *     c. available-brokers: broker3                   => result: NULL
 *  2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2
 *     a. available-brokers: broker1, broker2, broker3 => result: broker1, broker2
 *     b. available-brokers: broker2, broker3          => result: broker2
 *     c. available-brokers: broker3                   => result: NULL
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() throws Exception {
    final String property = "my-property";
    final String cluster = "use";
    final String namespace = "my-ns";
    final String broker1Address = pulsar1.getAdvertisedAddress() + "0";
    final String broker2Address = pulsar2.getAdvertisedAddress() + "1";
    final String sharedBroker = "broker3";
    admin1.clusters().createCluster(cluster, new ClusterData("http://" + pulsar1.getAdvertisedAddress()));
    admin1.properties().createProperty(property, new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet(cluster)));
    admin1.namespaces().createNamespace(property + "/" + cluster + "/" + namespace);
    // set a new policy
    String newPolicyJsonTemplate = "{\"namespaces\":[\"%s/%s/%s.*\"],\"primary\":[\"%s\"]," + "\"secondary\":[\"%s\"],\"auto_failover_policy\":{\"policy_type\":\"min_available\",\"parameters\":{\"min_limit\":%s,\"usage_threshold\":80}}}";
    String newPolicyJson = String.format(newPolicyJsonTemplate, property, cluster, namespace, broker1Address, broker2Address, 1);
    String newPolicyName = "my-ns-isolation-policies";
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    NamespaceIsolationData nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationData.class);
    admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData);
    SimpleResourceAllocationPolicies simpleResourceAllocationPolicies = new SimpleResourceAllocationPolicies(pulsar1);
    ServiceUnitId serviceUnit = LoadBalancerTestingUtils.makeBundles(nsFactory, property, cluster, namespace, 1)[0];
    BrokerTopicLoadingPredicate brokerTopicLoadingPredicate = new BrokerTopicLoadingPredicate() {

        @Override
        public boolean isEnablePersistentTopics(String brokerUrl) {
            return true;
        }

        @Override
        public boolean isEnableNonPersistentTopics(String brokerUrl) {
            return true;
        }
    };
    // (1) now we have isolation policy : primary=broker1, secondary=broker2, minLimit=1
    // test1: shared=1, primary=1, secondary=1 => It should return 1 primary broker only
    Set<String> brokerCandidateCache = Sets.newHashSet();
    Set<String> availableBrokers = Sets.newHashSet(sharedBroker, broker1Address, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker1Address));
    // test2: shared=1, primary=0, secondary=1 => It should return 1 secondary broker only
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test3: shared=1, primary=0, secondary=0 => It should return 0 broker
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 0);
    // (2) now we will have isolation policy : primary=broker1, secondary=broker2, minLimit=2
    newPolicyJson = String.format(newPolicyJsonTemplate, property, cluster, namespace, broker1Address, broker2Address, 2);
    nsPolicyData = jsonMapper.readValue(newPolicyJson.getBytes(), NamespaceIsolationData.class);
    admin1.clusters().createNamespaceIsolationPolicy("use", newPolicyName, nsPolicyData);
    // test1: shared=1, primary=1, secondary=1 => It should return primary + secondary
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker1Address, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 2);
    assertTrue(brokerCandidateCache.contains(broker1Address));
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test2: shared=1, secondary=1 => It should return secondary
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker, broker2Address);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 1);
    assertTrue(brokerCandidateCache.contains(broker2Address));
    // test3: shared=1, => It should return 0 broker
    brokerCandidateCache = Sets.newHashSet();
    availableBrokers = Sets.newHashSet(sharedBroker);
    LoadManagerShared.applyNamespacePolicies(serviceUnit, simpleResourceAllocationPolicies, brokerCandidateCache, availableBrokers, brokerTopicLoadingPredicate);
    assertEquals(brokerCandidateCache.size(), 0);
}

Example 9 with NamespaceIsolationData

use of org.apache.pulsar.common.policies.data.NamespaceIsolationData in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method createNamespacePolicies.

private void createNamespacePolicies(PulsarService pulsar) throws Exception {
    NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies();
    // set up policy that use this broker as primary
    NamespaceIsolationData policyData = new NamespaceIsolationData();
    policyData.namespaces = new ArrayList<String>();
    policyData.namespaces.add("pulsar/use/primary-ns.*");
    policyData.primary = new ArrayList<String>();
    policyData.primary.add(pulsar1.getAdvertisedAddress() + "*");
    policyData.secondary = new ArrayList<String>();
    policyData.secondary.add("prod2-broker([78]).messaging.usw.example.co.*");
    policyData.auto_failover_policy = new AutoFailoverPolicyData();
    policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
    policyData.auto_failover_policy.parameters = new HashMap<String, String>();
    policyData.auto_failover_policy.parameters.put("min_limit", "1");
    policyData.auto_failover_policy.parameters.put("usage_threshold", "100");
    policies.setPolicy("primaryBrokerPolicy", policyData);
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    ZooKeeper globalZk = pulsar.getGlobalZkCache().getZooKeeper();
    ZkUtils.createFullPathOptimistic(globalZk, AdminResource.path("clusters", "use", "namespaceIsolationPolicies"), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    byte[] content = jsonMapper.writeValueAsBytes(policies.getPolicies());
    globalZk.setData(AdminResource.path("clusters", "use", "namespaceIsolationPolicies"), content, -1);
}

Example 10 with NamespaceIsolationData

use of org.apache.pulsar.common.policies.data.NamespaceIsolationData in project incubator-pulsar by apache.

the class AdminApiTest method clusterNamespaceIsolationPolicies.

@Test
public void clusterNamespaceIsolationPolicies() throws PulsarAdminException {
    try {
        // create
        String policyName1 = "policy-1";
        NamespaceIsolationData nsPolicyData1 = new NamespaceIsolationData();
        nsPolicyData1.namespaces = new ArrayList<String>();
        nsPolicyData1.namespaces.add("other/use/other.*");
        nsPolicyData1.primary = new ArrayList<String>();
        nsPolicyData1.primary.add("prod1-broker[4-6].messaging.use.example.com");
        nsPolicyData1.secondary = new ArrayList<String>();
        nsPolicyData1.secondary.add("prod1-broker.*.messaging.use.example.com");
        nsPolicyData1.auto_failover_policy = new AutoFailoverPolicyData();
        nsPolicyData1.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
        nsPolicyData1.auto_failover_policy.parameters = new HashMap<String, String>();
        nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "1");
        nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "100");
        admin.clusters().createNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
        String policyName2 = "policy-2";
        NamespaceIsolationData nsPolicyData2 = new NamespaceIsolationData();
        nsPolicyData2.namespaces = new ArrayList<String>();
        nsPolicyData2.namespaces.add("other/use/other.*");
        nsPolicyData2.primary = new ArrayList<String>();
        nsPolicyData2.primary.add("prod1-broker[4-6].messaging.use.example.com");
        nsPolicyData2.secondary = new ArrayList<String>();
        nsPolicyData2.secondary.add("prod1-broker.*.messaging.use.example.com");
        nsPolicyData2.auto_failover_policy = new AutoFailoverPolicyData();
        nsPolicyData2.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
        nsPolicyData2.auto_failover_policy.parameters = new HashMap<String, String>();
        nsPolicyData2.auto_failover_policy.parameters.put("min_limit", "1");
        nsPolicyData2.auto_failover_policy.parameters.put("usage_threshold", "100");
        admin.clusters().createNamespaceIsolationPolicy("use", policyName2, nsPolicyData2);
        // verify create indirectly with get
        Map<String, NamespaceIsolationData> policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
        assertEquals(policiesMap.get(policyName1), nsPolicyData1);
        assertEquals(policiesMap.get(policyName2), nsPolicyData2);
        // verify update of primary
        nsPolicyData1.primary.remove(0);
        nsPolicyData1.primary.add("prod1-broker[1-2].messaging.use.example.com");
        admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
        // verify primary change
        policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
        assertEquals(policiesMap.get(policyName1), nsPolicyData1);
        // verify update of secondary
        nsPolicyData1.secondary.remove(0);
        nsPolicyData1.secondary.add("prod1-broker[3-4].messaging.use.example.com");
        admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
        // verify secondary change
        policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
        assertEquals(policiesMap.get(policyName1), nsPolicyData1);
        // verify update of failover policy limit
        nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "10");
        admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
        // verify min_limit change
        policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
        assertEquals(policiesMap.get(policyName1), nsPolicyData1);
        // verify update of failover usage_threshold limit
        nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "80");
        admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
        // verify usage_threshold change
        policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
        assertEquals(policiesMap.get(policyName1), nsPolicyData1);
        // verify single get
        NamespaceIsolationData policy1Data = admin.clusters().getNamespaceIsolationPolicy("use", policyName1);
        assertEquals(policy1Data, nsPolicyData1);
        // verify creation of more than one policy
        admin.clusters().createNamespaceIsolationPolicy("use", policyName2, nsPolicyData1);
        try {
            admin.clusters().getNamespaceIsolationPolicy("use", "no-such-policy");
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof NotFoundException);
        }
        // verify delete cluster failed
        try {
            admin.clusters().deleteCluster("use");
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof PreconditionFailedException);
        }
        // verify delete
        admin.clusters().deleteNamespaceIsolationPolicy("use", policyName1);
        admin.clusters().deleteNamespaceIsolationPolicy("use", policyName2);
        try {
            admin.clusters().getNamespaceIsolationPolicy("use", policyName1);
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof NotFoundException);
        }
        try {
            admin.clusters().getNamespaceIsolationPolicy("use", policyName2);
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof NotFoundException);
        }
        try {
            admin.clusters().getNamespaceIsolationPolicies("usc");
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof NotFoundException);
        }
        try {
            admin.clusters().getNamespaceIsolationPolicy("usc", "no-such-cluster");
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof PreconditionFailedException);
        }
        try {
            admin.clusters().createNamespaceIsolationPolicy("usc", "no-such-cluster", nsPolicyData1);
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof PreconditionFailedException);
        }
        try {
            admin.clusters().updateNamespaceIsolationPolicy("usc", "no-such-cluster", policy1Data);
            fail("should have raised exception");
        } catch (PulsarAdminException e) {
            assertTrue(e instanceof PreconditionFailedException);
        }
    } catch (PulsarAdminException e) {
        LOG.warn("TEST FAILED [{}]", e.getMessage());
        throw e;
    }
}