Examples with EnsemblePlacementPolicyConfig org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig used on opensource projects

Example 1 with EnsemblePlacementPolicyConfig

use of org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig in project pulsar by apache.

the class BrokerBookieIsolationTest method testBookieIsolation.

/**
 * Validate that broker can support tenant based bookie isolation.
 *
 * <pre>
 * 1. create two bookie-info group : default-group and isolated-group
 * 2. namespace ns1 : uses default-group
 *    validate: bookie-ensemble for ns1-topics's ledger will be from default-group
 * 3. namespace ns2,ns3,ns4: uses isolated-group
 *    validate: bookie-ensemble for above namespace-topics's ledger will be from isolated-group
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testBookieIsolation() throws Exception {
    final String tenant1 = "tenant1";
    final String cluster = "use";
    final String ns1 = String.format("%s/%s/%s", tenant1, cluster, "ns1");
    final String ns2 = String.format("%s/%s/%s", tenant1, cluster, "ns2");
    final String ns3 = String.format("%s/%s/%s", tenant1, cluster, "ns3");
    final String ns4 = String.format("%s/%s/%s", tenant1, cluster, "ns4");
    final int totalPublish = 100;
    final String brokerBookkeeperClientIsolationGroups = "default-group";
    final String tenantNamespaceIsolationGroups = "tenant1-isolation";
    BookieServer[] bookies = bkEnsemble.getBookies();
    ZooKeeper zkClient = bkEnsemble.getZkClient();
    Set<BookieId> defaultBookies = Sets.newHashSet(bookies[0].getBookieId(), bookies[1].getBookieId());
    Set<BookieId> isolatedBookies = Sets.newHashSet(bookies[2].getBookieId(), bookies[3].getBookieId());
    setDefaultIsolationGroup(brokerBookkeeperClientIsolationGroups, zkClient, defaultBookies);
    setDefaultIsolationGroup(tenantNamespaceIsolationGroups, zkClient, isolatedBookies);
    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName(cluster);
    config.setWebServicePort(Optional.of(0));
    config.setMetadataStoreUrl("zk:127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerShutdownTimeoutMs(0L);
    config.setBrokerServicePort(Optional.of(0));
    config.setAdvertisedAddress("localhost");
    config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups);
    config.setManagedLedgerDefaultEnsembleSize(2);
    config.setManagedLedgerDefaultWriteQuorum(2);
    config.setManagedLedgerDefaultAckQuorum(2);
    config.setAllowAutoTopicCreationType("non-partitioned");
    int totalEntriesPerLedger = 20;
    int totalLedgers = totalPublish / totalEntriesPerLedger;
    config.setManagedLedgerMaxEntriesPerLedger(totalEntriesPerLedger);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsarService = new PulsarService(config);
    pulsarService.start();
    PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build();
    ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build();
    admin.clusters().createCluster(cluster, clusterData);
    TenantInfoImpl tenantInfo = new TenantInfoImpl(null, Sets.newHashSet(cluster));
    admin.tenants().createTenant(tenant1, tenantInfo);
    admin.namespaces().createNamespace(ns1);
    admin.namespaces().createNamespace(ns2);
    admin.namespaces().createNamespace(ns3);
    admin.namespaces().createNamespace(ns4);
    admin.namespaces().setBookieAffinityGroup(ns2, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    admin.namespaces().setBookieAffinityGroup(ns3, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    admin.namespaces().setBookieAffinityGroup(ns4, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns2), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns3), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns4), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups).build());
    try {
        admin.namespaces().getBookieAffinityGroup(ns1);
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarService.getBrokerServiceUrl()).statsInterval(-1, TimeUnit.SECONDS).build();
    PersistentTopic topic1 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns1, "topic1", totalPublish);
    PersistentTopic topic2 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns2, "topic1", totalPublish);
    PersistentTopic topic3 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns3, "topic1", totalPublish);
    PersistentTopic topic4 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns4, "topic1", totalPublish);
    Bookie bookie1 = bookies[0].getBookie();
    Field ledgerManagerField = Bookie.class.getDeclaredField("ledgerManager");
    ledgerManagerField.setAccessible(true);
    LedgerManager ledgerManager = (LedgerManager) ledgerManagerField.get(bookie1);
    // namespace: ns1
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), defaultBookies);
    // namespace: ns2
    ml = (ManagedLedgerImpl) topic2.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies);
    // namespace: ns3
    ml = (ManagedLedgerImpl) topic3.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies);
    // namespace: ns4
    ml = (ManagedLedgerImpl) topic4.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies);
    ManagedLedgerClientFactory mlFactory = (ManagedLedgerClientFactory) pulsarService.getManagedLedgerClientFactory();
    Map<EnsemblePlacementPolicyConfig, BookKeeper> bkPlacementPolicyToBkClientMap = mlFactory.getBkEnsemblePolicyToBookKeeperMap();
    // broker should create only 1 bk-client and factory per isolation-group
    assertEquals(bkPlacementPolicyToBkClientMap.size(), 1);
    // make sure bk-isolation group also configure REPP_DNS_RESOLVER_CLASS as ZkBookieRackAffinityMapping to
    // configure rack-aware policy with in isolated group
    Map<EnsemblePlacementPolicyConfig, BookKeeper> bkMap = mlFactory.getBkEnsemblePolicyToBookKeeperMap();
    BookKeeper bk = bkMap.values().iterator().next();
    Method getConf = BookKeeper.class.getDeclaredMethod("getConf");
    getConf.setAccessible(true);
    ClientConfiguration clientConf = (ClientConfiguration) getConf.invoke(bk);
    assertEquals(clientConf.getProperty(REPP_DNS_RESOLVER_CLASS), BookieRackAffinityMapping.class.getName());
}

Example 2 with EnsemblePlacementPolicyConfig

use of org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig in project pulsar by apache.

the class BrokerBookieIsolationTest method testBookieIsolationWithSecondaryGroup.

/**
 * It verifies that "ZkIsolatedBookieEnsemblePlacementPolicy" considers secondary affinity-group if primary group
 * doesn't have enough non-faulty bookies.
 *
 * @throws Exception
 */
@Test
public void testBookieIsolationWithSecondaryGroup() throws Exception {
    final String tenant1 = "tenant1";
    final String cluster = "use";
    final String ns1 = String.format("%s/%s/%s", tenant1, cluster, "ns1");
    final String ns2 = String.format("%s/%s/%s", tenant1, cluster, "ns2");
    final String ns3 = String.format("%s/%s/%s", tenant1, cluster, "ns3");
    final String ns4 = String.format("%s/%s/%s", tenant1, cluster, "ns4");
    final int totalPublish = 100;
    final String brokerBookkeeperClientIsolationGroups = "default-group";
    final String tenantNamespaceIsolationGroupsPrimary = "tenant1-isolation-primary";
    final String tenantNamespaceIsolationGroupsSecondary = "tenant1-isolation=secondary";
    BookieServer[] bookies = bkEnsemble.getBookies();
    ZooKeeper zkClient = bkEnsemble.getZkClient();
    Set<BookieId> defaultBookies = Sets.newHashSet(bookies[0].getBookieId(), bookies[1].getBookieId());
    Set<BookieId> isolatedBookies = Sets.newHashSet(bookies[2].getBookieId(), bookies[3].getBookieId());
    Set<BookieId> downedBookies = Sets.newHashSet(BookieId.parse("1.1.1.1:1111"), BookieId.parse("1.1.1.1:1112"));
    setDefaultIsolationGroup(brokerBookkeeperClientIsolationGroups, zkClient, defaultBookies);
    // primary group empty
    setDefaultIsolationGroup(tenantNamespaceIsolationGroupsPrimary, zkClient, downedBookies);
    setDefaultIsolationGroup(tenantNamespaceIsolationGroupsSecondary, zkClient, isolatedBookies);
    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName(cluster);
    config.setWebServicePort(Optional.of(0));
    config.setMetadataStoreUrl("zk:127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerShutdownTimeoutMs(0L);
    config.setBrokerServicePort(Optional.of(0));
    config.setAdvertisedAddress("localhost");
    config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups);
    config.setManagedLedgerDefaultEnsembleSize(2);
    config.setManagedLedgerDefaultWriteQuorum(2);
    config.setManagedLedgerDefaultAckQuorum(2);
    config.setAllowAutoTopicCreationType("non-partitioned");
    int totalEntriesPerLedger = 20;
    int totalLedgers = totalPublish / totalEntriesPerLedger;
    config.setManagedLedgerMaxEntriesPerLedger(totalEntriesPerLedger);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsarService = new PulsarService(config);
    pulsarService.start();
    PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build();
    ClusterData clusterData = ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build();
    admin.clusters().createCluster(cluster, clusterData);
    TenantInfoImpl tenantInfo = new TenantInfoImpl(null, Sets.newHashSet(cluster));
    admin.tenants().createTenant(tenant1, tenantInfo);
    admin.namespaces().createNamespace(ns1);
    admin.namespaces().createNamespace(ns2);
    admin.namespaces().createNamespace(ns3);
    admin.namespaces().createNamespace(ns4);
    admin.namespaces().setBookieAffinityGroup(ns2, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).bookkeeperAffinityGroupSecondary(tenantNamespaceIsolationGroupsSecondary).build());
    admin.namespaces().setBookieAffinityGroup(ns3, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).bookkeeperAffinityGroupSecondary(tenantNamespaceIsolationGroupsSecondary).build());
    admin.namespaces().setBookieAffinityGroup(ns4, BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns2), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).bookkeeperAffinityGroupSecondary(tenantNamespaceIsolationGroupsSecondary).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns3), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).bookkeeperAffinityGroupSecondary(tenantNamespaceIsolationGroupsSecondary).build());
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns4), BookieAffinityGroupData.builder().bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroupsPrimary).build());
    try {
        admin.namespaces().getBookieAffinityGroup(ns1);
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    @Cleanup PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarService.getBrokerServiceUrl()).statsInterval(-1, TimeUnit.SECONDS).build();
    PersistentTopic topic1 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns1, "topic1", totalPublish);
    PersistentTopic topic2 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns2, "topic1", totalPublish);
    PersistentTopic topic3 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns3, "topic1", totalPublish);
    Bookie bookie1 = bookies[0].getBookie();
    Field ledgerManagerField = Bookie.class.getDeclaredField("ledgerManager");
    ledgerManagerField.setAccessible(true);
    LedgerManager ledgerManager = (LedgerManager) ledgerManagerField.get(bookie1);
    // namespace: ns1
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic1.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), defaultBookies);
    // namespace: ns2
    ml = (ManagedLedgerImpl) topic2.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies);
    // namespace: ns3
    ml = (ManagedLedgerImpl) topic3.getManagedLedger();
    assertEquals(ml.getLedgersInfoAsList().size(), totalLedgers);
    // validate ledgers' ensemble with affinity bookies
    assertAffinityBookies(ledgerManager, ml.getLedgersInfoAsList(), isolatedBookies);
    ManagedLedgerClientFactory mlFactory = (ManagedLedgerClientFactory) pulsarService.getManagedLedgerClientFactory();
    Map<EnsemblePlacementPolicyConfig, BookKeeper> bkPlacementPolicyToBkClientMap = mlFactory.getBkEnsemblePolicyToBookKeeperMap();
    // broker should create only 1 bk-client and factory per isolation-group
    assertEquals(bkPlacementPolicyToBkClientMap.size(), 1);
    // ns4 doesn't have secondary group so, publish should fail
    try {
        PersistentTopic topic4 = (PersistentTopic) createTopicAndPublish(pulsarClient, ns4, "topic1", 1);
        fail("should have failed due to not enough non-faulty bookie");
    } catch (BrokerPersistenceException e) {
    // Ok..
    }
}

Example 3 with EnsemblePlacementPolicyConfig

use of org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig in project pulsar by apache.

the class IsolatedBookieEnsemblePlacementPolicy method replaceBookie.

@Override
public PlacementResult<BookieId> replaceBookie(int ensembleSize, int writeQuorumSize, int ackQuorumSize, Map<String, byte[]> customMetadata, List<BookieId> currentEnsemble, BookieId bookieToReplace, Set<BookieId> excludeBookies) throws BKNotEnoughBookiesException {
    // parse the ensemble placement policy from the custom metadata, if it is present, we will apply it to
    // the isolation groups for filtering the bookies.
    Optional<EnsemblePlacementPolicyConfig> ensemblePlacementPolicyConfig = getEnsemblePlacementPolicyConfig(customMetadata);
    Set<BookieId> blacklistedBookies;
    if (ensemblePlacementPolicyConfig.isPresent()) {
        EnsemblePlacementPolicyConfig config = ensemblePlacementPolicyConfig.get();
        Pair<Set<String>, Set<String>> groups = getIsolationGroup(config);
        blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, groups);
    } else {
        blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, defaultIsolationGroups);
    }
    if (excludeBookies == null) {
        excludeBookies = new HashSet<BookieId>();
    }
    excludeBookies.addAll(blacklistedBookies);
    return super.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, currentEnsemble, bookieToReplace, excludeBookies);
}

Example 4 with EnsemblePlacementPolicyConfig

use of org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig in project pulsar by apache.

the class ManagedLedgerTest method testManagedLedgerWithPlacementPolicyInCustomMetadata.

@Test(timeOut = 10000)
public void testManagedLedgerWithPlacementPolicyInCustomMetadata() throws Exception {
    ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
    managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName(MockedPlacementPolicy.class);
    managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(Collections.singletonMap("key", "value"));
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", managedLedgerConfig);
    assertFalse(ledger.createdLedgerCustomMetadata.isEmpty());
    byte[] configData = ledger.createdLedgerCustomMetadata.get(EnsemblePlacementPolicyConfig.ENSEMBLE_PLACEMENT_POLICY_CONFIG);
    EnsemblePlacementPolicyConfig config = EnsemblePlacementPolicyConfig.decode(configData);
    assertEquals(config.getPolicyClass().getName(), MockedPlacementPolicy.class.getName());
    assertEquals(config.getProperties().size(), 1);
    assertTrue(config.getProperties().containsKey("key"));
    assertEquals(config.getProperties().get("key"), "value");
}

Example 5 with EnsemblePlacementPolicyConfig

use of org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig in project pulsar by apache.

the class ManagedLedgerClientFactory method initialize.

public void initialize(ServiceConfiguration conf, MetadataStoreExtended metadataStore, BookKeeperClientFactory bookkeeperProvider, EventLoopGroup eventLoopGroup) throws Exception {
    ManagedLedgerFactoryConfig managedLedgerFactoryConfig = new ManagedLedgerFactoryConfig();
    managedLedgerFactoryConfig.setMaxCacheSize(conf.getManagedLedgerCacheSizeMB() * 1024L * 1024L);
    managedLedgerFactoryConfig.setCacheEvictionWatermark(conf.getManagedLedgerCacheEvictionWatermark());
    managedLedgerFactoryConfig.setNumManagedLedgerSchedulerThreads(conf.getManagedLedgerNumSchedulerThreads());
    managedLedgerFactoryConfig.setCacheEvictionFrequency(conf.getManagedLedgerCacheEvictionFrequency());
    managedLedgerFactoryConfig.setCacheEvictionTimeThresholdMillis(conf.getManagedLedgerCacheEvictionTimeThresholdMillis());
    managedLedgerFactoryConfig.setCopyEntriesInCache(conf.isManagedLedgerCacheCopyEntries());
    managedLedgerFactoryConfig.setPrometheusStatsLatencyRolloverSeconds(conf.getManagedLedgerPrometheusStatsLatencyRolloverSeconds());
    managedLedgerFactoryConfig.setTraceTaskExecution(conf.isManagedLedgerTraceTaskExecution());
    managedLedgerFactoryConfig.setCursorPositionFlushSeconds(conf.getManagedLedgerCursorPositionFlushSeconds());
    managedLedgerFactoryConfig.setManagedLedgerInfoCompressionType(conf.getManagedLedgerInfoCompressionType());
    managedLedgerFactoryConfig.setStatsPeriodSeconds(conf.getManagedLedgerStatsPeriodSeconds());
    Configuration configuration = new ClientConfiguration();
    if (conf.isBookkeeperClientExposeStatsToPrometheus()) {
        configuration.addProperty(PrometheusMetricsProvider.PROMETHEUS_STATS_LATENCY_ROLLOVER_SECONDS, conf.getManagedLedgerPrometheusStatsLatencyRolloverSeconds());
        configuration.addProperty(PrometheusMetricsProvider.CLUSTER_NAME, conf.getClusterName());
        statsProvider = new PrometheusMetricsProvider();
    }
    statsProvider.start(configuration);
    StatsLogger statsLogger = statsProvider.getStatsLogger("pulsar_managedLedger_client");
    this.defaultBkClient = bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, Optional.empty(), null);
    BookkeeperFactoryForCustomEnsemblePlacementPolicy bkFactory = (EnsemblePlacementPolicyConfig ensemblePlacementPolicyConfig) -> {
        BookKeeper bkClient = null;
        // find or create bk-client in cache for a specific ensemblePlacementPolicy
        if (ensemblePlacementPolicyConfig != null && ensemblePlacementPolicyConfig.getPolicyClass() != null) {
            bkClient = bkEnsemblePolicyToBkClientMap.computeIfAbsent(ensemblePlacementPolicyConfig, (key) -> {
                try {
                    return bookkeeperProvider.create(conf, metadataStore, eventLoopGroup, Optional.ofNullable(ensemblePlacementPolicyConfig.getPolicyClass()), ensemblePlacementPolicyConfig.getProperties());
                } catch (Exception e) {
                    log.error("Failed to initialize bk-client for policy {}, properties {}", ensemblePlacementPolicyConfig.getPolicyClass(), ensemblePlacementPolicyConfig.getProperties(), e);
                }
                return this.defaultBkClient;
            });
        }
        return bkClient != null ? bkClient : defaultBkClient;
    };
    this.managedLedgerFactory = new ManagedLedgerFactoryImpl(metadataStore, bkFactory, managedLedgerFactoryConfig, statsLogger);
}