Examples with ZooKeeperChildrenCache org.apache.pulsar.zookeeper.ZooKeeperChildrenCache used on opensource projects

Example 1 with ZooKeeperChildrenCache

use of org.apache.pulsar.zookeeper.ZooKeeperChildrenCache in project incubator-pulsar by apache.

the class SimpleLoadManagerImplTest method testPrimarySecondary.

@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
    createNamespacePolicies(pulsar1);
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
    log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    zkCacheField.set(pulsar1, mockCache);
    LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
    log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
    log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(NamespaceName.get("pulsar/use/primary-ns.10")).get();
    assertEquals(found.getResourceId(), ru1.getResourceId());
    zkCacheField.set(pulsar1, originalLZK1);
}

Example 2 with ZooKeeperChildrenCache

use of org.apache.pulsar.zookeeper.ZooKeeperChildrenCache in project incubator-pulsar by apache.

the class DiscoveryServiceTest method addBrokerToZk.

private void addBrokerToZk(int number) throws Exception {
    for (int i = 0; i < number; i++) {
        LoadReport report = new LoadReport(null, null, "pulsar://broker-:15000" + i, null);
        String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report);
        ZkUtils.createFullPathOptimistic(mockZookKeeper, LOADBALANCE_BROKERS_ROOT + "/" + "broker-" + i, reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    }
    // sometimes test-environment takes longer time to trigger async mockZK-watch: so reload cache explicitly
    Field field = ZookeeperCacheLoader.class.getDeclaredField("availableBrokersCache");
    field.setAccessible(true);
    ZooKeeperChildrenCache availableBrokersCache = (ZooKeeperChildrenCache) field.get(service.getDiscoveryProvider().localZkCache);
    availableBrokersCache.reloadCache(LOADBALANCE_BROKERS_ROOT);
}

Example 3 with ZooKeeperChildrenCache

use of org.apache.pulsar.zookeeper.ZooKeeperChildrenCache in project incubator-pulsar by apache.

the class SimpleLoadManagerImpl method initialize.

@Override
public void initialize(final PulsarService pulsar) {
    if (SystemUtils.IS_OS_LINUX) {
        brokerHostUsage = new LinuxBrokerHostUsageImpl(pulsar);
    } else {
        brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar);
    }
    this.policies = new SimpleResourceAllocationPolicies(pulsar);
    lastLoadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    lastLoadReport.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    lastLoadReport.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    loadReportCacheZk = new ZooKeeperDataCache<LoadReport>(pulsar.getLocalZkCache()) {

        @Override
        public LoadReport deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, LoadReport.class);
        }
    };
    loadReportCacheZk.registerListener(this);
    this.dynamicConfigurationCache = new ZooKeeperDataCache<Map<String, String>>(pulsar.getLocalZkCache()) {

        @Override
        public Map<String, String> deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, HashMap.class);
        }
    };
    int entryExpiryTime = (int) pulsar.getConfiguration().getLoadBalancerSheddingGracePeriodMinutes();
    unloadedHotNamespaceCache = CacheBuilder.newBuilder().expireAfterWrite(entryExpiryTime, TimeUnit.MINUTES).build(new CacheLoader<String, Long>() {

        @Override
        public Long load(String key) throws Exception {
            return System.currentTimeMillis();
        }
    });
    availableActiveBrokers = new ZooKeeperChildrenCache(pulsar.getLocalZkCache(), LOADBALANCE_BROKERS_ROOT);
    availableActiveBrokers.registerListener(new ZooKeeperCacheListener<Set<String>>() {

        @Override
        public void onUpdate(String path, Set<String> data, Stat stat) {
            if (log.isDebugEnabled()) {
                log.debug("Update Received for path {}", path);
            }
            scheduler.submit(SimpleLoadManagerImpl.this::updateRanking);
        }
    });
    this.pulsar = pulsar;
}

Example 4 with ZooKeeperChildrenCache

use of org.apache.pulsar.zookeeper.ZooKeeperChildrenCache in project incubator-pulsar by apache.

the class ModularLoadManagerImpl method initialize.

/**
 * Initialize this load manager using the given PulsarService. Should be called only once, after invoking the
 * default constructor.
 *
 * @param pulsar
 *            The service to initialize with.
 */
public void initialize(final PulsarService pulsar) {
    this.pulsar = pulsar;
    availableActiveBrokers = new ZooKeeperChildrenCache(pulsar.getLocalZkCache(), LoadManager.LOADBALANCE_BROKERS_ROOT);
    availableActiveBrokers.registerListener(new ZooKeeperCacheListener<Set<String>>() {

        @Override
        public void onUpdate(String path, Set<String> data, Stat stat) {
            if (log.isDebugEnabled()) {
                log.debug("Update Received for path {}", path);
            }
            reapDeadBrokerPreallocations(data);
            scheduler.submit(ModularLoadManagerImpl.this::updateAll);
        }
    });
    brokerDataCache = new ZooKeeperDataCache<LocalBrokerData>(pulsar.getLocalZkCache()) {

        @Override
        public LocalBrokerData deserialize(String key, byte[] content) throws Exception {
            return ObjectMapperFactory.getThreadLocal().readValue(content, LocalBrokerData.class);
        }
    };
    brokerDataCache.registerListener(this);
    if (SystemUtils.IS_OS_LINUX) {
        brokerHostUsage = new LinuxBrokerHostUsageImpl(pulsar);
    } else {
        brokerHostUsage = new GenericBrokerHostUsageImpl(pulsar);
    }
    bundleSplitStrategy = new BundleSplitterTask(pulsar);
    conf = pulsar.getConfiguration();
    // Initialize the default stats to assume for unseen bundles (hard-coded for now).
    defaultStats.msgThroughputIn = DEFAULT_MESSAGE_THROUGHPUT;
    defaultStats.msgThroughputOut = DEFAULT_MESSAGE_THROUGHPUT;
    defaultStats.msgRateIn = DEFAULT_MESSAGE_RATE;
    defaultStats.msgRateOut = DEFAULT_MESSAGE_RATE;
    lastData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
    localData.setBrokerVersionString(pulsar.getBrokerVersion());
    // configure broker-topic mode
    lastData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    lastData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    localData.setPersistentTopicsEnabled(pulsar.getConfiguration().isEnablePersistentTopics());
    localData.setNonPersistentTopicsEnabled(pulsar.getConfiguration().isEnableNonPersistentTopics());
    placementStrategy = ModularLoadManagerStrategy.create(conf);
    policies = new SimpleResourceAllocationPolicies(pulsar);
    zkClient = pulsar.getZkClient();
    filterPipeline.add(new BrokerVersionFilter());
    refreshBrokerToFailureDomainMap();
    // register listeners for domain changes
    pulsar.getConfigurationCache().failureDomainListCache().registerListener((path, data, stat) -> scheduler.execute(() -> refreshBrokerToFailureDomainMap()));
    pulsar.getConfigurationCache().failureDomainCache().registerListener((path, data, stat) -> scheduler.execute(() -> refreshBrokerToFailureDomainMap()));
}