Examples with Policies com.yahoo.pulsar.common.policies.data.Policies used on opensource projects

Example 41 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class PersistentDispatcherFailoverConsumerTest method setup.

@BeforeMethod
public void setup() throws Exception {
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    PulsarService pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = mock(ZooKeeper.class);
    doReturn(mockZk).when(pulsar).getZkClient();
    configCacheService = mock(ConfigurationCacheService.class);
    @SuppressWarnings("unchecked") ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    doReturn(configCacheService).when(pulsar).getConfigurationCache();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    doReturn(true).when(serverCnx).isWritable();
    doReturn(new InetSocketAddress("localhost", 1234)).when(serverCnx).clientAddress();
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(DestinationName.class));
    setupMLAsyncCallbackMocks();
}

Example 42 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class LoadBalancerTest method createNamespace.

private void createNamespace(PulsarService pulsar, String namespace, int numBundles) throws Exception {
    Policies policies = new Policies();
    policies.bundles = getBundles(numBundles);
    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    ZooKeeper globalZk = pulsar.getGlobalZkCache().getZooKeeper();
    String zpath = AdminResource.path("policies", namespace);
    ZkUtils.createFullPathOptimistic(globalZk, zpath, jsonMapper.writeValueAsBytes(policies), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
}

Example 43 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class NamespaceServiceTest method testSplitAndOwnBundles.

@Test
public void testSplitAndOwnBundles() throws Exception {
    OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
    doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
    Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
    ownership.setAccessible(true);
    ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
    NamespaceService namespaceService = pulsar.getNamespaceService();
    NamespaceName nsname = new NamespaceName("pulsar/global/ns1");
    DestinationName dn = DestinationName.get("persistent://pulsar/global/ns1/topic-1");
    NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname);
    NamespaceBundle originalBundle = bundles.findBundle(dn);
    // Split bundle and take ownership of split bundles
    CompletableFuture<Void> result = namespaceService.splitAndOwnBundle(originalBundle);
    try {
        result.get();
    } catch (Exception e) {
        // make sure: no failure
        fail("split bundle faild", e);
    }
    NamespaceBundleFactory bundleFactory = this.pulsar.getNamespaceService().getNamespaceBundleFactory();
    NamespaceBundles updatedNsBundles = bundleFactory.getBundles(nsname);
    // new updated bundles shouldn't be null
    assertNotNull(updatedNsBundles);
    List<NamespaceBundle> bundleList = updatedNsBundles.getBundles();
    assertNotNull(bundles);
    NamespaceBundleFactory utilityFactory = NamespaceBundleFactory.createFactory(Hashing.crc32());
    // (1) validate bundleFactory-cache has newly split bundles and removed old parent bundle
    Pair<NamespaceBundles, List<NamespaceBundle>> splitBundles = splitBundles(utilityFactory, nsname, bundles, originalBundle);
    assertNotNull(splitBundles);
    Set<NamespaceBundle> splitBundleSet = new HashSet<>(splitBundles.getRight());
    splitBundleSet.removeAll(bundleList);
    assertTrue(splitBundleSet.isEmpty());
    // (2) validate LocalZookeeper policies updated with newly created split
    // bundles
    String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString());
    byte[] content = this.pulsar.getLocalZkCache().getZooKeeper().getData(path, null, new Stat());
    Policies policies = ObjectMapperFactory.getThreadLocal().readValue(content, Policies.class);
    NamespaceBundles localZkBundles = bundleFactory.getBundles(nsname, policies.bundles);
    assertTrue(updatedNsBundles.equals(localZkBundles));
    log.info("Policies: {}", policies);
    // (3) validate ownership of new split bundles by local owner
    bundleList.stream().forEach(b -> {
        try {
            byte[] data = this.pulsar.getLocalZkCache().getZooKeeper().getData(ServiceUnitZkUtils.path(b), null, new Stat());
            NamespaceEphemeralData node = ObjectMapperFactory.getThreadLocal().readValue(data, NamespaceEphemeralData.class);
            Assert.assertEquals(node.getNativeUrl(), this.pulsar.getBrokerServiceUrl());
        } catch (Exception e) {
            fail("failed to setup ownership", e);
        }
    });
}

Example 44 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class PersistentTopic method startReplProducers.

public void startReplProducers() {
    // read repl-cluster from policies to avoid restart of replicator which are in process of disconnect and close
    try {
        Policies policies = brokerService.pulsar().getConfigurationCache().policiesCache().get(AdminResource.path("policies", DestinationName.get(topic).getNamespace())).orElseThrow(() -> new KeeperException.NoNodeException());
        if (policies.replication_clusters != null) {
            Set<String> configuredClusters = Sets.newTreeSet(policies.replication_clusters);
            replicators.forEach((region, replicator) -> {
                if (configuredClusters.contains(region)) {
                    replicator.startProducer();
                }
            });
        }
    } catch (Exception e) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Error getting policies while starting repl-producers {}", topic, e.getMessage());
        }
        replicators.forEach((region, replicator) -> replicator.startProducer());
    }
}

Example 45 with Policies

use of com.yahoo.pulsar.common.policies.data.Policies in project pulsar by yahoo.

the class PersistentTopic method checkReplication.

@Override
public CompletableFuture<Void> checkReplication() {
    DestinationName name = DestinationName.get(topic);
    if (!name.isGlobal()) {
        return CompletableFuture.completedFuture(null);
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Checking replication status", name);
    }
    Policies policies = null;
    try {
        policies = brokerService.pulsar().getConfigurationCache().policiesCache().get(AdminResource.path("policies", name.getNamespace())).orElseThrow(() -> new KeeperException.NoNodeException());
    } catch (Exception e) {
        CompletableFuture<Void> future = new CompletableFuture<>();
        future.completeExceptionally(new ServerMetadataException(e));
        return future;
    }
    final int newMessageTTLinSeconds = policies.message_ttl_in_seconds;
    Set<String> configuredClusters;
    if (policies.replication_clusters != null) {
        configuredClusters = Sets.newTreeSet(policies.replication_clusters);
    } else {
        configuredClusters = Collections.emptySet();
    }
    String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
    List<CompletableFuture<Void>> futures = Lists.newArrayList();
    // Check for missing replicators
    for (String cluster : configuredClusters) {
        if (cluster.equals(localCluster)) {
            continue;
        }
        if (!replicators.containsKey(cluster)) {
            futures.add(startReplicator(cluster));
        }
    }
    // Check for replicators to be stopped
    replicators.forEach((cluster, replicator) -> {
        // Update message TTL
        replicator.updateMessageTTL(newMessageTTLinSeconds);
        if (!cluster.equals(localCluster)) {
            if (!configuredClusters.contains(cluster)) {
                futures.add(removeReplicator(cluster));
            }
        }
    });
    return FutureUtil.waitForAll(futures);
}