Examples with NamespaceName com.yahoo.pulsar.common.naming.NamespaceName used on opensource projects

Example 86 with NamespaceName

use of com.yahoo.pulsar.common.naming.NamespaceName in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testPrimarySecondary.

@Test(enabled = false)
public void testPrimarySecondary() throws Exception {
    LocalZooKeeperCache mockCache = mock(LocalZooKeeperCache.class);
    ZooKeeperChildrenCache zooKeeperChildrenCache = mock(ZooKeeperChildrenCache.class);
    Set<String> activeBrokers = Sets.newHashSet("prod2-broker7.messaging.use.example.com:8080", "prod2-broker8.messaging.use.example.com:8080", "prod2-broker9.messaging.use.example.com:8080");
    when(mockCache.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get()).thenReturn(activeBrokers);
    when(zooKeeperChildrenCache.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT)).thenReturn(activeBrokers);
    Field zkCacheField = PulsarService.class.getDeclaredField("localZkCache");
    zkCacheField.setAccessible(true);
    LocalZooKeeperCache originalLZK1 = (LocalZooKeeperCache) zkCacheField.get(pulsar1);
    LocalZooKeeperCache originalLZK2 = (LocalZooKeeperCache) zkCacheField.get(pulsar2);
    log.info("lzk are {} 2: {}", originalLZK1.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT), originalLZK2.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    zkCacheField.set(pulsar1, mockCache);
    LocalZooKeeperCache newZk = (LocalZooKeeperCache) pulsar1.getLocalZkCache();
    log.info("lzk mocked are {}", newZk.getChildren(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    ZooKeeperChildrenCache availableActiveBrokers = new ZooKeeperChildrenCache(pulsar1.getLocalZkCache(), SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT);
    log.info("lzk mocked active brokers are {}", availableActiveBrokers.get(SimpleLoadManagerImpl.LOADBALANCE_BROKERS_ROOT));
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://prod2-broker7.messaging.usw.example.com:8080", rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    Field sortedRankings = SimpleLoadManagerImpl.class.getDeclaredField("sortedRankings");
    sortedRankings.setAccessible(true);
    sortedRankings.set(loadManager, sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
    assertEquals(found.getResourceId(), ru1.getResourceId());
    zkCacheField.set(pulsar1, originalLZK1);
}

Example 87 with NamespaceName

use of com.yahoo.pulsar.common.naming.NamespaceName in project pulsar by yahoo.

the class SimpleLoadManagerImplTest method testPrimary.

@Test(enabled = true)
public void testPrimary() throws Exception {
    LoadManager loadManager = new SimpleLoadManagerImpl(pulsar1);
    PulsarResourceDescription rd = new PulsarResourceDescription();
    rd.put("memory", new ResourceUsage(1024, 4096));
    rd.put("cpu", new ResourceUsage(10, 100));
    rd.put("bandwidthIn", new ResourceUsage(250 * 1024, 1024 * 1024));
    rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024));
    ResourceUnit ru1 = new SimpleResourceUnit("http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd);
    Set<ResourceUnit> rus = new HashSet<ResourceUnit>();
    rus.add(ru1);
    LoadRanker lr = new ResourceAvailabilityRanker();
    // inject the load report and rankings
    Map<ResourceUnit, com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport> loadReports = new HashMap<>();
    com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport();
    loadReport.setSystemResourceUsage(new SystemResourceUsage());
    loadReports.put(ru1, loadReport);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports);
    ResourceUnitRanking ranking = new ResourceUnitRanking(loadReport.getSystemResourceUsage(), new HashSet<String>(), new ResourceQuota(), new HashSet<String>(), new ResourceQuota());
    Map<ResourceUnit, ResourceUnitRanking> rankings = new HashMap<>();
    rankings.put(ru1, ranking);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "resourceUnitRankings", rankings);
    AtomicReference<Map<Long, Set<ResourceUnit>>> sortedRankingsInstance = new AtomicReference<>(Maps.newTreeMap());
    sortedRankingsInstance.get().put(lr.getRank(rd), rus);
    setObjectField(SimpleLoadManagerImpl.class, loadManager, "sortedRankings", sortedRankingsInstance);
    ResourceUnit found = ((SimpleLoadManagerImpl) loadManager).getLeastLoaded(new NamespaceName("pulsar/use/primary-ns.10"));
    // broker is not active so found should be null
    assertNotEquals(found, null, "did not find a broker when expected one to be found");
}

Example 88 with NamespaceName

use of com.yahoo.pulsar.common.naming.NamespaceName in project pulsar by yahoo.

the class SimpleLoadManagerImpl method doNamespaceBundleSplit.

/**
     * Detect and split hot namespace bundles
     */
@Override
public void doNamespaceBundleSplit() throws Exception {
    int maxBundleCount = pulsar.getConfiguration().getLoadBalancerNamespaceMaximumBundles();
    long maxBundleTopics = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxTopics();
    long maxBundleSessions = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxSessions();
    long maxBundleMsgRate = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxMsgRate();
    long maxBundleBandwidth = pulsar.getConfiguration().getLoadBalancerNamespaceBundleMaxBandwidthMbytes() * MBytes;
    log.info("Running namespace bundle split with thresholds: topics {}, sessions {}, msgRate {}, bandwidth {}, maxBundles {}", maxBundleTopics, maxBundleSessions, maxBundleMsgRate, maxBundleBandwidth, maxBundleCount);
    if (this.lastLoadReport == null || this.lastLoadReport.getBundleStats() == null) {
        return;
    }
    Map<String, NamespaceBundleStats> bundleStats = this.lastLoadReport.getBundleStats();
    Set<String> bundlesToBeSplit = new HashSet<>();
    for (Map.Entry<String, NamespaceBundleStats> statsEntry : bundleStats.entrySet()) {
        String bundleName = statsEntry.getKey();
        NamespaceBundleStats stats = statsEntry.getValue();
        long totalSessions = stats.consumerCount + stats.producerCount;
        double totalMsgRate = stats.msgRateIn + stats.msgRateOut;
        double totalBandwidth = stats.msgThroughputIn + stats.msgThroughputOut;
        boolean needSplit = false;
        if (stats.topics > maxBundleTopics || totalSessions > maxBundleSessions || totalMsgRate > maxBundleMsgRate || totalBandwidth > maxBundleBandwidth) {
            if (stats.topics <= 1) {
                log.info("Unable to split hot namespace bundle {} since there is only one topic.", bundleName);
            } else {
                NamespaceName namespaceName = new NamespaceName(getNamespaceNameFromBundleName(bundleName));
                int numBundles = pulsar.getNamespaceService().getBundleCount(namespaceName);
                if (numBundles >= maxBundleCount) {
                    log.info("Unable to split hot namespace bundle {} since the namespace has too many bundles.", bundleName);
                } else {
                    needSplit = true;
                }
            }
        }
        if (needSplit) {
            if (this.getLoadBalancerAutoBundleSplitEnabled()) {
                log.info("Will split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
                bundlesToBeSplit.add(bundleName);
            } else {
                log.info("DRY RUN - split hot namespace bundle {}, topics {}, producers+consumers {}, msgRate in+out {}, bandwidth in+out {}", bundleName, stats.topics, totalSessions, totalMsgRate, totalBandwidth);
            }
        }
    }
    if (bundlesToBeSplit.size() > 0) {
        for (String bundleName : bundlesToBeSplit) {
            try {
                pulsar.getAdminClient().namespaces().splitNamespaceBundle(getNamespaceNameFromBundleName(bundleName), getBundleRangeFromBundleName(bundleName));
                log.info("Successfully split namespace bundle {}", bundleName);
            } catch (Exception e) {
                log.error("Failed to split namespace bundle {}", bundleName, e);
            }
        }
        this.setLoadReportForceUpdateFlag();
    }
}

Example 89 with NamespaceName

use of com.yahoo.pulsar.common.naming.NamespaceName in project pulsar by yahoo.

the class PulsarService method loadNamespaceDestinations.

/**
     * Load all the destination contained in a namespace
     *
     * @param bundle
     *            <code>NamespaceBundle</code> to identify the service unit
     * @throws Exception
     */
public void loadNamespaceDestinations(NamespaceBundle bundle) {
    executor.submit(() -> {
        LOG.info("Loading all topics on bundle: {}", bundle);
        NamespaceName nsName = bundle.getNamespaceObject();
        List<CompletableFuture<Topic>> persistentTopics = Lists.newArrayList();
        long topicLoadStart = System.nanoTime();
        for (String topic : getNamespaceService().getListOfDestinations(nsName.getProperty(), nsName.getCluster(), nsName.getLocalName())) {
            try {
                DestinationName dn = DestinationName.get(topic);
                if (bundle.includes(dn)) {
                    CompletableFuture<Topic> future = brokerService.getTopic(topic);
                    if (future != null) {
                        persistentTopics.add(future);
                    }
                }
            } catch (Throwable t) {
                LOG.warn("Failed to preload topic {}", topic, t);
            }
        }
        if (!persistentTopics.isEmpty()) {
            FutureUtil.waitForAll(persistentTopics).thenRun(() -> {
                double topicLoadTimeSeconds = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - topicLoadStart) / 1000.0;
                LOG.info("Loaded {} topics on {} -- time taken: {} seconds", persistentTopics.size(), bundle, topicLoadTimeSeconds);
            });
        }
        return null;
    });
}

Example 90 with NamespaceName

use of com.yahoo.pulsar.common.naming.NamespaceName in project pulsar by yahoo.

the class NamespacesTest method testDeleteNamespaceWithBundles.

@Test
public void testDeleteNamespaceWithBundles() throws Exception {
    URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
    String bundledNsLocal = "test-bundled-namespace-1";
    BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"));
    createBundledTestNamespaces(this.testProperty, this.testLocalCluster, bundledNsLocal, bundleData);
    final NamespaceName testNs = new NamespaceName(this.testProperty, this.testLocalCluster, bundledNsLocal);
    com.yahoo.pulsar.client.admin.Namespaces namespacesAdmin = mock(com.yahoo.pulsar.client.admin.Namespaces.class);
    doReturn(namespacesAdmin).when(admin).namespaces();
    doReturn(null).when(nsSvc).getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {

        @Override
        public void describeTo(Description description) {
        }

        @Override
        public boolean matches(Object item) {
            if (item instanceof NamespaceBundle) {
                NamespaceBundle bundle = (NamespaceBundle) item;
                return bundle.getNamespaceObject().equals(testNs);
            }
            return false;
        }

        @Override
        public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
        }
    }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
    doReturn(false).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {

        @Override
        public void describeTo(Description description) {
        }

        @Override
        public boolean matches(Object item) {
            if (item instanceof NamespaceBundle) {
                NamespaceBundle bundle = (NamespaceBundle) item;
                return bundle.getNamespaceObject().equals(testNs);
            }
            return false;
        }

        @Override
        public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
        }
    }));
    doReturn(Optional.of(new NamespaceEphemeralData())).when(nsSvc).getOwner(Mockito.argThat(new Matcher<NamespaceBundle>() {

        @Override
        public void describeTo(Description description) {
        }

        @Override
        public boolean matches(Object item) {
            if (item instanceof NamespaceBundle) {
                NamespaceBundle bundle = (NamespaceBundle) item;
                return bundle.getNamespaceObject().equals(testNs);
            }
            return false;
        }

        @Override
        public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
        }
    }));
    doThrow(new PulsarAdminException.PreconditionFailedException(new ClientErrorException(Status.PRECONDITION_FAILED))).when(namespacesAdmin).deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString());
    try {
        namespaces.deleteNamespaceBundle(testProperty, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000", false);
        fail("Should have failed");
    } catch (RestException re) {
        assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    try {
        namespaces.deleteNamespace(testProperty, testLocalCluster, bundledNsLocal, false);
        fail("Should have failed");
    } catch (RestException re) {
        assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData);
    // make one bundle owned
    doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(nsBundles.getBundles().get(0), false, true, false);
    doReturn(true).when(nsSvc).isServiceUnitOwned(nsBundles.getBundles().get(0));
    doNothing().when(namespacesAdmin).deleteNamespaceBundle(testProperty + "/" + testLocalCluster + "/" + bundledNsLocal, "0x00000000_0x80000000");
    try {
        namespaces.deleteNamespaceBundle(testProperty, testLocalCluster, bundledNsLocal, "0x80000000_0xffffffff", false);
        fail("Should have failed");
    } catch (RestException re) {
        assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    try {
        namespaces.deleteNamespace(testProperty, testLocalCluster, bundledNsLocal, false);
        fail("should have failed");
    } catch (RestException re) {
        assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
    }
    // ensure all three bundles are owned by the local broker
    for (NamespaceBundle bundle : nsBundles.getBundles()) {
        doReturn(localWebServiceUrl).when(nsSvc).getWebServiceUrl(bundle, false, true, false);
        doReturn(true).when(nsSvc).isServiceUnitOwned(bundle);
    }
    doNothing().when(namespacesAdmin).deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString());
}