Search in sources :

Example 1 with BrokerService

use of com.yahoo.pulsar.broker.service.BrokerService in project pulsar by yahoo.

the class PersistentTopic method startReplicator.

CompletableFuture<Void> startReplicator(String remoteCluster) {
    log.info("[{}] Starting replicator to remote: {}", topic, remoteCluster);
    final CompletableFuture<Void> future = new CompletableFuture<>();
    String name = PersistentReplicator.getReplicatorName(replicatorPrefix, remoteCluster);
    ledger.asyncOpenCursor(name, new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            String localCluster = brokerService.pulsar().getConfiguration().getClusterName();
            replicators.computeIfAbsent(remoteCluster, r -> new PersistentReplicator(PersistentTopic.this, cursor, localCluster, remoteCluster, brokerService));
            future.complete(null);
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}
Also used : ReplicationMetrics(com.yahoo.pulsar.broker.stats.ReplicationMetrics) SubType(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) PersistentSubscriptionStats(com.yahoo.pulsar.common.policies.data.PersistentSubscriptionStats) LedgerInfo(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo) NamingException(com.yahoo.pulsar.broker.service.BrokerServiceException.NamingException) ConsumerStats(com.yahoo.pulsar.common.policies.data.ConsumerStats) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) PersistentTopicInternalStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats) LoggerFactory(org.slf4j.LoggerFactory) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) NamespaceBundleStats(com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats) Policies(com.yahoo.pulsar.common.policies.data.Policies) BacklogQuota(com.yahoo.pulsar.common.policies.data.BacklogQuota) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) TopicBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicBusyException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ReplicatorStats(com.yahoo.pulsar.common.policies.data.ReplicatorStats) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) Objects(com.google.common.base.Objects) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) CursorStats(com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) Instant(java.time.Instant) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) TopicFencedException(com.yahoo.pulsar.broker.service.BrokerServiceException.TopicFencedException) Lists(com.beust.jcommander.internal.Lists) ZoneId(java.time.ZoneId) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Topic(com.yahoo.pulsar.broker.service.Topic) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) Consumer(com.yahoo.pulsar.broker.service.Consumer) List(java.util.List) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) AdminResource(com.yahoo.pulsar.broker.admin.AdminResource) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) StatsOutputStream(com.yahoo.pulsar.utils.StatsOutputStream) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) UnsupportedVersionException(com.yahoo.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) SubscriptionBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) MessageImpl(com.yahoo.pulsar.client.impl.MessageImpl) PersistentTopicStats(com.yahoo.pulsar.common.policies.data.PersistentTopicStats) ServerCnx(com.yahoo.pulsar.broker.service.ServerCnx) ByteBuf(io.netty.buffer.ByteBuf) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) ConcurrentOpenHashSet(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet) ConsumerBusyException(com.yahoo.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ConcurrentOpenHashMap(com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap) Codec(com.yahoo.pulsar.common.util.Codec) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) ClusterReplicationMetrics(com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics) Logger(org.slf4j.Logger) KeeperException(org.apache.zookeeper.KeeperException) BrokerServiceException(com.yahoo.pulsar.broker.service.BrokerServiceException) PublisherStats(com.yahoo.pulsar.common.policies.data.PublisherStats) Producer(com.yahoo.pulsar.broker.service.Producer) ServerMetadataException(com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) NamespaceStats(com.yahoo.pulsar.broker.stats.NamespaceStats) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) DateTimeFormatter(java.time.format.DateTimeFormatter) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistenceException(com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor)

Example 2 with BrokerService

use of com.yahoo.pulsar.broker.service.BrokerService in project pulsar by yahoo.

the class ApiVersionFilterTest method testAuthFilterTest.

@Test
public void testAuthFilterTest() throws Exception {
    BrokerService nativeService = mock(BrokerService.class);
    AuthenticationService authService = mock(AuthenticationService.class);
    doReturn(nativeService).when(pulsar).getBrokerService();
    doReturn(authService).when(nativeService).getAuthenticationService();
    doReturn("test-role").when(authService).authenticateHttpRequest(mock(HttpServletRequest.class));
    AuthenticationFilter filter = new AuthenticationFilter(pulsar);
    HttpServletRequest request = mock(HttpServletRequest.class);
    ServletResponse response = null;
    doNothing().when(request).setAttribute(anyString(), anyObject());
    filter.doFilter(request, response, chain);
}
Also used : HttpServletRequest(javax.servlet.http.HttpServletRequest) HttpServletResponse(javax.servlet.http.HttpServletResponse) ServletResponse(javax.servlet.ServletResponse) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) AuthenticationService(com.yahoo.pulsar.broker.authentication.AuthenticationService) Test(org.testng.annotations.Test)

Example 3 with BrokerService

use of com.yahoo.pulsar.broker.service.BrokerService in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method setup.

@BeforeMethod
public void setup(Method m) throws Exception {
    super.setUp(m);
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    PulsarService pulsar = spy(new PulsarService(svcConfig));
    doReturn(svcConfig).when(pulsar).getConfiguration();
    mlFactoryMock = mock(ManagedLedgerFactory.class);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
    final ManagedCursor cursor = ledger.openCursor("c1");
    cursorMock = cursor;
    ledgerMock = ledger;
    mlFactoryMock = factory;
    doReturn(mlFactoryMock).when(pulsar).getManagedLedgerFactory();
    ZooKeeper mockZk = mock(ZooKeeper.class);
    doReturn(mockZk).when(pulsar).getZkClient();
    brokerService = spy(new BrokerService(pulsar));
    doReturn(brokerService).when(pulsar).getBrokerService();
    serverCnx = spy(new ServerCnx(brokerService));
    doReturn(true).when(serverCnx).isActive();
    NamespaceService nsSvc = mock(NamespaceService.class);
    doReturn(nsSvc).when(pulsar).getNamespaceService();
    doReturn(true).when(nsSvc).isServiceUnitOwned(any(NamespaceBundle.class));
    doReturn(true).when(nsSvc).isServiceUnitActive(any(DestinationName.class));
    final List<Position> addedEntries = Lists.newArrayList();
    for (int i = 0; i < 100; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
}
Also used : NamespaceBundle(com.yahoo.pulsar.common.naming.NamespaceBundle) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ServerCnx(com.yahoo.pulsar.broker.service.ServerCnx) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ZooKeeper(org.apache.zookeeper.ZooKeeper) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) PulsarService(com.yahoo.pulsar.broker.PulsarService) NamespaceService(com.yahoo.pulsar.broker.namespace.NamespaceService) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) DestinationName(com.yahoo.pulsar.common.naming.DestinationName) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) ManagedLedgerFactoryImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 4 with BrokerService

use of com.yahoo.pulsar.broker.service.BrokerService in project pulsar by yahoo.

the class HttpDestinationLookupv2Test method setUp.

@SuppressWarnings("unchecked")
@BeforeMethod
public void setUp() throws Exception {
    pulsar = mock(PulsarService.class);
    ns = mock(NamespaceService.class);
    auth = mock(AuthorizationManager.class);
    mockConfigCache = mock(ConfigurationCacheService.class);
    clustersListCache = mock(ZooKeeperChildrenCache.class);
    clustersCache = mock(ZooKeeperDataCache.class);
    policiesCache = mock(ZooKeeperDataCache.class);
    config = spy(new ServiceConfiguration());
    config.setClusterName("use");
    clusters = new TreeSet<String>();
    clusters.add("use");
    clusters.add("usc");
    clusters.add("usw");
    ClusterData useData = new ClusterData("http://broker.messaging.use.example.com:8080");
    ClusterData uscData = new ClusterData("http://broker.messaging.usc.example.com:8080");
    ClusterData uswData = new ClusterData("http://broker.messaging.usw.example.com:8080");
    doReturn(config).when(pulsar).getConfiguration();
    doReturn(mockConfigCache).when(pulsar).getConfigurationCache();
    doReturn(clustersListCache).when(mockConfigCache).clustersListCache();
    doReturn(clustersCache).when(mockConfigCache).clustersCache();
    doReturn(policiesCache).when(mockConfigCache).policiesCache();
    doReturn(Optional.of(useData)).when(clustersCache).get(AdminResource.path("clusters", "use"));
    doReturn(Optional.of(uscData)).when(clustersCache).get(AdminResource.path("clusters", "usc"));
    doReturn(Optional.of(uswData)).when(clustersCache).get(AdminResource.path("clusters", "usw"));
    doReturn(CompletableFuture.completedFuture(Optional.of(useData))).when(clustersCache).getAsync(AdminResource.path("clusters", "use"));
    doReturn(CompletableFuture.completedFuture(Optional.of(uscData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usc"));
    doReturn(CompletableFuture.completedFuture(Optional.of(uswData))).when(clustersCache).getAsync(AdminResource.path("clusters", "usw"));
    doReturn(clusters).when(clustersListCache).get();
    doReturn(ns).when(pulsar).getNamespaceService();
    BrokerService brokerService = mock(BrokerService.class);
    doReturn(brokerService).when(pulsar).getBrokerService();
    doReturn(auth).when(brokerService).getAuthorizationManager();
    doReturn(new Semaphore(1000)).when(brokerService).getLookupRequestSemaphore();
}
Also used : ClusterData(com.yahoo.pulsar.common.policies.data.ClusterData) PulsarService(com.yahoo.pulsar.broker.PulsarService) NamespaceService(com.yahoo.pulsar.broker.namespace.NamespaceService) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) ConfigurationCacheService(com.yahoo.pulsar.broker.cache.ConfigurationCacheService) ZooKeeperChildrenCache(com.yahoo.pulsar.zookeeper.ZooKeeperChildrenCache) AuthorizationManager(com.yahoo.pulsar.broker.authorization.AuthorizationManager) ZooKeeperDataCache(com.yahoo.pulsar.zookeeper.ZooKeeperDataCache) Semaphore(java.util.concurrent.Semaphore) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 5 with BrokerService

use of com.yahoo.pulsar.broker.service.BrokerService in project pulsar by yahoo.

the class OwnershipCacheTest method setup.

@BeforeMethod
public void setup() throws Exception {
    final int port = 8080;
    selfBrokerUrl = "tcp://localhost:" + port;
    pulsar = mock(PulsarService.class);
    config = mock(ServiceConfiguration.class);
    executor = new OrderedSafeExecutor(1, "test");
    zkCache = new LocalZooKeeperCache(MockZooKeeper.newInstance(), executor);
    localCache = new LocalZooKeeperCacheService(zkCache, null);
    bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
    nsService = mock(NamespaceService.class);
    brokerService = mock(BrokerService.class);
    doReturn(CompletableFuture.completedFuture(1)).when(brokerService).unloadServiceUnit(anyObject());
    doReturn(zkCache).when(pulsar).getLocalZkCache();
    doReturn(localCache).when(pulsar).getLocalZkCacheService();
    doReturn(config).when(pulsar).getConfiguration();
    doReturn(nsService).when(pulsar).getNamespaceService();
    doReturn(port).when(config).getBrokerServicePort();
    doReturn(brokerService).when(pulsar).getBrokerService();
    doReturn(webAddress(config)).when(pulsar).getWebServiceAddress();
    doReturn(selfBrokerUrl).when(pulsar).getBrokerServiceUrl();
}
Also used : PulsarService(com.yahoo.pulsar.broker.PulsarService) ServiceConfiguration(com.yahoo.pulsar.broker.ServiceConfiguration) LocalZooKeeperCache(com.yahoo.pulsar.zookeeper.LocalZooKeeperCache) OrderedSafeExecutor(org.apache.bookkeeper.util.OrderedSafeExecutor) NamespaceBundleFactory(com.yahoo.pulsar.common.naming.NamespaceBundleFactory) BrokerService(com.yahoo.pulsar.broker.service.BrokerService) LocalZooKeeperCacheService(com.yahoo.pulsar.broker.cache.LocalZooKeeperCacheService) BeforeMethod(org.testng.annotations.BeforeMethod)

Aggregations

BrokerService (com.yahoo.pulsar.broker.service.BrokerService)8 PulsarService (com.yahoo.pulsar.broker.PulsarService)4 ServiceConfiguration (com.yahoo.pulsar.broker.ServiceConfiguration)4 BeforeMethod (org.testng.annotations.BeforeMethod)4 NamespaceService (com.yahoo.pulsar.broker.namespace.NamespaceService)3 ServerCnx (com.yahoo.pulsar.broker.service.ServerCnx)3 DestinationName (com.yahoo.pulsar.common.naming.DestinationName)3 ConfigurationCacheService (com.yahoo.pulsar.broker.cache.ConfigurationCacheService)2 NamespaceBundle (com.yahoo.pulsar.common.naming.NamespaceBundle)2 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)2 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)2 Position (org.apache.bookkeeper.mledger.Position)2 Lists (com.beust.jcommander.internal.Lists)1 ObjectObjectHashMap (com.carrotsearch.hppc.ObjectObjectHashMap)1 Objects (com.google.common.base.Objects)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Maps (com.google.common.collect.Maps)1 Sets (com.google.common.collect.Sets)1 AdminResource (com.yahoo.pulsar.broker.admin.AdminResource)1 AuthenticationService (com.yahoo.pulsar.broker.authentication.AuthenticationService)1