Search in sources :

Example 1 with ConcurrentOpenHashMap

use of org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap in project incubator-pulsar by apache.

the class BrokerService method addTopicToStatsMaps.

private void addTopicToStatsMaps(TopicName topicName, Topic topic) {
    try {
        NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(topicName);
        if (namespaceBundle != null) {
            synchronized (multiLayerTopicsMap) {
                String serviceUnit = namespaceBundle.toString();
                // 
                multiLayerTopicsMap.computeIfAbsent(topicName.getNamespace(), // 
                k -> new ConcurrentOpenHashMap<>()).computeIfAbsent(serviceUnit, // 
                k -> new ConcurrentOpenHashMap<>()).put(topicName.toString(), topic);
            }
        }
        invalidateOfflineTopicStatCache(topicName);
    } catch (Exception e) {
        log.warn("Got exception when retrieving bundle name during create persistent topic", e);
    }
}
Also used : NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) CreateMode(org.apache.zookeeper.CreateMode) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) FieldContext(org.apache.pulsar.common.configuration.FieldContext) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) ZooKeeperDataCache(org.apache.pulsar.zookeeper.ZooKeeperDataCache) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) Set(java.util.Set) ZooKeeperCacheListener(org.apache.pulsar.zookeeper.ZooKeeperCacheListener) PooledByteBufAllocator(io.netty.buffer.PooledByteBufAllocator) Executors(java.util.concurrent.Executors) ZkUtils(org.apache.bookkeeper.util.ZkUtils) NamespaceBundleFactory(org.apache.pulsar.common.naming.NamespaceBundleFactory) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ClientBuilderImpl(org.apache.pulsar.client.impl.ClientBuilderImpl) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ChannelOption(io.netty.channel.ChannelOption) AuthenticationService(org.apache.pulsar.broker.authentication.AuthenticationService) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) EventType(org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect.EventType) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) Lists(com.google.common.collect.Lists) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PulsarClient(org.apache.pulsar.client.api.PulsarClient) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) PersistentDispatcherMultipleConsumers(org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) ClientCnxnAspect(org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect) PulsarWebResource(org.apache.pulsar.broker.web.PulsarWebResource) SslContext(io.netty.handler.ssl.SslContext) IOException(java.io.IOException) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) POLICIES(org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) Metrics(org.apache.pulsar.common.stats.Metrics) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) PersistentTopicStats(org.apache.pulsar.common.policies.data.PersistentTopicStats) Ids(org.apache.zookeeper.ZooDefs.Ids) AdminResource(org.apache.pulsar.broker.admin.AdminResource) ObjectMapperFactory(org.apache.pulsar.common.util.ObjectMapperFactory) LoggerFactory(org.slf4j.LoggerFactory) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) Stat(org.apache.zookeeper.data.Stat) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) URI(java.net.URI) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) EventLoopUtil(org.apache.pulsar.common.util.netty.EventLoopUtil) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) Predicate(java.util.function.Predicate) InetSocketAddress(java.net.InetSocketAddress) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Queues(com.google.common.collect.Queues) ClientConfigurationData(org.apache.pulsar.client.impl.conf.ClientConfigurationData) EventListner(org.apache.pulsar.broker.zookeeper.aspectj.ClientCnxnAspect.EventListner) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) ByteBuf(io.netty.buffer.ByteBuf) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) FieldParser(org.apache.pulsar.common.util.FieldParser) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) EventLoopGroup(io.netty.channel.EventLoopGroup) KeeperException(org.apache.zookeeper.KeeperException) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Maps(com.google.common.collect.Maps) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Policies(org.apache.pulsar.common.policies.data.Policies) Closeable(java.io.Closeable) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) Collections(java.util.Collections) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) IOException(java.io.IOException) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) KeeperException(org.apache.zookeeper.KeeperException) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException)

Example 2 with ConcurrentOpenHashMap

use of org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap in project incubator-pulsar by apache.

the class PersistentTopicTest method testMaxConsumersShared.

public void testMaxConsumersShared() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    PersistentSubscription sub2 = new PersistentSubscription(topic, "sub-2", cursorMock);
    // for count consumers on topic
    ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = new ConcurrentOpenHashMap<>(16, 1);
    subscriptions.put("sub-1", sub);
    subscriptions.put("sub-2", sub2);
    Field field = topic.getClass().getDeclaredField("subscriptions");
    field.setAccessible(true);
    field.set(topic, subscriptions);
    // 1. add consumer1
    Consumer consumer = new Consumer(sub, SubType.Shared, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub.addConsumer(consumer);
    assertEquals(sub.getConsumers().size(), 1);
    // 2. add consumer2
    Consumer consumer2 = new Consumer(sub, SubType.Shared, topic.getName(), 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub.addConsumer(consumer2);
    assertEquals(sub.getConsumers().size(), 2);
    // 3. add consumer3 but reach maxConsumersPerSubscription
    try {
        Consumer consumer3 = new Consumer(sub, SubType.Shared, topic.getName(), 3, /* consumer id */
        0, "Cons3", /* consumer name */
        50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
        InitialPosition.Latest);
        sub.addConsumer(consumer3);
        fail("should have failed");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
    }
    // check number of consumers on topic
    assertEquals(topic.getNumberOfConsumers(), 2);
    // 4. add consumer4 to sub2
    Consumer consumer4 = new Consumer(sub2, SubType.Shared, topic.getName(), 4, /* consumer id */
    0, "Cons4", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub2.addConsumer(consumer4);
    assertEquals(sub2.getConsumers().size(), 1);
    // check number of consumers on topic
    assertEquals(topic.getNumberOfConsumers(), 3);
    // 5. add consumer5 to sub2 but reach maxConsumersPerTopic
    try {
        Consumer consumer5 = new Consumer(sub2, SubType.Shared, topic.getName(), 5, /* consumer id */
        0, "Cons5", /* consumer name */
        50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
        InitialPosition.Latest);
        sub2.addConsumer(consumer5);
        fail("should have failed");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
    }
}
Also used : Field(java.lang.reflect.Field) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) PersistentDispatcherSingleActiveConsumer(org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) Matchers.anyString(org.mockito.Matchers.anyString) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription)

Example 3 with ConcurrentOpenHashMap

use of org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap in project incubator-pulsar by apache.

the class PersistentTopicTest method testMaxConsumersFailover.

public void testMaxConsumersFailover() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "sub-1", cursorMock);
    PersistentSubscription sub2 = new PersistentSubscription(topic, "sub-2", cursorMock);
    // for count consumers on topic
    ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = new ConcurrentOpenHashMap<>(16, 1);
    subscriptions.put("sub-1", sub);
    subscriptions.put("sub-2", sub2);
    Field field = topic.getClass().getDeclaredField("subscriptions");
    field.setAccessible(true);
    field.set(topic, subscriptions);
    // 1. add consumer1
    Consumer consumer = new Consumer(sub, SubType.Failover, topic.getName(), 1, /* consumer id */
    0, "Cons1", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub.addConsumer(consumer);
    assertEquals(sub.getConsumers().size(), 1);
    // 2. add consumer2
    Consumer consumer2 = new Consumer(sub, SubType.Failover, topic.getName(), 2, /* consumer id */
    0, "Cons2", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub.addConsumer(consumer2);
    assertEquals(sub.getConsumers().size(), 2);
    // 3. add consumer3 but reach maxConsumersPerSubscription
    try {
        Consumer consumer3 = new Consumer(sub, SubType.Failover, topic.getName(), 3, /* consumer id */
        0, "Cons3", /* consumer name */
        50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
        InitialPosition.Latest);
        sub.addConsumer(consumer3);
        fail("should have failed");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
    }
    // check number of consumers on topic
    assertEquals(topic.getNumberOfConsumers(), 2);
    // 4. add consumer4 to sub2
    Consumer consumer4 = new Consumer(sub2, SubType.Failover, topic.getName(), 4, /* consumer id */
    0, "Cons4", /* consumer name */
    50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
    InitialPosition.Latest);
    sub2.addConsumer(consumer4);
    assertEquals(sub2.getConsumers().size(), 1);
    // check number of consumers on topic
    assertEquals(topic.getNumberOfConsumers(), 3);
    // 5. add consumer5 to sub2 but reach maxConsumersPerTopic
    try {
        Consumer consumer5 = new Consumer(sub2, SubType.Failover, topic.getName(), 5, /* consumer id */
        0, "Cons5", /* consumer name */
        50000, serverCnx, "myrole-1", Collections.emptyMap(), false, /* read compacted */
        InitialPosition.Latest);
        sub2.addConsumer(consumer5);
        fail("should have failed");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.ConsumerBusyException);
    }
}
Also used : Field(java.lang.reflect.Field) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) PersistentDispatcherSingleActiveConsumer(org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) Matchers.anyString(org.mockito.Matchers.anyString) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription)

Example 4 with ConcurrentOpenHashMap

use of org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap in project incubator-pulsar by apache.

the class ReplicatorTest method testConcurrentReplicator.

@SuppressWarnings("unchecked")
@Test(timeOut = 30000)
public void testConcurrentReplicator() throws Exception {
    log.info("--- Starting ReplicatorTest::testConcurrentReplicator ---");
    final String namespace = "pulsar/global/concurrent";
    admin1.namespaces().createNamespace(namespace);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2"));
    final TopicName topicName = TopicName.get(String.format("persistent://" + namespace + "/topic-%d", 0));
    PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Producer<byte[]> producer = client1.newProducer().topic(topicName.toString()).create();
    producer.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName.toString()).get();
    PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService().getReplicationClient("r3"));
    final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class);
    startRepl.setAccessible(true);
    Field replClientField = BrokerService.class.getDeclaredField("replicationClients");
    replClientField.setAccessible(true);
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients = (ConcurrentOpenHashMap<String, PulsarClient>) replClientField.get(pulsar1.getBrokerService());
    replicationClients.put("r3", pulsarClient);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2", "r3"));
    ExecutorService executor = Executors.newFixedThreadPool(5);
    for (int i = 0; i < 5; i++) {
        executor.submit(() -> {
            try {
                startRepl.invoke(topic, "r3");
            } catch (Exception e) {
                fail("setting replicator failed", e);
            }
        });
    }
    Thread.sleep(3000);
    Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.any(ProducerConfigurationData.class), Mockito.any(Schema.class));
    client1.shutdown();
}
Also used : ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) Schema(org.apache.pulsar.client.api.Schema) Method(java.lang.reflect.Method) BeforeMethod(org.testng.annotations.BeforeMethod) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) PreconditionFailedException(org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CursorAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException) TopicName(org.apache.pulsar.common.naming.TopicName) ProducerConfigurationData(org.apache.pulsar.client.impl.conf.ProducerConfigurationData) Field(java.lang.reflect.Field) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) ExecutorService(java.util.concurrent.ExecutorService) PulsarClient(org.apache.pulsar.client.api.PulsarClient) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) Test(org.testng.annotations.Test)

Example 5 with ConcurrentOpenHashMap

use of org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap in project incubator-pulsar by apache.

the class ConcurrentOpenHashMapTest method concurrentInsertionsAndReads.

@Test
public void concurrentInsertionsAndReads() throws Throwable {
    ConcurrentOpenHashMap<Long, String> map = new ConcurrentOpenHashMap<>();
    ExecutorService executor = Executors.newCachedThreadPool();
    final int nThreads = 16;
    final int N = 100_000;
    String value = "value";
    List<Future<?>> futures = new ArrayList<>();
    for (int i = 0; i < nThreads; i++) {
        final int threadIdx = i;
        futures.add(executor.submit(() -> {
            Random random = new Random();
            for (int j = 0; j < N; j++) {
                long key = random.nextLong();
                // Ensure keys are uniques
                key -= key % (threadIdx + 1);
                map.put(key, value);
            }
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
    assertEquals(map.size(), N * nThreads);
    executor.shutdown();
}
Also used : ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) Random(java.util.Random) ExecutorService(java.util.concurrent.ExecutorService) ArrayList(java.util.ArrayList) Future(java.util.concurrent.Future) Test(org.testng.annotations.Test)

Aggregations

ConcurrentOpenHashMap (org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap)9 PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)5 Field (java.lang.reflect.Field)4 ExecutorService (java.util.concurrent.ExecutorService)4 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)4 Test (org.testng.annotations.Test)4 ArrayList (java.util.ArrayList)3 List (java.util.List)3 Random (java.util.Random)3 NamingException (org.apache.pulsar.broker.service.BrokerServiceException.NamingException)3 TopicName (org.apache.pulsar.common.naming.TopicName)3 Lists (com.google.common.collect.Lists)2 IOException (java.io.IOException)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2 Future (java.util.concurrent.Future)2 PersistentDispatcherSingleActiveConsumer (org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer)2 PersistentSubscription (org.apache.pulsar.broker.service.persistent.PersistentSubscription)2 PulsarClient (org.apache.pulsar.client.api.PulsarClient)2 PulsarClientImpl (org.apache.pulsar.client.impl.PulsarClientImpl)2