Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 21 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ReplicatorTest method testDeleteReplicatorFailure.

/**
     * It verifies that: if it fails while removing replicator-cluster-cursor: it should not restart the replicator and
     * it should have cleaned up from the list
     * 
     * @throws Exception
     */
@Test
public void testDeleteReplicatorFailure() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final DestinationName dest = DestinationName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger();
    CountDownLatch latch = new CountDownLatch(1);
    // delete cursor already : so next time if topic.removeReplicator will get exception but then it should
    // remove-replicator from the list even with failure
    ledger.asyncDeleteCursor("pulsar.repl." + replicatorClusterName, new DeleteCursorCallback() {

        @Override
        public void deleteCursorComplete(Object ctx) {
            latch.countDown();
        }

        @Override
        public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
        }
    }, null);
    latch.await();
    Method removeReplicator = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeReplicator.setAccessible(true);
    // invoke removeReplicator : it fails as cursor is not present: but still it should remove the replicator from
    // list without restarting it
    CompletableFuture<Void> result = (CompletableFuture<Void>) removeReplicator.invoke(topic, replicatorClusterName);
    result.thenApply((v) -> {
        assertNull(topic.getPersistentReplicator(replicatorClusterName));
        return null;
    });
}

Example 22 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.

/**
     * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
     * EntryCache should be cleaned : Once active subscription consumes messages
     *
     * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
     * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
     * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
     * cleared
     *
     * @throws Exception
     */
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    /************ usecase-1: *************/
    // 1. Subscriber Faster subscriber
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
    EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
    cacheField.set(ledger, entryCache);
    Message msg = null;
    // 2. Produce messages
    for (int i = 0; i < 30; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages
    for (int i = 0; i < 30; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // Verify: EntryCache has been invalidated
    verify(entryCache, atLeastOnce()).invalidateEntries(any());
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
    assertTrue(entryCache.getSize() == 0);
    /************ usecase-2: *************/
    // 1.b Subscriber slower-subscriber
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    // Produce messages
    final int moreMessages = 10;
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // Consume messages
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
    assertTrue(entryCache.getSize() != 0);
    // 3.b Close subscriber2: which will trigger cache to clear the cache
    subscriber2.close();
    // Verify: EntryCache should be cleared
    assertTrue(entryCache.getSize() == 0);
    subscriber1.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 23 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.

/**
 * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
 * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
 * closed
 *
 * @throws Exception
 */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message<byte[]> msg;
    int recvQueueSize = 4;
    // (1) Create subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(recvQueueSize).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
    /**
     *********** Validation on non-empty active-cursor *************
     */
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());
    // (4.3) Validate: entryCache should have cached messages
    assertTrue(entryCache.getSize() != 0);
    /**
     *********** Validation on empty active-cursor *************
     */
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertTrue(entryCache.getSize() == 0);
}

Example 24 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testDeactivatingBacklogConsumer.

@Test
public void testDeactivatingBacklogConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    // 1. Subscriber Faster subscriber: let it consume all messages immediately
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    // 1.b. Subscriber Slow subscriber:
    conf.setReceiverQueueSize(receiverSize);
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    // reflection to set/get cache-backlog fields value:
    final long maxMessageCacheRetentionTimeMillis = 100;
    Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
    backlogThresholdField.setAccessible(true);
    Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
    field.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
    field.set(ledger, maxMessageCacheRetentionTimeMillis);
    final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
    Message msg = null;
    final int totalMsgs = (int) maxActiveCursorBacklogEntries + receiverSize + 1;
    // 2. Produce messages
    for (int i = 0; i < totalMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages: at Faster subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber1.receive(100, TimeUnit.MILLISECONDS);
        subscriber1.acknowledge(msg);
    }
    // wait : so message can be eligible to to be evict from cache
    Thread.sleep(maxMessageCacheRetentionTimeMillis);
    // 4. deactivate subscriber which has built the backlog
    ledger.checkBackloggedCursors();
    Thread.sleep(100);
    // 5. verify: active subscribers
    Set<String> activeSubscriber = Sets.newHashSet();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertFalse(activeSubscriber.contains(sub2));
    // 6. consume messages : at slower subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber2.receive(100, TimeUnit.MILLISECONDS);
        subscriber2.acknowledge(msg);
    }
    ledger.checkBackloggedCursors();
    activeSubscriber.clear();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertTrue(activeSubscriber.contains(sub2));
}

Example 25 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.

/**
 * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
 * EntryCache should be cleaned : Once active subscription consumes messages
 *
 * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
 * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
 * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
 * cleared
 *
 * @throws Exception
 */
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    /**
     ********** usecase-1: ************
     */
    // 1. Subscriber Faster subscriber
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
    EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
    cacheField.set(ledger, entryCache);
    Message msg = null;
    // 2. Produce messages
    for (int i = 0; i < 30; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages
    for (int i = 0; i < 30; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // Verify: EntryCache has been invalidated
    verify(entryCache, atLeastOnce()).invalidateEntries(any());
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    // 
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
    assertEquals(entryCache.getSize(), 0, 1);
    /**
     ********** usecase-2: ************
     */
    // 1.b Subscriber slower-subscriber
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    // Produce messages
    final int moreMessages = 10;
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // Consume messages
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    // 
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
    assertTrue(entryCache.getSize() != 0);
    // 3.b Close subscriber2: which will trigger cache to clear the cache
    subscriber2.close();
    // retry strategically until broker clean up closed subscribers and invalidate all cache entries
    retryStrategically((test) -> entryCache.getSize() == 0, 5, 100);
    // Verify: EntryCache should be cleared
    assertTrue(entryCache.getSize() == 0);
    subscriber1.close();
    log.info("-- Exiting {} test --", methodName);
}