Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 11 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class SimpleProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.

/**
 * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
 * EntryCache should be cleaned : Once active subscription consumes messages
 *
 * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
 * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
 * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
 * cleared
 *
 * @throws Exception
 */
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    /**
     ********** usecase-1: ************
     */
    // 1. Subscriber Faster subscriber
    Consumer<byte[]> subscriber1 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub1).subscriptionType(SubscriptionType.Shared).receiverQueueSize(receiverSize).subscribe();
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topic);
    if (batchMessageDelayMs != 0) {
        producerBuilder.batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerBuilder.batchingMaxMessages(5);
        producerBuilder.enableBatching(true);
    }
    Producer<byte[]> producer = producerBuilder.create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
    EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
    cacheField.set(ledger, entryCache);
    Message<byte[]> msg = null;
    // 2. Produce messages
    for (int i = 0; i < 30; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages
    for (int i = 0; i < 30; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // Verify: EntryCache has been invalidated
    verify(entryCache, atLeastOnce()).invalidateEntries(any());
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    // 
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
    assertEquals(entryCache.getSize(), 0, 1);
    /**
     ********** usecase-2: ************
     */
    // 1.b Subscriber slower-subscriber
    Consumer<byte[]> subscriber2 = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/" + topicName).subscriptionName(sub2).subscribe();
    // Produce messages
    final int moreMessages = 10;
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // Consume messages
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    // 
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
    assertTrue(entryCache.getSize() != 0);
    // 3.b Close subscriber2: which will trigger cache to clear the cache
    subscriber2.close();
    // retry strategically until broker clean up closed subscribers and invalidate all cache entries
    retryStrategically((test) -> entryCache.getSize() == 0, 5, 100);
    // Verify: EntryCache should be cleared
    assertTrue(entryCache.getSize() == 0);
    subscriber1.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 12 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testClusterMsgByteRateLimitingClusterConfig.

/**
 * It verifies that dispatch-rate throttling with cluster-configuration
 *
 * @param subscription
 * @param dispatchRateType
 * @throws Exception
 */
@Test()
public void testClusterMsgByteRateLimitingClusterConfig() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 5;
    // 1MB rate enough to let all msg to be delivered
    final long byteRate = 1024 * 1024;
    int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
    // (1) Update message-dispatch-rate limit
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate));
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInByte", Long.toString(byteRate));
    // sleep incrementally as zk-watch notification is async and may take some time
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), initValue);
    admin.namespaces().createNamespace(namespace);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numMessages);
    consumer.close();
    producer.close();
    pulsar.getConfiguration().setDispatchThrottlingRatePerTopicInMsg(initValue);
    log.info("-- Exiting {} test --", methodName);
}

Example 13 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateLimitingNotReceiveAllMessages.

/**
 * verify: consumer should not receive all messages due to message-rate throttling
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptionAndDispatchRateType", timeOut = 5000)
public void testMessageRateLimitingNotReceiveAllMessages(SubscriptionType subscription, DispatchRateType dispatchRateType) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 100;
    DispatchRate dispatchRate = null;
    if (DispatchRateType.messageRate.equals(dispatchRateType)) {
        dispatchRate = new DispatchRate(messageRate, -1, 360);
    } else {
        dispatchRate = new DispatchRate(-1, messageRate, 360);
    }
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // consumer should not have received all publihsed message due to message-rate throttling
    Assert.assertTrue(totalReceived.get() < messageRate * 2);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 14 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class BrokerBkEnsemblesTests method testSkipCorruptDataLedger.

/**
 * It verifies broker-configuration using which broker can skip non-recoverable data-ledgers.
 *
 * <pre>
 * 1. publish messages in 5 data-ledgers each with 20 entries under managed-ledger
 * 2. delete first 4 data-ledgers
 * 3. consumer will fail to consume any message as first data-ledger is non-recoverable
 * 4. enable dynamic config to skip non-recoverable data-ledgers
 * 5. consumer will be able to consume 20 messages from last non-deleted ledger
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 6000)
public void testSkipCorruptDataLedger() throws Exception {
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String ns1 = "prop/usc/crash-broker";
    final int totalMessages = 100;
    final int totalDataLedgers = 5;
    final int entriesPerLedger = totalMessages / totalDataLedgers;
    admin.namespaces().createNamespace(ns1);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    // Create subscription
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").receiverQueueSize(5).subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next();
    Field configField = ManagedCursorImpl.class.getDeclaredField("config");
    configField.setAccessible(true);
    // Create multiple data-ledger
    ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor);
    config.setMaxEntriesPerLedger(entriesPerLedger);
    config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
    // bookkeeper client
    Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper");
    bookKeeperField.setAccessible(true);
    // Create multiple data-ledger
    BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml);
    // (1) publish messages in 5 data-ledgers each with 20 entries under managed-ledger
    Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // validate: consumer is able to consume msg and close consumer after reading 1 entry
    Assert.assertNotNull(consumer.receive(1, TimeUnit.SECONDS));
    consumer.close();
    NavigableMap<Long, LedgerInfo> ledgerInfo = ml.getLedgersInfo();
    Assert.assertEquals(ledgerInfo.size(), totalDataLedgers);
    Entry<Long, LedgerInfo> lastLedger = ledgerInfo.lastEntry();
    // (2) delete first 4 data-ledgers
    ledgerInfo.entrySet().forEach(entry -> {
        if (!entry.equals(lastLedger)) {
            try {
                bookKeeper.deleteLedger(entry.getKey());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    // clean managed-ledger and recreate topic to clean any data from the cache
    producer.close();
    pulsar.getBrokerService().removeTopicFromCache(topic1);
    ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(factory);
    ledgers.clear();
    // (3) consumer will fail to consume any message as first data-ledger is non-recoverable
    Message<byte[]> msg = null;
    // start consuming message
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    msg = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    consumer.close();
    // (4) enable dynamic config to skip non-recoverable data-ledgers
    admin.brokers().updateDynamicConfiguration("autoSkipNonRecoverableData", "true");
    retryStrategically((test) -> config.isAutoSkipNonRecoverableData(), 5, 100);
    // (5) consumer will be able to consume 20 messages from last non-deleted ledger
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    for (int i = 0; i < entriesPerLedger; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        System.out.println(i);
        consumer.acknowledge(msg);
    }
    producer.close();
    consumer.close();
    client.close();
}

Example 15 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class ManagedLedgerMetricsTest method testManagedLedgerMetrics.

@Test
public void testManagedLedgerMetrics() throws Exception {
    ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar);
    final String addEntryRateKey = "brk_ml_AddEntryMessagesRate";
    List<Metrics> list1 = metrics.generate();
    Assert.assertTrue(list1.isEmpty());
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list2 = metrics.generate();
    Assert.assertEquals(list2.get(0).getMetrics().get(addEntryRateKey), 10.0D);
    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list3 = metrics.generate();
    Assert.assertEquals(list3.get(0).getMetrics().get(addEntryRateKey), 5.0D);
}