Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 6 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ManagedLedgerMetricsTest method testManagedLedgerMetrics.

@Test
public void testManagedLedgerMetrics() throws Exception {
    ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar);
    final String addEntryRateKey = "brk_ml_AddEntryMessagesRate";
    List<Metrics> list1 = metrics.generate();
    Assert.assertTrue(list1.isEmpty());
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list2 = metrics.generate();
    Assert.assertEquals(list2.get(0).getMetrics().get(addEntryRateKey), 10.0D);
    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list3 = metrics.generate();
    Assert.assertEquals(list3.get(0).getMetrics().get(addEntryRateKey), 5.0D);
}

Example 7 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        Message msg = MessageBuilder.create().setContent(content.getBytes()).build();
        final int index = i;
        producer.sendAsync(msg).thenRun(() -> {
            assertEquals(msg.getMessageId(), new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 8 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class PersistentTopic method getInternalStats.

public PersistentTopicInternalStats getInternalStats() {
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(ml.getLastLedgerCreatedTimestamp()));
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(ml.getLastLedgerCreationFailureTimestamp()));
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    ml.getLedgersInfo().forEach((id, li) -> {
        LedgerInfo info = new LedgerInfo();
        info.ledgerId = li.getLedgerId();
        info.entries = li.getEntries();
        info.size = li.getSize();
        stats.ledgers.add(info);
    });
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(cursor.getLastLedgerSwitchTimestamp()));
        cs.state = cursor.getState();
        stats.cursors.put(cursor.getName(), cs);
    });
    return stats;
}

Example 9 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ReplicatorTest method testDeleteReplicatorFailure.

/**
     * It verifies that: if it fails while removing replicator-cluster-cursor: it should not restart the replicator and
     * it should have cleaned up from the list
     * 
     * @throws Exception
     */
@Test
public void testDeleteReplicatorFailure() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final DestinationName dest = DestinationName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topic.getManagedLedger();
    CountDownLatch latch = new CountDownLatch(1);
    // delete cursor already : so next time if topic.removeReplicator will get exception but then it should
    // remove-replicator from the list even with failure
    ledger.asyncDeleteCursor("pulsar.repl." + replicatorClusterName, new DeleteCursorCallback() {

        @Override
        public void deleteCursorComplete(Object ctx) {
            latch.countDown();
        }

        @Override
        public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
        }
    }, null);
    latch.await();
    Method removeReplicator = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeReplicator.setAccessible(true);
    // invoke removeReplicator : it fails as cursor is not present: but still it should remove the replicator from
    // list without restarting it
    CompletableFuture<Void> result = (CompletableFuture<Void>) removeReplicator.invoke(topic, replicatorClusterName);
    result.thenApply((v) -> {
        assertNull(topic.getPersistentReplicator(replicatorClusterName));
        return null;
    });
}

Example 10 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.

/**
     * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
     * EntryCache should be cleaned : Once active subscription consumes messages
     *
     * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
     * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
     * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
     * cleared
     *
     * @throws Exception
     */
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    /************ usecase-1: *************/
    // 1. Subscriber Faster subscriber
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
    EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
    cacheField.set(ledger, entryCache);
    Message msg = null;
    // 2. Produce messages
    for (int i = 0; i < 30; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages
    for (int i = 0; i < 30; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // Verify: EntryCache has been invalidated
    verify(entryCache, atLeastOnce()).invalidateEntries(any());
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
    assertTrue(entryCache.getSize() == 0);
    /************ usecase-2: *************/
    // 1.b Subscriber slower-subscriber
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    // Produce messages
    final int moreMessages = 10;
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // Consume messages
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
    assertTrue(entryCache.getSize() != 0);
    // 3.b Close subscriber2: which will trigger cache to clear the cache
    subscriber2.close();
    // Verify: EntryCache should be cleared
    assertTrue(entryCache.getSize() == 0);
    subscriber1.close();
    log.info("-- Exiting {} test --", methodName);
}