Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 16 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class PersistentTopic method getInternalStats.

public PersistentTopicInternalStats getInternalStats() {
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp());
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp());
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    ml.getLedgersInfo().forEach((id, li) -> {
        LedgerInfo info = new LedgerInfo();
        info.ledgerId = li.getLedgerId();
        info.entries = li.getEntries();
        info.size = li.getSize();
        stats.ledgers.add(info);
    });
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp());
        cs.state = cursor.getState();
        cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage();
        cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange();
        cs.properties = cursor.getProperties();
        stats.cursors.put(cursor.getName(), cs);
    });
    return stats;
}

Example 17 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class AdminApiTest2 method testUpdatePersistencePolicyUpdateManagedCursor.

/**
 * validates update of persistent-policies reflects on managed-ledger and managed-cursor
 *
 * @throws Exception
 */
@Test
public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
    final String namespace = "prop-xyz/use/ns2";
    final String topicName = "persistent://" + namespace + "/topic1";
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 50.0));
    assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 50.0));
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.getCursors().iterator().next();
    final double newThrottleRate = 100;
    final int newEnsembleSize = 5;
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(newEnsembleSize, 3, 3, newThrottleRate));
    retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() == newEnsembleSize && cursor.getThrottleMarkDelete() != newThrottleRate, 5, 200);
    // (1) verify cursor.markDelete has been updated
    assertEquals(cursor.getThrottleMarkDelete(), newThrottleRate);
    // (2) verify new ledger creation takes new config
    producer.close();
    consumer.close();
}

Example 18 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class ManagedLedgerMetrics method aggregate.

/**
 * Aggregation by namespace (not thread-safe)
 *
 * @param ledgersByDimension
 * @return
 */
private List<Metrics> aggregate(Map<Metrics, List<ManagedLedgerImpl>> ledgersByDimension) {
    metricsCollection.clear();
    for (Entry<Metrics, List<ManagedLedgerImpl>> e : ledgersByDimension.entrySet()) {
        Metrics metrics = e.getKey();
        List<ManagedLedgerImpl> ledgers = e.getValue();
        // prepare aggregation map
        tempAggregatedMetricsMap.clear();
        for (ManagedLedgerImpl ledger : ledgers) {
            ManagedLedgerMXBean lStats = ledger.getStats();
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryBytesRate", lStats.getAddEntryBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryErrors", (double) lStats.getAddEntryErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryMessagesRate", lStats.getAddEntryMessagesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntrySucceed", (double) lStats.getAddEntrySucceed());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_NumberOfMessagesInBacklog", (double) lStats.getNumberOfMessagesInBacklog());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesBytesRate", lStats.getReadEntriesBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesErrors", (double) lStats.getReadEntriesErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesRate", lStats.getReadEntriesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesSucceeded", (double) lStats.getReadEntriesSucceeded());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_StoredMessagesSize", (double) lStats.getStoredMessagesSize());
            // handle bucket entries initialization here
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_AddEntryLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getAddEntryLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_LedgerSwitchLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getLedgerSwitchLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_EntrySizeBuckets", ENTRY_SIZE_BUCKETS_BYTES, lStats.getEntrySizeBuckets());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_MarkDeleteRate", lStats.getMarkDeleteRate());
        }
        for (Entry<String, Double> ma : tempAggregatedMetricsMap.entrySet()) {
            metrics.put(ma.getKey(), ma.getValue());
        }
        metricsCollection.add(metrics);
    }
    return metricsCollection;
}

Example 19 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        Message msg = MessageBuilder.create().setContent(content.getBytes()).build();
        final int index = i;
        producer.sendAsync(msg).thenRun(() -> {
            assertEquals(msg.getMessageId(), new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 20 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class PersistentTopic method getInternalStats.

public PersistentTopicInternalStats getInternalStats() {
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(ml.getLastLedgerCreatedTimestamp()));
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(ml.getLastLedgerCreationFailureTimestamp()));
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    ml.getLedgersInfo().forEach((id, li) -> {
        LedgerInfo info = new LedgerInfo();
        info.ledgerId = li.getLedgerId();
        info.entries = li.getEntries();
        info.size = li.getSize();
        stats.ledgers.add(info);
    });
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(cursor.getLastLedgerSwitchTimestamp()));
        cs.state = cursor.getState();
        stats.cursors.put(cursor.getName(), cs);
    });
    return stats;
}