Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 1 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ManagedLedgerMetrics method groupLedgersByDimension.

/**
     * Build a map of dimensions key to list of destination stats (not thread-safe)
     * <p>
     * 
     * @return
     */
private Map<Metrics, List<ManagedLedgerImpl>> groupLedgersByDimension() {
    ledgersByDimensionMap.clear();
    for (Entry<String, ManagedLedgerImpl> e : getManagedLedgers().entrySet()) {
        String ledgerName = e.getKey();
        ManagedLedgerImpl ledger = e.getValue();
        // we want to aggregate by NS dimension
        String namespace = parseNamespaceFromLedgerName(ledgerName);
        Metrics metrics = createMetricsByDimension(namespace);
        populateDimensionMap(ledgersByDimensionMap, metrics, ledger);
    }
    return ledgersByDimensionMap;
}

Example 2 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ManagedLedgerMetrics method aggregate.

/**
     * Aggregation by namespace (not thread-safe)
     * 
     * @param ledgersByDimension
     * @return
     */
private List<Metrics> aggregate(Map<Metrics, List<ManagedLedgerImpl>> ledgersByDimension) {
    metricsCollection.clear();
    for (Entry<Metrics, List<ManagedLedgerImpl>> e : ledgersByDimension.entrySet()) {
        Metrics metrics = e.getKey();
        List<ManagedLedgerImpl> ledgers = e.getValue();
        // prepare aggregation map
        tempAggregatedMetricsMap.clear();
        for (ManagedLedgerImpl ledger : ledgers) {
            ManagedLedgerMXBean lStats = ledger.getStats();
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryBytesRate", lStats.getAddEntryBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryErrors", (double) lStats.getAddEntryErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntryMessagesRate", lStats.getAddEntryMessagesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_AddEntrySucceed", (double) lStats.getAddEntrySucceed());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_NumberOfMessagesInBacklog", (double) lStats.getNumberOfMessagesInBacklog());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesBytesRate", lStats.getReadEntriesBytesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesErrors", (double) lStats.getReadEntriesErrors());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesRate", lStats.getReadEntriesRate());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_ReadEntriesSucceeded", (double) lStats.getReadEntriesSucceeded());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_StoredMessagesSize", (double) lStats.getStoredMessagesSize());
            // handle bucket entries initialization here
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_AddEntryLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getAddEntryLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_LedgerSwitchLatencyBuckets", ENTRY_LATENCY_BUCKETS_MS, lStats.getLedgerSwitchLatencyBuckets());
            populateBucketEntries(tempAggregatedMetricsMap, "brk_ml_EntrySizeBuckets", ENTRY_SIZE_BUCKETS_BYTES, lStats.getEntrySizeBuckets());
            populateAggregationMapWithSum(tempAggregatedMetricsMap, "brk_ml_MarkDeleteRate", lStats.getMarkDeleteRate());
        }
        for (Entry<String, Double> ma : tempAggregatedMetricsMap.entrySet()) {
            metrics.put(ma.getKey(), ma.getValue());
        }
        metricsCollection.add(metrics);
    }
    return metricsCollection;
}

Example 3 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testDeactivatingBacklogConsumer.

@Test
public void testDeactivatingBacklogConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    // 1. Subscriber Faster subscriber: let it consume all messages immediately
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    // 1.b. Subscriber Slow subscriber:
    conf.setReceiverQueueSize(receiverSize);
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    // reflection to set/get cache-backlog fields value:
    final long maxMessageCacheRetentionTimeMillis = 100;
    Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
    backlogThresholdField.setAccessible(true);
    Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
    field.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
    field.set(ledger, maxMessageCacheRetentionTimeMillis);
    final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
    Message msg = null;
    final int totalMsgs = (int) maxActiveCursorBacklogEntries + receiverSize + 1;
    // 2. Produce messages
    for (int i = 0; i < totalMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages: at Faster subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber1.receive(100, TimeUnit.MILLISECONDS);
        subscriber1.acknowledge(msg);
    }
    // wait : so message can be eligible to to be evict from cache
    Thread.sleep(maxMessageCacheRetentionTimeMillis);
    // 4. deactivate subscriber which has built the backlog
    ledger.checkBackloggedCursors();
    Thread.sleep(100);
    // 5. verify: active subscribers
    Set<String> activeSubscriber = Sets.newHashSet();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertFalse(activeSubscriber.contains(sub2));
    // 6. consume messages : at slower subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber2.receive(100, TimeUnit.MILLISECONDS);
        subscriber2.acknowledge(msg);
    }
    ledger.checkBackloggedCursors();
    activeSubscriber.clear();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertTrue(activeSubscriber.contains(sub2));
}

Example 4 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class ManagedLedgerMetricsTest method testManagedLedgerMetrics.

@Test
public void testManagedLedgerMetrics() throws Exception {
    ManagedLedgerMetrics metrics = new ManagedLedgerMetrics(pulsar);
    final String addEntryRateKey = "brk_ml_AddEntryMessagesRate";
    List<Metrics> list1 = metrics.generate();
    Assert.assertTrue(list1.isEmpty());
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1");
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list2 = metrics.generate();
    Assert.assertEquals(list2.get(0).getMetrics().get(addEntryRateKey), 10.0D);
    for (int i = 0; i < 5; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (Entry<String, ManagedLedgerImpl> ledger : ((ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory()).getManagedLedgers().entrySet()) {
        ManagedLedgerMBeanImpl stats = (ManagedLedgerMBeanImpl) ledger.getValue().getStats();
        stats.refreshStats(1, TimeUnit.SECONDS);
    }
    List<Metrics> list3 = metrics.generate();
    Assert.assertEquals(list3.get(0).getMetrics().get(addEntryRateKey), 5.0D);
}

Example 5 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testActiveSubscriptionWithCache.

/**
     * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
     * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
     * closed
     *
     * @throws Exception
     */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int recvQueueSize = 4;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    // (1) Create subscription
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
    /************* Validation on non-empty active-cursor **************/
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());
    // (4.3) Validate: entryCache should have cached messages
    assertTrue(entryCache.getSize() != 0);
    /************* Validation on empty active-cursor **************/
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertTrue(entryCache.getSize() == 0);
}