Examples with ManagedLedgerFactoryConfig org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig used on opensource projects

Example 1 with ManagedLedgerFactoryConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig in project pulsar by yahoo.

the class EntryCacheManagerTest method verifyHitsMisses.

@Test
void verifyHitsMisses() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(7 * 10);
    config.setCacheEvictionWatermark(0.8);
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ledger");
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2");
    for (int i = 0; i < 10; i++) {
        ledger.addEntry(("entry-" + i).getBytes());
    }
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    List<Entry> entries = c1.readEntries(10);
    assertEquals(entries.size(), 10);
    entries.forEach(e -> e.release());
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 10.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 70.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    ledger.deactivateCursor(c1);
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    entries = c2.readEntries(10);
    assertEquals(entries.size(), 10);
    entries.forEach(e -> e.release());
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 10.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 70.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    PositionImpl pos = (PositionImpl) entries.get(entries.size() - 1).getPosition();
    c2.setReadPosition(pos);
    ledger.discardEntriesFromCache(c2, pos);
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
}

Example 2 with ManagedLedgerFactoryConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig in project pulsar by yahoo.

the class EntryCacheManagerTest method simple.

@Test
void simple() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(10);
    config.setCacheEvictionWatermark(0.8);
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    EntryCache cache1 = cacheManager.getEntryCache(ml1);
    EntryCache cache2 = cacheManager.getEntryCache(ml2);
    cache1.insert(new EntryImpl(1, 1, new byte[4]));
    cache1.insert(new EntryImpl(1, 0, new byte[3]));
    assertEquals(cache1.getSize(), 7);
    assertEquals(cacheManager.getSize(), 7);
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 10);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 7);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    cache2.insert(new EntryImpl(2, 0, new byte[1]));
    cache2.insert(new EntryImpl(2, 1, new byte[1]));
    cache2.insert(new EntryImpl(2, 2, new byte[1]));
    assertEquals(cache2.getSize(), 3);
    assertEquals(cacheManager.getSize(), 10);
    // Next insert should trigger a cache eviction to force the size to 8
    // The algorithm should evict entries from cache1
    cache2.insert(new EntryImpl(2, 3, new byte[1]));
    // Wait for eviction to be completed in background
    Thread.sleep(100);
    assertEquals(cacheManager.getSize(), 7);
    assertEquals(cache1.getSize(), 4);
    assertEquals(cache2.getSize(), 3);
    cacheManager.removeEntryCache("cache1");
    assertEquals(cacheManager.getSize(), 3);
    assertEquals(cache2.getSize(), 3);
    // Should remove 2 entries
    cache2.invalidateEntries(new PositionImpl(2, 1));
    assertEquals(cacheManager.getSize(), 1);
    assertEquals(cache2.getSize(), 1);
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 10);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 1);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 1);
}

Example 3 with ManagedLedgerFactoryConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig in project pulsar by yahoo.

the class EntryCacheManagerTest method verifyNoCacheIfNoConsumer.

@Test
void verifyNoCacheIfNoConsumer() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(7 * 10);
    config.setCacheEvictionWatermark(0.8);
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ledger");
    EntryCache cache1 = ledger.entryCache;
    for (int i = 0; i < 10; i++) {
        ledger.addEntry(("entry-" + i).getBytes());
    }
    assertEquals(cache1.getSize(), 0);
    assertEquals(cacheManager.getSize(), 0);
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 7 * 10);
    assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
}

Example 4 with ManagedLedgerFactoryConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig in project pulsar by yahoo.

the class ManagedLedgerBkTest method verifyConcurrentUsage.

@Test
public void verifyConcurrentUsage() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(100 * 1024 * 1024);
    ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger", conf);
    int NumProducers = 1;
    int NumConsumers = 1;
    final AtomicBoolean done = new AtomicBoolean();
    final CyclicBarrier barrier = new CyclicBarrier(NumProducers + NumConsumers + 1);
    List<Future<?>> futures = Lists.newArrayList();
    for (int i = 0; i < NumProducers; i++) {
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                while (!done.get()) {
                    ledger.addEntry("entry".getBytes());
                    Thread.sleep(1);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    for (int i = 0; i < NumConsumers; i++) {
        final int idx = i;
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                ManagedCursor cursor = ledger.openCursor("my-cursor-" + idx);
                while (!done.get()) {
                    List<Entry> entries = cursor.readEntries(1);
                    if (!entries.isEmpty()) {
                        cursor.markDelete(entries.get(0).getPosition());
                    }
                    entries.forEach(e -> e.release());
                    Thread.sleep(2);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    barrier.await();
    Thread.sleep(1 * 1000);
    done.set(true);
    for (Future<?> future : futures) {
        future.get();
    }
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsRate() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsThroughput() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    factory.shutdown();
}

Example 5 with ManagedLedgerFactoryConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig in project pulsar by yahoo.

the class ManagedLedgerBkTest method testOfflineTopicBacklog.

@Test
public void testOfflineTopicBacklog() throws Exception {
    ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
    factoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config);
    ManagedCursor cursor = ledger.openCursor("c1");
    int N = 1;
    for (int i = 0; i < N; i++) {
        String entry = "entry-" + i;
        ledger.addEntry(entry.getBytes());
    }
    List<Entry> entries = cursor.readEntries(N);
    assertEquals(N, entries.size());
    entries.forEach(e -> e.release());
    ledger.close();
    ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false);
    PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog((ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger");
    factory.shutdown();
    assertNotNull(offlineTopicStats);
}