Examples with ManagedLedgerConfig org.apache.bookkeeper.mledger.ManagedLedgerConfig used on opensource projects

Example 51 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedCursorTest method testResetCursor.

@Test(timeOut = 20000)
void testResetCursor() throws Exception {
    ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
    ManagedCursor cursor = ledger.openCursor("trc1");
    ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    final AtomicBoolean moveStatus = new AtomicBoolean(false);
    PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2);
    try {
        cursor.resetCursor(resetPosition);
        moveStatus.set(true);
    } catch (Exception e) {
        log.warn("error in reset cursor", e.getCause());
    }
    assertTrue(moveStatus.get());
    assertTrue(cursor.getReadPosition().equals(resetPosition));
    cursor.close();
    ledger.close();
}

Example 52 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method testConcurrentMarkDelete.

@Test
public void testConcurrentMarkDelete() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
    mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    // set the data ledger size
    mlConfig.setMaxEntriesPerLedger(100);
    // set the metadata ledger size to 1 to kick off many ledger switching cases
    mlConfig.setMetadataMaxEntriesPerLedger(10);
    ManagedLedger ledger = factory.open("ml-markdelete-ledger", mlConfig);
    final List<Position> addedEntries = Lists.newArrayList();
    int numCursors = 10;
    final CyclicBarrier barrier = new CyclicBarrier(numCursors);
    List<ManagedCursor> cursors = Lists.newArrayList();
    for (int i = 0; i < numCursors; i++) {
        cursors.add(ledger.openCursor(String.format("c%d", i)));
    }
    for (int i = 0; i < 50; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
    List<Future<?>> futures = Lists.newArrayList();
    for (ManagedCursor cursor : cursors) {
        futures.add(executor.submit(() -> {
            barrier.await();
            for (Position position : addedEntries) {
                cursor.markDelete(position);
            }
            return null;
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
    // Since in this test we roll-over the cursor ledger every 10 entries acknowledged, the background roll back
    // might still be happening when the futures are completed.
    Thread.sleep(1000);
    factory.shutdown();
}

Example 53 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method testBookieFailure.

@Test
public void testBookieFailure() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    ManagedLedger ledger = factory.open("my-ledger", config);
    ManagedCursor cursor = ledger.openCursor("my-cursor");
    ledger.addEntry("entry-0".getBytes());
    killBookie(1);
    // Now we want to simulate that:
    // 1. The write operation fails because we only have 1 bookie available
    // 2. The bk client cannot properly close the ledger (finalizing the number of entries) because ZK is also
    // not available
    // 3. When we re-establish the service one, the ledger recovery will be triggered and the half-committed entry
    // is restored
    // Force to close the ZK client object so that BK will fail to close the ledger
    bkc.getZkHandle().close();
    try {
        ledger.addEntry("entry-1".getBytes());
        fail("should fail");
    } catch (ManagedLedgerException e) {
    // ok
    }
    bkc = new BookKeeperTestClient(baseClientConf);
    startNewBookie();
    // Reconnect a new bk client
    factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ledger = factory.open("my-ledger", config);
    cursor = ledger.openCursor("my-cursor");
    // Next add should succeed
    ledger.addEntry("entry-2".getBytes());
    assertEquals(3, cursor.getNumberOfEntriesInBacklog());
    List<Entry> entries = cursor.readEntries(1);
    assertEquals(1, entries.size());
    assertEquals("entry-0", new String(entries.get(0).getData()));
    entries.forEach(e -> e.release());
    // entry-1 which was half-committed will get fully committed during the recovery phase
    entries = cursor.readEntries(1);
    assertEquals(1, entries.size());
    assertEquals("entry-1", new String(entries.get(0).getData()));
    entries.forEach(e -> e.release());
    entries = cursor.readEntries(1);
    assertEquals(1, entries.size());
    assertEquals("entry-2", new String(entries.get(0).getData()));
    entries.forEach(e -> e.release());
    factory.shutdown();
}

Example 54 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method verifyConcurrentUsage.

@Test
public void verifyConcurrentUsage() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(100 * 1024 * 1024);
    ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    EntryCacheManager cacheManager = factory.getEntryCacheManager();
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
    final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger", conf);
    int NumProducers = 1;
    int NumConsumers = 1;
    final AtomicBoolean done = new AtomicBoolean();
    final CyclicBarrier barrier = new CyclicBarrier(NumProducers + NumConsumers + 1);
    List<Future<?>> futures = Lists.newArrayList();
    for (int i = 0; i < NumProducers; i++) {
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                while (!done.get()) {
                    ledger.addEntry("entry".getBytes());
                    Thread.sleep(1);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    for (int i = 0; i < NumConsumers; i++) {
        final int idx = i;
        futures.add(executor.submit(() -> {
            try {
                barrier.await();
                ManagedCursor cursor = ledger.openCursor("my-cursor-" + idx);
                while (!done.get()) {
                    List<Entry> entries = cursor.readEntries(1);
                    if (!entries.isEmpty()) {
                        cursor.markDelete(entries.get(0).getPosition());
                    }
                    entries.forEach(e -> e.release());
                    Thread.sleep(2);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }));
    }
    barrier.await();
    Thread.sleep(1 * 1000);
    done.set(true);
    for (Future<?> future : futures) {
        future.get();
    }
    cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsRate() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
    assertTrue(cacheManager.mlFactoryMBean.getCacheHitsThroughput() > 0.0);
    assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
    factory.shutdown();
}

Example 55 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method testResetCursorAfterRecovery.

@Test(timeOut = 20000)
void testResetCursorAfterRecovery() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig conf = new ManagedLedgerConfig().setMaxEntriesPerLedger(10).setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", conf);
    ManagedCursor cursor = ledger.openCursor("trc1");
    Position p1 = ledger.addEntry("dummy-entry-1".getBytes());
    Position p2 = ledger.addEntry("dummy-entry-2".getBytes());
    Position p3 = ledger.addEntry("dummy-entry-3".getBytes());
    Position p4 = ledger.addEntry("dummy-entry-4".getBytes());
    cursor.markDelete(p3);
    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, zkc);
    ledger = factory2.open("my_test_move_cursor_ledger", conf);
    cursor = ledger.openCursor("trc1");
    assertEquals(cursor.getMarkDeletedPosition(), p3);
    assertEquals(cursor.getReadPosition(), p4);
    assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
    cursor.resetCursor(p2);
    assertEquals(cursor.getMarkDeletedPosition(), p1);
    assertEquals(cursor.getReadPosition(), p2);
    assertEquals(cursor.getNumberOfEntriesInBacklog(), 3);
    factory2.shutdown();
    factory.shutdown();
}