Search in sources :

Example 46 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testFindNewestMatchingEdgeCase4.

@Test(timeOut = 20000)
void testFindNewestMatchingEdgeCase4() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    Position p1 = ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("not-expired".getBytes(Encoding));
    ledger.addEntry("not-expired".getBytes(Encoding));
    assertEquals(c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))), p1);
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) ManagedCursorInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo) Future(java.util.concurrent.Future) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) CyclicBarrier(java.util.concurrent.CyclicBarrier) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Executors(java.util.concurrent.Executors) BKException(org.apache.bookkeeper.client.BKException) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Code(org.apache.zookeeper.KeeperException.Code) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Assert(org.testng.Assert) Charset(java.nio.charset.Charset) ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) ExecutorService(java.util.concurrent.ExecutorService) LedgerEntry(org.apache.bookkeeper.client.LedgerEntry) Charsets(com.google.common.base.Charsets) Logger(org.slf4j.Logger) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) PositionInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) Collections(java.util.Collections) ReadEntriesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Test(org.testng.annotations.Test)

Example 47 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testMarkDeleteTwice.

@Test(timeOut = 20000)
void testMarkDeleteTwice() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
    ManagedCursor cursor = ledger.openCursor("c1");
    Position p1 = ledger.addEntry("entry1".getBytes());
    cursor.markDelete(p1);
    cursor.markDelete(p1);
    assertEquals(cursor.getMarkDeletedPosition(), p1);
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 48 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testSkipEntries.

@Test(timeOut = 20000)
void testSkipEntries() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
    Position pos;
    ManagedCursor c1 = ledger.openCursor("c1");
    // test skip on empty ledger
    pos = c1.getReadPosition();
    c1.skipEntries(1, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getReadPosition(), pos);
    pos = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    pos = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    // skip entries in same ledger
    c1.skipEntries(1, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 1);
    // skip entries until end of ledger
    c1.skipEntries(1, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 0);
    assertEquals(c1.getReadPosition(), pos.getNext());
    assertEquals(c1.getMarkDeletedPosition(), pos);
    // skip entries across ledgers
    for (int i = 0; i < 6; i++) {
        pos = ledger.addEntry("dummy-entry".getBytes(Encoding));
    }
    c1.skipEntries(5, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 1);
    // skip more than the current set of entries
    c1.skipEntries(10, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 0);
    assertEquals(c1.hasMoreEntries(), false);
    assertEquals(c1.getReadPosition(), pos.getNext());
    assertEquals(c1.getMarkDeletedPosition(), pos);
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 49 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method asyncMarkDeleteBlocking.

@Test(timeOut = 20000)
public void asyncMarkDeleteBlocking() throws Exception {
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setMaxEntriesPerLedger(10);
    config.setMetadataMaxEntriesPerLedger(5);
    ManagedLedger ledger = factory.open("my_test_ledger", config);
    final ManagedCursor c1 = ledger.openCursor("c1");
    final AtomicReference<Position> lastPosition = new AtomicReference<Position>();
    final int N = 100;
    final CountDownLatch latch = new CountDownLatch(N);
    for (int i = 0; i < N; i++) {
        ledger.asyncAddEntry("entry".getBytes(Encoding), new AddEntryCallback() {

            @Override
            public void addFailed(ManagedLedgerException exception, Object ctx) {
            }

            @Override
            public void addComplete(Position position, Object ctx) {
                lastPosition.set(position);
                c1.asyncMarkDelete(position, new MarkDeleteCallback() {

                    @Override
                    public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
                    }

                    @Override
                    public void markDeleteComplete(Object ctx) {
                        latch.countDown();
                    }
                }, null);
            }
        }, null);
    }
    latch.await();
    assertEquals(c1.getNumberOfEntries(), 0);
    // Reopen
    ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ledger = factory2.open("my_test_ledger");
    ManagedCursor c2 = ledger.openCursor("c1");
    assertEquals(c2.getMarkDeletedPosition(), lastPosition.get());
    factory2.shutdown();
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) Test(org.testng.annotations.Test)

Example 50 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method testConcurrentMarkDelete.

@Test
public void testConcurrentMarkDelete() throws Exception {
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
    ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
    mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    // set the data ledger size
    mlConfig.setMaxEntriesPerLedger(100);
    // set the metadata ledger size to 1 to kick off many ledger switching cases
    mlConfig.setMetadataMaxEntriesPerLedger(10);
    ManagedLedger ledger = factory.open("ml-markdelete-ledger", mlConfig);
    final List<Position> addedEntries = Lists.newArrayList();
    int numCursors = 10;
    final CyclicBarrier barrier = new CyclicBarrier(numCursors);
    List<ManagedCursor> cursors = Lists.newArrayList();
    for (int i = 0; i < numCursors; i++) {
        cursors.add(ledger.openCursor(String.format("c%d", i)));
    }
    for (int i = 0; i < 50; i++) {
        Position pos = ledger.addEntry("entry".getBytes());
        addedEntries.add(pos);
    }
    List<Future<?>> futures = Lists.newArrayList();
    for (ManagedCursor cursor : cursors) {
        futures.add(executor.submit(() -> {
            barrier.await();
            for (Position position : addedEntries) {
                cursor.markDelete(position);
            }
            return null;
        }));
    }
    for (Future<?> future : futures) {
        future.get();
    }
    // Since in this test we roll-over the cursor ledger every 10 entries acknowledged, the background roll back
    // might still be happening when the futures are completed.
    Thread.sleep(1000);
    factory.shutdown();
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) CyclicBarrier(java.util.concurrent.CyclicBarrier) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) Future(java.util.concurrent.Future) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) Test(org.testng.annotations.Test)

Aggregations

Position (org.apache.bookkeeper.mledger.Position)201 Test (org.testng.annotations.Test)169 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)168 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)167 ManagedLedgerConfig (org.apache.bookkeeper.mledger.ManagedLedgerConfig)127 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)115 Entry (org.apache.bookkeeper.mledger.Entry)104 CountDownLatch (java.util.concurrent.CountDownLatch)97 AddEntryCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)72 ManagedLedgerFactory (org.apache.bookkeeper.mledger.ManagedLedgerFactory)68 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)58 BKException (org.apache.bookkeeper.client.BKException)56 List (java.util.List)55 MarkDeleteCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback)53 AtomicReference (java.util.concurrent.atomic.AtomicReference)52 AsyncCallbacks (org.apache.bookkeeper.mledger.AsyncCallbacks)52 Logger (org.slf4j.Logger)52 LoggerFactory (org.slf4j.LoggerFactory)52 Lists (com.google.common.collect.Lists)51 TimeUnit (java.util.concurrent.TimeUnit)51