Search in sources :

Example 81 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method internalTestFindNewestMatchingAllEntries.

void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger, final int expectedEntryId) throws Exception {
    final String ledgerAndCursorName = name;
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setRetentionSizeInMB(10);
    config.setMaxEntriesPerLedger(entriesPerLedger);
    config.setRetentionTime(1, TimeUnit.HOURS);
    ManagedLedger ledger = factory.open(ledgerAndCursorName, config);
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
    ledger.addEntry(getEntryPublishTime("retained1"));
    // space apart message publish times
    Thread.sleep(100);
    ledger.addEntry(getEntryPublishTime("retained2"));
    Thread.sleep(100);
    ledger.addEntry(getEntryPublishTime("retained3"));
    Thread.sleep(100);
    Position newPosition = ledger.addEntry(getEntryPublishTime("expectedresetposition"));
    long timestamp = System.currentTimeMillis();
    long ledgerId = ((PositionImpl) newPosition).getLedgerId();
    Thread.sleep(2);
    ledger.addEntry(getEntryPublishTime("not-read"));
    List<Entry> entries = c1.readEntries(3);
    c1.markDelete(entries.get(2).getPosition());
    c1.close();
    ledger.close();
    entries.forEach(e -> e.release());
    // give timed ledger trimming a chance to run
    Thread.sleep(100);
    ledger = factory.open(ledgerAndCursorName, config);
    c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
    PositionImpl found = (PositionImpl) findPositionFromAllEntries(c1, timestamp);
    assertEquals(found.getLedgerId(), ledgerId);
    assertEquals(found.getEntryId(), expectedEntryId);
}
Also used : Entry(org.apache.bookkeeper.mledger.Entry) LedgerEntry(org.apache.bookkeeper.client.LedgerEntry) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig)

Example 82 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testGetEntryAfterN.

@Test(timeOut = 20000)
void testGetEntryAfterN() throws Exception {
    ManagedLedger ledger = factory.open("testGetEntryAfterN");
    ManagedCursor c1 = ledger.openCursor("c1");
    Position pos1 = ledger.addEntry("msg1".getBytes());
    Position pos2 = ledger.addEntry("msg2".getBytes());
    Position pos3 = ledger.addEntry("msg3".getBytes());
    Position pos4 = ledger.addEntry("msg4".getBytes());
    Position pos5 = ledger.addEntry("msg5".getBytes());
    List<Entry> entries = c1.readEntries(4);
    entries.forEach(e -> e.release());
    long currentLedger = ((PositionImpl) c1.getMarkDeletedPosition()).getLedgerId();
    // check if the first message is returned for '0'
    Entry e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
    assertEquals(e.getDataAndRelease(), "msg1".getBytes());
    // check that if we call get entry for the same position twice, it returns the same entry
    e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
    assertEquals(e.getDataAndRelease(), "msg1".getBytes());
    // check for a position 'n' after md position
    e = c1.getNthEntry(3, IndividualDeletedEntries.Exclude);
    assertEquals(e.getDataAndRelease(), "msg3".getBytes());
    // check for the last position
    e = c1.getNthEntry(5, IndividualDeletedEntries.Exclude);
    assertEquals(e.getDataAndRelease(), "msg5".getBytes());
    // check for a position outside the limits of the number of entries that exists, it should return null
    e = c1.getNthEntry(10, IndividualDeletedEntries.Exclude);
    assertNull(e);
    // check that the mark delete and read positions have not been updated after all the previous operations
    assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(currentLedger, -1));
    assertEquals(c1.getReadPosition(), new PositionImpl(currentLedger, 4));
    c1.markDelete(pos4);
    assertEquals(c1.getMarkDeletedPosition(), pos4);
    e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
    assertEquals(e.getDataAndRelease(), "msg5".getBytes());
    c1.readEntries(1);
    c1.markDelete(pos5);
    e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
    assertNull(e);
}
Also used : Entry(org.apache.bookkeeper.mledger.Entry) LedgerEntry(org.apache.bookkeeper.client.LedgerEntry) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 83 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testFindNewestMatchingEdgeCase5.

@Test(timeOut = 20000)
void testFindNewestMatchingEdgeCase5() throws Exception {
    ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase5");
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    ledger.addEntry("expired".getBytes(Encoding));
    Position p2 = ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("not-expired".getBytes(Encoding));
    assertEquals(c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))), p2);
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) ManagedCursorInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo) Future(java.util.concurrent.Future) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) CyclicBarrier(java.util.concurrent.CyclicBarrier) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Executors(java.util.concurrent.Executors) BKException(org.apache.bookkeeper.client.BKException) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Code(org.apache.zookeeper.KeeperException.Code) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Assert(org.testng.Assert) Charset(java.nio.charset.Charset) ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) ExecutorService(java.util.concurrent.ExecutorService) LedgerEntry(org.apache.bookkeeper.client.LedgerEntry) Charsets(com.google.common.base.Charsets) Logger(org.slf4j.Logger) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) PositionInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) Collections(java.util.Collections) ReadEntriesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Test(org.testng.annotations.Test)

Example 84 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method randomOrderAcks.

@Test(timeOut = 20000)
void randomOrderAcks() throws Exception {
    ManagedLedger ledger = factory.open("outOfOrderAcks");
    ManagedCursor c1 = ledger.openCursor("c1");
    int N = 10;
    List<Position> positions = new ArrayList<>();
    for (int i = 0; i < N; i++) {
        positions.add(ledger.addEntry("entry".getBytes()));
    }
    assertEquals(c1.getNumberOfEntriesInBacklog(), N);
    // Randomize the ack sequence
    Collections.shuffle(positions);
    int toDelete = N;
    for (Position p : positions) {
        assertEquals(c1.getNumberOfEntriesInBacklog(), toDelete);
        c1.delete(p);
        --toDelete;
        assertEquals(c1.getNumberOfEntriesInBacklog(), toDelete);
    }
}
Also used : Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ArrayList(java.util.ArrayList) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) Test(org.testng.annotations.Test)

Example 85 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testFindNewestMatchingOdd5.

@Test(timeOut = 20000)
void testFindNewestMatchingOdd5() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger");
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("expired".getBytes(Encoding));
    ledger.addEntry("expired".getBytes(Encoding));
    Position p5 = ledger.addEntry("expired".getBytes(Encoding));
    assertEquals(c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))), p5);
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) MetaStoreCallback(org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback) ManagedCursorInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo) Future(java.util.concurrent.Future) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) CyclicBarrier(java.util.concurrent.CyclicBarrier) DigestType(org.apache.bookkeeper.client.BookKeeper.DigestType) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Executors(java.util.concurrent.Executors) BKException(org.apache.bookkeeper.client.BKException) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stat(org.apache.bookkeeper.mledger.impl.MetaStore.Stat) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Code(org.apache.zookeeper.KeeperException.Code) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) MockedBookKeeperTestCase(org.apache.bookkeeper.test.MockedBookKeeperTestCase) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Callable(java.util.concurrent.Callable) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) MetaStoreException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException) Assert(org.testng.Assert) Charset(java.nio.charset.Charset) ManagedLedgerFactoryConfig(org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig) ExecutorService(java.util.concurrent.ExecutorService) LedgerEntry(org.apache.bookkeeper.client.LedgerEntry) Charsets(com.google.common.base.Charsets) Logger(org.slf4j.Logger) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) PositionInfo(org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo) MarkDeleteCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback) VoidCallback(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback) Collections(java.util.Collections) ReadEntriesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback) Position(org.apache.bookkeeper.mledger.Position) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Test(org.testng.annotations.Test)

Aggregations

Position (org.apache.bookkeeper.mledger.Position)201 Test (org.testng.annotations.Test)169 ManagedLedger (org.apache.bookkeeper.mledger.ManagedLedger)168 ManagedCursor (org.apache.bookkeeper.mledger.ManagedCursor)167 ManagedLedgerConfig (org.apache.bookkeeper.mledger.ManagedLedgerConfig)127 ManagedLedgerException (org.apache.bookkeeper.mledger.ManagedLedgerException)115 Entry (org.apache.bookkeeper.mledger.Entry)104 CountDownLatch (java.util.concurrent.CountDownLatch)97 AddEntryCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback)72 ManagedLedgerFactory (org.apache.bookkeeper.mledger.ManagedLedgerFactory)68 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)58 BKException (org.apache.bookkeeper.client.BKException)56 List (java.util.List)55 MarkDeleteCallback (org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback)53 AtomicReference (java.util.concurrent.atomic.AtomicReference)52 AsyncCallbacks (org.apache.bookkeeper.mledger.AsyncCallbacks)52 Logger (org.slf4j.Logger)52 LoggerFactory (org.slf4j.LoggerFactory)52 Lists (com.google.common.collect.Lists)51 TimeUnit (java.util.concurrent.TimeUnit)51