Examples with ManagedCursorInfo org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo used on opensource projects

Example 6 with ManagedCursorInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo in project incubator-pulsar by apache.

the class MetaStoreImplZookeeper method asyncGetCursorInfo.

@Override
public void asyncGetCursorInfo(String ledgerName, String consumerName, final MetaStoreCallback<ManagedCursorInfo> callback) {
    String path = prefix + ledgerName + "/" + consumerName;
    if (log.isDebugEnabled()) {
        log.debug("Reading from {}", path);
    }
    zk.getData(path, false, (rc, path1, ctx, data, stat) -> executor.submit(safeRun(() -> {
        if (rc != Code.OK.intValue()) {
            callback.operationFailed(new MetaStoreException(KeeperException.create(Code.get(rc))));
        } else {
            try {
                ManagedCursorInfo info = parseManagedCursorInfo(data);
                callback.operationComplete(info, new ZKStat(stat));
            } catch (ParseException | InvalidProtocolBufferException e) {
                callback.operationFailed(new MetaStoreException(e));
            }
        }
    })), null);
    if (log.isDebugEnabled()) {
        log.debug("Reading from {} ok", path);
    }
}

Example 7 with ManagedCursorInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo in project incubator-pulsar by apache.

the class ManagedCursorTest method testOutOfOrderDeletePersistenceIntoLedgerWithClose.

/**
 * Verifies cursor persists individually unack range into cursor-ledger if range count is higher than
 * MaxUnackedRangesToPersistInZk
 *
 * @throws Exception
 */
@Test(timeOut = 20000)
public void testOutOfOrderDeletePersistenceIntoLedgerWithClose() throws Exception {
    final int totalAddEntries = 100;
    String ledgerName = "my_test_ledger";
    String cursorName = "c1";
    ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
    // metaStore is allowed to store only up to 10 deleted entries range
    managedLedgerConfig.setMaxUnackedRangesToPersistInZk(10);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
    List<Position> addedPositions = new ArrayList<>();
    for (int i = 0; i < totalAddEntries; i++) {
        Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
        addedPositions.add(p);
        if (i % 2 == 0) {
            // Acknowledge alternative message to create totalEntries/2 holes
            c1.delete(addedPositions.get(i));
        }
    }
    assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
    // Close ledger to persist individual-deleted positions into cursor-ledger
    ledger.close();
    // verify cursor-ledgerId is updated properly into cursor-metaStore
    CountDownLatch cursorLedgerLatch = new CountDownLatch(1);
    AtomicLong cursorLedgerId = new AtomicLong(0);
    ledger.getStore().asyncGetCursorInfo(ledger.getName(), cursorName, new MetaStoreCallback<ManagedCursorInfo>() {

        @Override
        public void operationComplete(ManagedCursorInfo result, Stat stat) {
            cursorLedgerId.set(result.getCursorsLedgerId());
            cursorLedgerLatch.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            cursorLedgerLatch.countDown();
        }
    });
    cursorLedgerLatch.await();
    assertEquals(cursorLedgerId.get(), c1.getCursorLedger());
    // verify cursor-ledger's last entry has individual-deleted positions
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicInteger individualDeletedMessagesCount = new AtomicInteger(0);
    bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.CRC32C, "".getBytes(), (rc, lh, ctx) -> {
        if (rc == BKException.Code.OK) {
            long lastEntry = lh.getLastAddConfirmed();
            lh.asyncReadEntries(lastEntry, lastEntry, (rc1, lh1, seq, ctx1) -> {
                try {
                    LedgerEntry entry = seq.nextElement();
                    PositionInfo positionInfo;
                    positionInfo = PositionInfo.parseFrom(entry.getEntry());
                    individualDeletedMessagesCount.set(positionInfo.getIndividualDeletedMessagesCount());
                } catch (Exception e) {
                }
                latch.countDown();
            }, null);
        } else {
            latch.countDown();
        }
    }, null);
    latch.await();
    assertEquals(individualDeletedMessagesCount.get(), totalAddEntries / 2 - 1);
    // Re-Open
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
    c1 = (ManagedCursorImpl) ledger.openCursor("c1");
    // verify cursor has been recovered
    assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
    // try to read entries which should only read non-deleted positions
    List<Entry> entries = c1.readEntries(totalAddEntries);
    assertEquals(entries.size(), totalAddEntries / 2);
}

Example 8 with ManagedCursorInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo in project incubator-pulsar by apache.

the class ManagedLedgerFactoryImpl method asyncGetManagedLedgerInfo.

@Override
public void asyncGetManagedLedgerInfo(String name, ManagedLedgerInfoCallback callback, Object ctx) {
    store.getManagedLedgerInfo(name, new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {

        @Override
        public void operationComplete(MLDataFormats.ManagedLedgerInfo pbInfo, Stat stat) {
            ManagedLedgerInfo info = new ManagedLedgerInfo();
            info.version = stat.getVersion();
            info.creationDate = DateFormatter.format(stat.getCreationTimestamp());
            info.modificationDate = DateFormatter.format(stat.getModificationTimestamp());
            info.ledgers = new ArrayList<>(pbInfo.getLedgerInfoCount());
            if (pbInfo.hasTerminatedPosition()) {
                info.terminatedPosition = new PositionInfo();
                info.terminatedPosition.ledgerId = pbInfo.getTerminatedPosition().getLedgerId();
                info.terminatedPosition.entryId = pbInfo.getTerminatedPosition().getEntryId();
            }
            for (int i = 0; i < pbInfo.getLedgerInfoCount(); i++) {
                MLDataFormats.ManagedLedgerInfo.LedgerInfo pbLedgerInfo = pbInfo.getLedgerInfo(i);
                LedgerInfo ledgerInfo = new LedgerInfo();
                ledgerInfo.ledgerId = pbLedgerInfo.getLedgerId();
                ledgerInfo.entries = pbLedgerInfo.hasEntries() ? pbLedgerInfo.getEntries() : null;
                ledgerInfo.size = pbLedgerInfo.hasSize() ? pbLedgerInfo.getSize() : null;
                info.ledgers.add(ledgerInfo);
            }
            store.getCursors(name, new MetaStoreCallback<List<String>>() {

                @Override
                public void operationComplete(List<String> cursorsList, Stat stat) {
                    // Get the info for each cursor
                    info.cursors = new ConcurrentSkipListMap<>();
                    List<CompletableFuture<Void>> cursorsFutures = new ArrayList<>();
                    for (String cursorName : cursorsList) {
                        CompletableFuture<Void> cursorFuture = new CompletableFuture<>();
                        cursorsFutures.add(cursorFuture);
                        store.asyncGetCursorInfo(name, cursorName, new MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                            @Override
                            public void operationComplete(ManagedCursorInfo pbCursorInfo, Stat stat) {
                                CursorInfo cursorInfo = new CursorInfo();
                                cursorInfo.version = stat.getVersion();
                                cursorInfo.creationDate = DateFormatter.format(stat.getCreationTimestamp());
                                cursorInfo.modificationDate = DateFormatter.format(stat.getModificationTimestamp());
                                cursorInfo.cursorsLedgerId = pbCursorInfo.getCursorsLedgerId();
                                if (pbCursorInfo.hasMarkDeleteLedgerId()) {
                                    cursorInfo.markDelete = new PositionInfo();
                                    cursorInfo.markDelete.ledgerId = pbCursorInfo.getMarkDeleteLedgerId();
                                    cursorInfo.markDelete.entryId = pbCursorInfo.getMarkDeleteEntryId();
                                }
                                if (pbCursorInfo.getPropertiesCount() > 0) {
                                    cursorInfo.properties = Maps.newTreeMap();
                                    for (int i = 0; i < pbCursorInfo.getPropertiesCount(); i++) {
                                        LongProperty property = pbCursorInfo.getProperties(i);
                                        cursorInfo.properties.put(property.getName(), property.getValue());
                                    }
                                }
                                if (pbCursorInfo.getIndividualDeletedMessagesCount() > 0) {
                                    cursorInfo.individualDeletedMessages = new ArrayList<>();
                                    for (int i = 0; i < pbCursorInfo.getIndividualDeletedMessagesCount(); i++) {
                                        MessageRange range = pbCursorInfo.getIndividualDeletedMessages(i);
                                        MessageRangeInfo rangeInfo = new MessageRangeInfo();
                                        rangeInfo.from.ledgerId = range.getLowerEndpoint().getLedgerId();
                                        rangeInfo.from.entryId = range.getLowerEndpoint().getEntryId();
                                        rangeInfo.to.ledgerId = range.getUpperEndpoint().getLedgerId();
                                        rangeInfo.to.entryId = range.getUpperEndpoint().getEntryId();
                                        cursorInfo.individualDeletedMessages.add(rangeInfo);
                                    }
                                }
                                info.cursors.put(cursorName, cursorInfo);
                                cursorFuture.complete(null);
                            }

                            @Override
                            public void operationFailed(MetaStoreException e) {
                                cursorFuture.completeExceptionally(e);
                            }
                        });
                    }
                    Futures.waitForAll(cursorsFutures).thenRun(() -> {
                        // Completed all the cursors info
                        callback.getInfoComplete(info, ctx);
                    }).exceptionally((ex) -> {
                        callback.getInfoFailed(getManagedLedgerException(ex.getCause()), ctx);
                        return null;
                    });
                }

                @Override
                public void operationFailed(MetaStoreException e) {
                    callback.getInfoFailed(e, ctx);
                }
            });
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.getInfoFailed(e, ctx);
        }
    });
}

Example 9 with ManagedCursorInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo in project incubator-pulsar by apache.

the class MetaStoreImplZookeeperTest method updatingCursorNode.

@Test(timeOut = 20000)
void updatingCursorNode() throws Exception {
    final MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
    zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    final CountDownLatch latch = new CountDownLatch(1);
    ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(1).build();
    store.asyncUpdateCursorInfo("my_test", "c1", info, null, new MetaStoreCallback<Void>() {

        public void operationFailed(MetaStoreException e) {
            fail("should have succeeded");
        }

        public void operationComplete(Void result, Stat version) {
            // Update again using the version
            zkc.failNow(Code.CONNECTIONLOSS);
            ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(2).build();
            store.asyncUpdateCursorInfo("my_test", "c1", info, version, new MetaStoreCallback<Void>() {

                public void operationFailed(MetaStoreException e) {
                    // ok
                    latch.countDown();
                }

                @Override
                public void operationComplete(Void result, Stat version) {
                    fail("should have failed");
                }
            });
        }
    });
    latch.await();
}