Examples with Stat org.apache.pulsar.metadata.api.Stat used on opensource projects

Example 11 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class ManagedLedgerImpl method initializeCursors.

private void initializeCursors(final ManagedLedgerInitializeLedgerCallback callback) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] initializing cursors", name);
    }
    store.getCursors(name, new MetaStoreCallback<List<String>>() {

        @Override
        public void operationComplete(List<String> consumers, Stat s) {
            // Load existing cursors
            final AtomicInteger cursorCount = new AtomicInteger(consumers.size());
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found {} cursors", name, consumers.size());
            }
            if (consumers.isEmpty()) {
                callback.initializeComplete();
                return;
            }
            if (!ManagedLedgerImpl.this.config.isLazyCursorRecovery()) {
                log.debug("[{}] Loading cursors", name);
                for (final String cursorName : consumers) {
                    log.info("[{}] Loading cursor {}", name, cursorName);
                    final ManagedCursorImpl cursor;
                    cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName);
                    cursor.recover(new VoidCallback() {

                        @Override
                        public void operationComplete() {
                            log.info("[{}] Recovery for cursor {} completed. pos={} -- todo={}", name, cursorName, cursor.getMarkDeletedPosition(), cursorCount.get() - 1);
                            cursor.setActive();
                            cursors.add(cursor);
                            if (cursorCount.decrementAndGet() == 0) {
                                // The initialization is now completed, register the jmx mbean
                                callback.initializeComplete();
                            }
                        }

                        @Override
                        public void operationFailed(ManagedLedgerException exception) {
                            log.warn("[{}] Recovery for cursor {} failed", name, cursorName, exception);
                            cursorCount.set(-1);
                            callback.initializeFailed(exception);
                        }
                    });
                }
            } else {
                // Lazily recover cursors by put them to uninitializedCursors map.
                for (final String cursorName : consumers) {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Recovering cursor {} lazily", name, cursorName);
                    }
                    final ManagedCursorImpl cursor;
                    cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName);
                    CompletableFuture<ManagedCursor> cursorRecoveryFuture = new CompletableFuture<>();
                    uninitializedCursors.put(cursorName, cursorRecoveryFuture);
                    cursor.recover(new VoidCallback() {

                        @Override
                        public void operationComplete() {
                            log.info("[{}] Lazy recovery for cursor {} completed. pos={} -- todo={}", name, cursorName, cursor.getMarkDeletedPosition(), cursorCount.get() - 1);
                            cursor.setActive();
                            synchronized (ManagedLedgerImpl.this) {
                                cursors.add(cursor);
                                uninitializedCursors.remove(cursor.getName()).complete(cursor);
                            }
                        }

                        @Override
                        public void operationFailed(ManagedLedgerException exception) {
                            log.warn("[{}] Lazy recovery for cursor {} failed", name, cursorName, exception);
                            synchronized (ManagedLedgerImpl.this) {
                                uninitializedCursors.remove(cursor.getName()).completeExceptionally(exception);
                            }
                        }
                    });
                }
                // Complete ledger recovery.
                callback.initializeComplete();
            }
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            log.warn("[{}] Failed to get the cursors list", name, e);
            callback.initializeFailed(new ManagedLedgerException(e));
        }
    });
}

Example 12 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class ManagedLedgerOfflineBacklog method calculateCursorBacklogs.

private void calculateCursorBacklogs(final ManagedLedgerFactoryImpl factory, final TopicName topicName, final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers, final PersistentOfflineTopicStats offlineTopicStats) throws Exception {
    if (ledgers.isEmpty()) {
        return;
    }
    String managedLedgerName = topicName.getPersistenceNamingEncoding();
    MetaStore store = factory.getMetaStore();
    BookKeeper bk = factory.getBookKeeper();
    final CountDownLatch allCursorsCounter = new CountDownLatch(1);
    final long errorInReadingCursor = -1;
    ConcurrentOpenHashMap<String, Long> ledgerRetryMap = new ConcurrentOpenHashMap<>();
    final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
    final PositionImpl lastLedgerPosition = new PositionImpl(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
    if (log.isDebugEnabled()) {
        log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition);
    }
    store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback<List<String>>() {

        @Override
        public void operationComplete(List<String> cursors, Stat v) {
            // Load existing cursors
            if (log.isDebugEnabled()) {
                log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size());
            }
            if (cursors.isEmpty()) {
                allCursorsCounter.countDown();
                return;
            }
            final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
            for (final String cursorName : cursors) {
                // determine subscription position from cursor ledger
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName);
                }
                AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
                    long ledgerId = lh.getId();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId, cursorName, rc);
                    }
                    if (rc != BKException.Code.OK) {
                        log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                        cursorCounter.countDown();
                        return;
                    }
                    long lac = lh.getLastAddConfirmed();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                    }
                    if (lac == LedgerHandle.INVALID_ENTRY_ID) {
                        // save the ledger id and cursor to retry outside of this call back
                        // since we are trying to read the same cursor ledger, we will block until
                        // this current callback completes, since an attempt to read the entry
                        // will block behind this current operation to complete
                        ledgerRetryMap.put(cursorName, ledgerId);
                        log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac, ledgerId);
                        cursorCounter.countDown();
                        return;
                    }
                    final long entryId = lac;
                    // read last acked message position for subscription
                    lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {

                        @Override
                        public void readComplete(int rc, LedgerHandle lh, Enumeration<LedgerEntry> seq, Object ctx) {
                            try {
                                if (log.isDebugEnabled()) {
                                    log.debug("readComplete rc={} entryId={}", rc, entryId);
                                }
                                if (rc != BKException.Code.OK) {
                                    log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
                                    // indicate that this cursor should be excluded
                                    offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                } else {
                                    LedgerEntry entry = seq.nextElement();
                                    MLDataFormats.PositionInfo positionInfo;
                                    try {
                                        positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry());
                                    } catch (InvalidProtocolBufferException e) {
                                        log.warn("[{}] Error reading position from metadata ledger {} for cursor {}: {}", managedLedgerName, ledgerId, cursorName, e);
                                        offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor, lh.getId());
                                        return;
                                    }
                                    final PositionImpl lastAckedMessagePosition = new PositionImpl(positionInfo);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Cursor {} MD {} read last ledger position {}", managedLedgerName, cursorName, lastAckedMessagePosition, lastLedgerPosition);
                                    }
                                    // calculate cursor backlog
                                    Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                                    if (log.isDebugEnabled()) {
                                        log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                                    }
                                    long cursorBacklog = getNumberOfEntries(range, ledgers);
                                    offlineTopicStats.messageBacklog += cursorBacklog;
                                    offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
                                }
                            } finally {
                                cursorCounter.countDown();
                            }
                        }
                    }, null);
                };
                // end of cursor meta read callback
                store.asyncGetCursorInfo(managedLedgerName, cursorName, new MetaStore.MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                    @Override
                    public void operationComplete(MLDataFormats.ManagedCursorInfo info, Stat stat) {
                        long cursorLedgerId = info.getCursorsLedgerId();
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName, cursorName, cursorLedgerId);
                        }
                        if (cursorLedgerId != -1) {
                            bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password, cursorLedgerOpenCb, null);
                        } else {
                            PositionImpl lastAckedMessagePosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
                            Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                            }
                            long cursorBacklog = getNumberOfEntries(range, ledgers);
                            offlineTopicStats.messageBacklog += cursorBacklog;
                            offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
                            cursorCounter.countDown();
                        }
                    }

                    @Override
                    public void operationFailed(ManagedLedgerException.MetaStoreException e) {
                        log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName, cursorName, e);
                        cursorCounter.countDown();
                    }
                });
            }
            // for every cursor find backlog
            try {
                if (accurate) {
                    cursorCounter.await();
                } else {
                    cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e);
            } finally {
                allCursorsCounter.countDown();
            }
        }

        @Override
        public void operationFailed(ManagedLedgerException.MetaStoreException e) {
            log.warn("[{}] Failed to get the cursors list", managedLedgerName, e);
            allCursorsCounter.countDown();
        }
    });
    if (accurate) {
        allCursorsCounter.await();
    } else {
        allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
    }
    // go through ledgers where LAC was -1
    if (accurate && ledgerRetryMap.size() > 0) {
        ledgerRetryMap.forEach((cursorName, ledgerId) -> {
            if (log.isDebugEnabled()) {
                log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId);
            }
            PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
            if (lastAckedMessagePosition == null) {
                log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", managedLedgerName, cursorName, ledgerId);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName, cursorName, ledgerId, lastAckedMessagePosition);
                }
                // calculate cursor backlog
                Range<PositionImpl> range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName, cursorName, range);
                }
                long cursorBacklog = getNumberOfEntries(range, ledgers);
                offlineTopicStats.messageBacklog += cursorBacklog;
                offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
            }
        });
    }
}

Example 13 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class ManagedCursorImpl method persistPositionMetaStore.

private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map<String, Long> properties, MetaStoreCallback<Void> callback, boolean persistIndividualDeletedMessageRanges) {
    if (state == State.Closed) {
        ledger.getExecutor().execute(safeRun(() -> callback.operationFailed(new MetaStoreException(new CursorAlreadyClosedException(name + " cursor already closed")))));
        return;
    }
    // When closing we store the last mark-delete position in the z-node itself, so we won't need the cursor ledger,
    // hence we write it as -1. The cursor ledger is deleted once the z-node write is confirmed.
    ManagedCursorInfo.Builder info = // 
    ManagedCursorInfo.newBuilder().setCursorsLedgerId(// 
    cursorsLedgerId).setMarkDeleteLedgerId(// 
    position.getLedgerId()).setMarkDeleteEntryId(// 
    position.getEntryId()).setLastActive(// 
    lastActive);
    info.addAllProperties(buildPropertiesMap(properties));
    if (persistIndividualDeletedMessageRanges) {
        info.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges());
        if (config.isDeletionAtBatchIndexLevelEnabled()) {
            info.addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList());
        }
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}][{}]  Closing cursor at md-position: {}", ledger.getName(), name, position);
    }
    ledger.getStore().asyncUpdateCursorInfo(ledger.getName(), name, info.build(), cursorLedgerStat, new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void result, Stat stat) {
            cursorLedgerStat = stat;
            callback.operationComplete(result, stat);
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            if (e instanceof MetaStoreException.BadVersionException) {
                log.warn("[{}] Failed to update cursor metadata for {} due to version conflict {}", ledger.name, name, e.getMessage());
                // the ownership and refresh the version again.
                if (ledger.mlOwnershipChecker != null && ledger.mlOwnershipChecker.get()) {
                    ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, new MetaStoreCallback<ManagedCursorInfo>() {

                        @Override
                        public void operationComplete(ManagedCursorInfo info, Stat stat) {
                            cursorLedgerStat = stat;
                        }

                        @Override
                        public void operationFailed(MetaStoreException e) {
                            if (log.isDebugEnabled()) {
                                log.debug("[{}] Failed to refresh cursor metadata-version for {} due " + "to {}", ledger.name, name, e.getMessage());
                            }
                        }
                    });
                }
            }
            callback.operationFailed(e);
        }
    });
}

Example 14 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class ManagedLedgerFactoryImpl method asyncGetManagedLedgerInfo.

@Override
public void asyncGetManagedLedgerInfo(String name, ManagedLedgerInfoCallback callback, Object ctx) {
    store.getManagedLedgerInfo(name, false, /* createIfMissing */
    new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {

        @Override
        public void operationComplete(MLDataFormats.ManagedLedgerInfo pbInfo, Stat stat) {
            ManagedLedgerInfo info = new ManagedLedgerInfo();
            info.version = stat.getVersion();
            info.creationDate = DateFormatter.format(stat.getCreationTimestamp());
            info.modificationDate = DateFormatter.format(stat.getModificationTimestamp());
            info.ledgers = new ArrayList<>(pbInfo.getLedgerInfoCount());
            if (pbInfo.hasTerminatedPosition()) {
                info.terminatedPosition = new PositionInfo();
                info.terminatedPosition.ledgerId = pbInfo.getTerminatedPosition().getLedgerId();
                info.terminatedPosition.entryId = pbInfo.getTerminatedPosition().getEntryId();
            }
            if (pbInfo.getPropertiesCount() > 0) {
                info.properties = Maps.newTreeMap();
                for (int i = 0; i < pbInfo.getPropertiesCount(); i++) {
                    MLDataFormats.KeyValue property = pbInfo.getProperties(i);
                    info.properties.put(property.getKey(), property.getValue());
                }
            }
            for (int i = 0; i < pbInfo.getLedgerInfoCount(); i++) {
                MLDataFormats.ManagedLedgerInfo.LedgerInfo pbLedgerInfo = pbInfo.getLedgerInfo(i);
                LedgerInfo ledgerInfo = new LedgerInfo();
                ledgerInfo.ledgerId = pbLedgerInfo.getLedgerId();
                ledgerInfo.entries = pbLedgerInfo.hasEntries() ? pbLedgerInfo.getEntries() : null;
                ledgerInfo.size = pbLedgerInfo.hasSize() ? pbLedgerInfo.getSize() : null;
                ledgerInfo.isOffloaded = pbLedgerInfo.hasOffloadContext();
                info.ledgers.add(ledgerInfo);
            }
            store.getCursors(name, new MetaStoreCallback<List<String>>() {

                @Override
                public void operationComplete(List<String> cursorsList, Stat stat) {
                    // Get the info for each cursor
                    info.cursors = new ConcurrentSkipListMap<>();
                    List<CompletableFuture<Void>> cursorsFutures = new ArrayList<>();
                    for (String cursorName : cursorsList) {
                        CompletableFuture<Void> cursorFuture = new CompletableFuture<>();
                        cursorsFutures.add(cursorFuture);
                        store.asyncGetCursorInfo(name, cursorName, new MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {

                            @Override
                            public void operationComplete(ManagedCursorInfo pbCursorInfo, Stat stat) {
                                CursorInfo cursorInfo = new CursorInfo();
                                cursorInfo.version = stat.getVersion();
                                cursorInfo.creationDate = DateFormatter.format(stat.getCreationTimestamp());
                                cursorInfo.modificationDate = DateFormatter.format(stat.getModificationTimestamp());
                                cursorInfo.cursorsLedgerId = pbCursorInfo.getCursorsLedgerId();
                                if (pbCursorInfo.hasMarkDeleteLedgerId()) {
                                    cursorInfo.markDelete = new PositionInfo();
                                    cursorInfo.markDelete.ledgerId = pbCursorInfo.getMarkDeleteLedgerId();
                                    cursorInfo.markDelete.entryId = pbCursorInfo.getMarkDeleteEntryId();
                                }
                                if (pbCursorInfo.getPropertiesCount() > 0) {
                                    cursorInfo.properties = Maps.newTreeMap();
                                    for (int i = 0; i < pbCursorInfo.getPropertiesCount(); i++) {
                                        LongProperty property = pbCursorInfo.getProperties(i);
                                        cursorInfo.properties.put(property.getName(), property.getValue());
                                    }
                                }
                                if (pbCursorInfo.getIndividualDeletedMessagesCount() > 0) {
                                    cursorInfo.individualDeletedMessages = new ArrayList<>();
                                    for (int i = 0; i < pbCursorInfo.getIndividualDeletedMessagesCount(); i++) {
                                        MessageRange range = pbCursorInfo.getIndividualDeletedMessages(i);
                                        MessageRangeInfo rangeInfo = new MessageRangeInfo();
                                        rangeInfo.from.ledgerId = range.getLowerEndpoint().getLedgerId();
                                        rangeInfo.from.entryId = range.getLowerEndpoint().getEntryId();
                                        rangeInfo.to.ledgerId = range.getUpperEndpoint().getLedgerId();
                                        rangeInfo.to.entryId = range.getUpperEndpoint().getEntryId();
                                        cursorInfo.individualDeletedMessages.add(rangeInfo);
                                    }
                                }
                                info.cursors.put(cursorName, cursorInfo);
                                cursorFuture.complete(null);
                            }

                            @Override
                            public void operationFailed(MetaStoreException e) {
                                cursorFuture.completeExceptionally(e);
                            }
                        });
                    }
                    Futures.waitForAll(cursorsFutures).thenRun(() -> {
                        // Completed all the cursors info
                        callback.getInfoComplete(info, ctx);
                    }).exceptionally((ex) -> {
                        callback.getInfoFailed(getManagedLedgerException(ex.getCause()), ctx);
                        return null;
                    });
                }

                @Override
                public void operationFailed(MetaStoreException e) {
                    callback.getInfoFailed(e, ctx);
                }
            });
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            callback.getInfoFailed(e, ctx);
        }
    });
}

Example 15 with Stat

use of org.apache.pulsar.metadata.api.Stat in project pulsar by apache.

the class RocksdbMetadataStoreTest method testOpenDbWithConfigFile.

@Test
public void testOpenDbWithConfigFile() throws Exception {
    MetadataStore store;
    Path tempDir;
    tempDir = Files.createTempDirectory("RocksdbMetadataStoreTest");
    log.info("Temp dir:{}", tempDir.toAbsolutePath());
    String optionFilePath = getClass().getClassLoader().getResource("rocksdb_option_file_example.ini").getPath();
    log.info("optionFilePath={}", optionFilePath);
    store = MetadataStoreFactory.create("rocksdb:" + tempDir.toAbsolutePath(), MetadataStoreConfig.builder().configFilePath(optionFilePath).build());
    Assert.assertTrue(store instanceof RocksdbMetadataStore);
    String path = "/test";
    byte[] data = "data".getBytes();
    // test put
    CompletableFuture<Stat> f = store.put(path, data, Optional.of(-1L));
    CompletableFuture<Stat> failedPut = store.put(path, data, Optional.of(100L));
    Assert.expectThrows(MetadataStoreException.BadVersionException.class, () -> {
        try {
            failedPut.get();
        } catch (ExecutionException t) {
            throw t.getCause();
        }
    });
    Assert.assertNotNull(f.get());
    log.info("put result:{}", f.get());
    Assert.assertNotNull(store.put(path + "/a", data, Optional.of(-1L)));
    Assert.assertNotNull(store.put(path + "/b", data, Optional.of(-1L)));
    Assert.assertNotNull(store.put(path + "/c", data, Optional.of(-1L)));
    // reopen db
    store.close();
    store = MetadataStoreFactory.create("rocksdb:" + tempDir.toAbsolutePath(), MetadataStoreConfig.builder().configFilePath(optionFilePath).build());
    // test get
    CompletableFuture<Optional<GetResult>> readResult = store.get(path);
    Assert.assertNotNull(readResult.get());
    Assert.assertTrue(readResult.get().isPresent());
    GetResult r = readResult.get().get();
    Assert.assertEquals(path, r.getStat().getPath());
    Assert.assertEquals(0, r.getStat().getVersion());
    Assert.assertEquals(data, r.getValue());
    store.close();
    FileUtils.deleteQuietly(tempDir.toFile());
}