Examples with LedgerInfo org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo used on opensource projects

Example 6 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerTest method testBackwardCompatiblityForMeta.

@Test
public void testBackwardCompatiblityForMeta() throws Exception {
    final ManagedLedgerInfo[] storedMLInfo = new ManagedLedgerInfo[3];
    final Stat[] versions = new Stat[1];
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
    ManagedLedgerConfig conf = new ManagedLedgerConfig();
    conf.setMaxEntriesPerLedger(1);
    conf.setRetentionSizeInMB(10);
    conf.setRetentionTime(1, TimeUnit.HOURS);
    ManagedLedger ml = factory.open("backward_test_ledger", conf);
    ml.openCursor("c1");
    ml.addEntry("msg1".getBytes());
    ml.addEntry("msg2".getBytes());
    ml.close();
    MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
    CountDownLatch l1 = new CountDownLatch(1);
    // obtain the ledger info
    store.getManagedLedgerInfo("backward_test_ledger", new MetaStoreCallback<ManagedLedgerInfo>() {

        @Override
        public void operationComplete(ManagedLedgerInfo result, Stat version) {
            storedMLInfo[0] = result;
            versions[0] = version;
            l1.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            fail("on get ManagedLedgerInfo backward_test_ledger");
        }
    });
    l1.await();
    ManagedLedgerInfo.Builder builder1 = ManagedLedgerInfo.newBuilder();
    // simulate test for old ledger with no timestampl
    for (LedgerInfo info : storedMLInfo[0].getLedgerInfoList()) {
        LedgerInfo noTimestamp = ManagedLedgerInfo.LedgerInfo.newBuilder().mergeFrom(info).clearTimestamp().build();
        assertFalse(noTimestamp.hasTimestamp(), "expected old version info with no timestamp");
        builder1.addLedgerInfo(noTimestamp);
    }
    storedMLInfo[1] = builder1.build();
    // test timestamp on new ledger
    CountDownLatch l2 = new CountDownLatch(1);
    store.asyncUpdateLedgerIds("backward_test_ledger", storedMLInfo[1], versions[0], new MetaStoreCallback<Void>() {

        @Override
        public void operationComplete(Void result, Stat version) {
            l2.countDown();
        }

        @Override
        public void operationFailed(MetaStoreException e) {
            fail("on asyncUpdateLedgerIds");
        }
    });
    // verify that after update ledgers have timestamp
    ManagedLedgerImpl newVersionLedger = (ManagedLedgerImpl) factory.open("backward_test_ledger", conf);
    List<LedgerInfo> mlInfo = newVersionLedger.getLedgersInfoAsList();
    assertTrue(mlInfo.stream().allMatch(new Predicate<LedgerInfo>() {

        @Override
        public boolean test(LedgerInfo ledgerInfo) {
            return ledgerInfo.hasTimestamp();
        }
    }));
}

Example 7 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method ledgerClosed.

// //////////////////////////////////////////////////////////////////////
// Private helpers
synchronized void ledgerClosed(final LedgerHandle lh) {
    final State state = STATE_UPDATER.get(this);
    if (state == State.ClosingLedger || state == State.LedgerOpened) {
        STATE_UPDATER.set(this, State.ClosedLedger);
    } else {
        // just once
        return;
    }
    long entriesInLedger = lh.getLastAddConfirmed() + 1;
    if (log.isDebugEnabled()) {
        log.debug("[{}] Ledger has been closed id={} entries={}", name, lh.getId(), entriesInLedger);
    }
    if (entriesInLedger > 0) {
        LedgerInfo info = LedgerInfo.newBuilder().setLedgerId(lh.getId()).setEntries(entriesInLedger).setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build();
        ledgers.put(lh.getId(), info);
    } else {
        // The last ledger was empty, so we can discard it
        ledgers.remove(lh.getId());
        mbean.startDataLedgerDeleteOp();
        bookKeeper.asyncDeleteLedger(lh.getId(), (rc, ctx) -> {
            mbean.endDataLedgerDeleteOp();
            log.info("[{}] Delete complete for empty ledger {}. rc={}", name, lh.getId(), rc);
        }, null);
    }
    trimConsumedLedgersInBackground();
    if (!pendingAddEntries.isEmpty()) {
        // Need to create a new ledger to write pending entries
        if (log.isDebugEnabled()) {
            log.debug("[{}] Creating a new ledger", name);
        }
        STATE_UPDATER.set(this, State.CreatingLedger);
        this.lastLedgerCreationInitiationTimestamp = System.nanoTime();
        mbean.startDataLedgerCreateOp();
        bookKeeper.asyncCreateLedger(config.getEnsembleSize(), config.getWriteQuorumSize(), config.getAckQuorumSize(), config.getDigestType(), config.getPassword(), this, null);
    }
}

Example 8 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method getEstimatedBacklogSize.

@Override
public long getEstimatedBacklogSize() {
    PositionImpl pos = getMarkDeletePositionOfSlowestConsumer();
    while (true) {
        if (pos == null) {
            return 0;
        }
        long size = 0;
        final long slowestConsumerLedgerId = pos.getLedgerId();
        // Subtract size of ledgers that were already fully consumed but not trimmed yet
        synchronized (this) {
            size = getTotalSize();
            size -= ledgers.values().stream().filter(li -> li.getLedgerId() < slowestConsumerLedgerId).mapToLong(li -> li.getSize()).sum();
        }
        LedgerInfo ledgerInfo = null;
        synchronized (this) {
            ledgerInfo = ledgers.get(pos.getLedgerId());
        }
        if (ledgerInfo == null) {
            // ledger was removed
            if (pos.compareTo(getMarkDeletePositionOfSlowestConsumer()) == 0) {
                // position still has not moved
                return size;
            }
            // retry with new slowest consumer
            pos = getMarkDeletePositionOfSlowestConsumer();
            continue;
        }
        long numEntries = pos.getEntryId();
        if (ledgerInfo.getEntries() == 0) {
            size -= consumedLedgerSize(currentLedgerSize, currentLedgerEntries, numEntries);
            return size;
        } else {
            size -= consumedLedgerSize(ledgerInfo.getSize(), ledgerInfo.getEntries(), numEntries);
            return size;
        }
    }
}

Example 9 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method asyncReadEntries.

void asyncReadEntries(OpReadEntry opReadEntry) {
    final State state = STATE_UPDATER.get(this);
    if (state == State.Fenced || state == State.Closed) {
        opReadEntry.readEntriesFailed(new ManagedLedgerFencedException(), opReadEntry.ctx);
        return;
    }
    long ledgerId = opReadEntry.readPosition.getLedgerId();
    LedgerHandle currentLedger = this.currentLedger;
    if (ledgerId == currentLedger.getId()) {
        // Current writing ledger is not in the cache (since we don't want
        // it to be automatically evicted), and we cannot use 2 different
        // ledger handles (read & write)for the same ledger.
        internalReadFromLedger(currentLedger, opReadEntry);
    } else {
        LedgerInfo ledgerInfo = ledgers.get(ledgerId);
        if (ledgerInfo == null || ledgerInfo.getEntries() == 0) {
            // Cursor is pointing to a empty ledger, there's no need to try opening it. Skip this ledger and
            // move to the next one
            opReadEntry.updateReadPosition(new PositionImpl(opReadEntry.readPosition.getLedgerId() + 1, 0));
            opReadEntry.checkReadCompletion();
            return;
        }
        // Get a ledger handle to read from
        getLedgerHandle(ledgerId).thenAccept(ledger -> {
            internalReadFromLedger(ledger, opReadEntry);
        }).exceptionally(ex -> {
            log.error("[{}] Error opening ledger for reading at position {} - {}", name, opReadEntry.readPosition, ex.getMessage());
            opReadEntry.readEntriesFailed(new ManagedLedgerException(ex), opReadEntry.ctx);
            return null;
        });
    }
}

Example 10 with LedgerInfo

use of org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo in project pulsar by yahoo.

the class ManagedLedgerImpl method getNumberOfEntries.

/**
     * Get the number of entries between a contiguous range of two positions
     *
     * @param range
     *            the position range
     * @return the count of entries
     */
long getNumberOfEntries(Range<PositionImpl> range) {
    PositionImpl fromPosition = range.lowerEndpoint();
    boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED;
    PositionImpl toPosition = range.upperEndpoint();
    boolean toIncluded = range.upperBoundType() == BoundType.CLOSED;
    if (fromPosition.getLedgerId() == toPosition.getLedgerId()) {
        // If the 2 positions are in the same ledger
        long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1;
        count += fromIncluded ? 1 : 0;
        count += toIncluded ? 1 : 0;
        return count;
    } else {
        long count = 0;
        // If the from & to are pointing to different ledgers, then we need to :
        // 1. Add the entries in the ledger pointed by toPosition
        count += toPosition.getEntryId();
        count += toIncluded ? 1 : 0;
        // 2. Add the entries in the ledger pointed by fromPosition
        LedgerInfo li = ledgers.get(fromPosition.getLedgerId());
        if (li != null) {
            count += li.getEntries() - (fromPosition.getEntryId() + 1);
            count += fromIncluded ? 1 : 0;
        }
        // 3. Add the whole ledgers entries in between
        for (LedgerInfo ls : ledgers.subMap(fromPosition.getLedgerId(), false, toPosition.getLedgerId(), false).values()) {
            count += ls.getEntries();
        }
        return count;
    }
}