Examples with LedgerRangeIterator org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator used on opensource projects

Example 6 with LedgerRangeIterator

use of org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator in project bookkeeper by apache.

the class ScanAndCompareGarbageCollector method gc.

@Override
public void gc(GarbageCleaner garbageCleaner) {
    if (null == ledgerManager) {
        // so skip garbage collection
        return;
    }
    try {
        // Get a set of all ledgers on the bookie
        NavigableSet<Long> bkActiveLedgers = Sets.newTreeSet(ledgerStorage.getActiveLedgersInRange(0, Long.MAX_VALUE));
        this.activeLedgerCounter = bkActiveLedgers.size();
        long curTime = MathUtils.now();
        boolean checkOverreplicatedLedgers = (enableGcOverReplicatedLedger && curTime - lastOverReplicatedLedgerGcTimeMillis > gcOverReplicatedLedgerIntervalMillis);
        if (checkOverreplicatedLedgers) {
            zk = ZooKeeperClient.newBuilder().connectString(conf.getZkServers()).sessionTimeoutMs(conf.getZkTimeout()).build();
            // remove all the overreplicated ledgers from the local bookie
            Set<Long> overReplicatedLedgers = removeOverReplicatedledgers(bkActiveLedgers, garbageCleaner);
            if (overReplicatedLedgers.isEmpty()) {
                LOG.info("No over-replicated ledgers found.");
            } else {
                LOG.info("Removed over-replicated ledgers: {}", overReplicatedLedgers);
            }
            lastOverReplicatedLedgerGcTimeMillis = MathUtils.now();
        }
        // Iterate over all the ledger on the metadata store
        LedgerRangeIterator ledgerRangeIterator = ledgerManager.getLedgerRanges();
        Set<Long> ledgersInMetadata = null;
        long start;
        long end = -1;
        boolean done = false;
        while (!done) {
            start = end + 1;
            if (ledgerRangeIterator.hasNext()) {
                LedgerRange lRange = ledgerRangeIterator.next();
                ledgersInMetadata = lRange.getLedgers();
                end = lRange.end();
            } else {
                ledgersInMetadata = new TreeSet<>();
                end = Long.MAX_VALUE;
                done = true;
            }
            Iterable<Long> subBkActiveLedgers = bkActiveLedgers.subSet(start, true, end, true);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Active in metadata {}, Active in bookie {}", ledgersInMetadata, subBkActiveLedgers);
            }
            for (Long bkLid : subBkActiveLedgers) {
                if (!ledgersInMetadata.contains(bkLid)) {
                    if (verifyMetadataOnGc) {
                        CountDownLatch latch = new CountDownLatch(1);
                        final AtomicInteger metaRC = new AtomicInteger(0);
                        ledgerManager.readLedgerMetadata(bkLid, (int rc, LedgerMetadata x) -> {
                            metaRC.set(rc);
                            latch.countDown();
                        });
                        latch.await();
                        if (metaRC.get() != BKException.Code.NoSuchLedgerExistsException) {
                            LOG.warn("Ledger {} Missing in metadata list, but ledgerManager returned rc: {}.", bkLid, metaRC.get());
                            continue;
                        }
                    }
                    deletedLedgerCounter.inc();
                    garbageCleaner.clean(bkLid);
                }
            }
        }
    } catch (Throwable t) {
        // ignore exception, collecting garbage next time
        LOG.warn("Exception when iterating over the metadata {}", t);
    } finally {
        if (zk != null) {
            try {
                zk.close();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                LOG.error("Error closing zk session", e);
            }
            zk = null;
        }
    }
}

Example 7 with LedgerRangeIterator

use of org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersNotLast.

@Test
public void testGcLedgersNotLast() throws Exception {
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final List<Long> cleaned = new ArrayList<Long>();
    // Create enough ledgers to span over 4 ranges in the hierarchical ledger manager implementation
    final int numLedgers = 30001;
    createLedgers(numLedgers, createdLedgers);
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(), new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    SortedSet<Long> scannedLedgers = new TreeSet<Long>();
    LedgerRangeIterator iterator = getLedgerManager().getLedgerRanges();
    while (iterator.hasNext()) {
        LedgerRange ledgerRange = iterator.next();
        scannedLedgers.addAll(ledgerRange.getLedgers());
    }
    assertEquals(createdLedgers, scannedLedgers);
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
    long first = createdLedgers.first();
    removeLedger(first);
    garbageCollector.gc(cleaner);
    assertEquals("Should have cleaned something", 1, cleaned.size());
    assertEquals("Should have cleaned first ledger" + first, (long) first, (long) cleaned.get(0));
}

Example 8 with LedgerRangeIterator

use of org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersIfLedgerManagerIteratorFails.

/*
     * in this test scenario no created ledger is deleted, but ledgeriterator is screwed up and returns hasNext to be
     * false and next to be null. So even in this case it is expected not to clean any ledger's data.
     *
     * This testcase is needed for validating fix of bug - W-4292747.
     *
     * ScanAndCompareGarbageCollector/GC should clean data of ledger only if both the LedgerManager.getLedgerRanges says
     * that ledger is not existing and also ledgerManager.readLedgerMetadata fails with error
     * NoSuchLedgerExistsException.
     *
     */
@Test
public void testGcLedgersIfLedgerManagerIteratorFails() throws Exception {
    baseConf.setVerifyMetadataOnGc(true);
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final SortedSet<Long> cleaned = Collections.synchronizedSortedSet(new TreeSet<Long>());
    // Create few ledgers
    final int numLedgers = 5;
    createLedgers(numLedgers, createdLedgers);
    LedgerManager mockLedgerManager = new CleanupLedgerManager(getLedgerManager()) {

        @Override
        public LedgerRangeIterator getLedgerRanges() {
            return new LedgerRangeIterator() {

                @Override
                public LedgerRange next() throws IOException {
                    return null;
                }

                @Override
                public boolean hasNext() throws IOException {
                    return false;
                }
            };
        }
    };
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(mockLedgerManager, new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    validateLedgerRangeIterator(createdLedgers);
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
}

Example 9 with LedgerRangeIterator

use of org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator in project bookkeeper by apache.

the class GcLedgersTest method validateLedgerRangeIterator.

public void validateLedgerRangeIterator(SortedSet<Long> createdLedgers) throws IOException {
    SortedSet<Long> scannedLedgers = new TreeSet<Long>();
    LedgerRangeIterator iterator = getLedgerManager().getLedgerRanges();
    while (iterator.hasNext()) {
        LedgerRange ledgerRange = iterator.next();
        scannedLedgers.addAll(ledgerRange.getLedgers());
    }
    assertEquals(createdLedgers, scannedLedgers);
}

Example 10 with LedgerRangeIterator

use of org.apache.bookkeeper.meta.LedgerManager.LedgerRangeIterator in project bookkeeper by apache.

the class HierarchicalLedgerManager method getLedgerRanges.

@Override
public LedgerRangeIterator getLedgerRanges() {
    LedgerRangeIterator legacyLedgerRangeIterator = legacyLM.getLedgerRanges();
    LedgerRangeIterator longLedgerRangeIterator = longLM.getLedgerRanges();
    return new HierarchicalLedgerRangeIterator(legacyLedgerRangeIterator, longLedgerRangeIterator);
}