Examples with ScanAndCompareGarbageCollector org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector used on opensource projects

Example 1 with ScanAndCompareGarbageCollector

use of org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers.

/*
     * In this test scenario all the created ledgers are deleted, but LedgerManager.readLedgerMetadata fails with
     * ZKException. So even in this case, GarbageCollector.gc shouldn't delete ledgers data.
     *
     * ScanAndCompareGarbageCollector/GC should clean data of ledger only if both the LedgerManager.getLedgerRanges says
     * that ledger is not existing and also ledgerManager.readLedgerMetadata fails with error
     * NoSuchLedgerExistsException, but is shouldn't delete if the readLedgerMetadata fails with any other error.
     */
@Test
public void testGcLedgersIfReadLedgerMetadataFailsForDeletedLedgers() throws Exception {
    baseConf.setVerifyMetadataOnGc(true);
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final SortedSet<Long> cleaned = Collections.synchronizedSortedSet(new TreeSet<Long>());
    // Create few ledgers
    final int numLedgers = 5;
    createLedgers(numLedgers, createdLedgers);
    LedgerManager mockLedgerManager = new CleanupLedgerManager(getLedgerManager()) {

        @Override
        public void readLedgerMetadata(long ledgerId, GenericCallback<LedgerMetadata> readCb) {
            readCb.operationComplete(BKException.Code.ZKException, null);
        }
    };
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(mockLedgerManager, new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    validateLedgerRangeIterator(createdLedgers);
    for (long ledgerId : createdLedgers) {
        removeLedger(ledgerId);
    }
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
}

Example 2 with ScanAndCompareGarbageCollector

use of org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersNotLast.

@Test
public void testGcLedgersNotLast() throws Exception {
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final List<Long> cleaned = new ArrayList<Long>();
    // Create enough ledgers to span over 4 ranges in the hierarchical ledger manager implementation
    final int numLedgers = 30001;
    createLedgers(numLedgers, createdLedgers);
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(), new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    SortedSet<Long> scannedLedgers = new TreeSet<Long>();
    LedgerRangeIterator iterator = getLedgerManager().getLedgerRanges();
    while (iterator.hasNext()) {
        LedgerRange ledgerRange = iterator.next();
        scannedLedgers.addAll(ledgerRange.getLedgers());
    }
    assertEquals(createdLedgers, scannedLedgers);
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
    long first = createdLedgers.first();
    removeLedger(first);
    garbageCollector.gc(cleaner);
    assertEquals("Should have cleaned something", 1, cleaned.size());
    assertEquals("Should have cleaned first ledger" + first, (long) first, (long) cleaned.get(0));
}

Example 3 with ScanAndCompareGarbageCollector

use of org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector in project bookkeeper by apache.

the class GcLedgersTest method testGarbageCollectLedgers.

@Test
public void testGarbageCollectLedgers() throws Exception {
    int numLedgers = 100;
    int numRemovedLedgers = 10;
    final Set<Long> createdLedgers = new HashSet<Long>();
    final Set<Long> removedLedgers = new HashSet<Long>();
    // create 100 ledgers
    createLedgers(numLedgers, createdLedgers);
    Random r = new Random(System.currentTimeMillis());
    final List<Long> tmpList = new ArrayList<Long>();
    tmpList.addAll(createdLedgers);
    Collections.shuffle(tmpList, r);
    // random remove several ledgers
    for (int i = 0; i < numRemovedLedgers; i++) {
        long ledgerId = tmpList.get(i);
        synchronized (removedLedgers) {
            getLedgerManager().removeLedgerMetadata(ledgerId, Version.ANY, new GenericCallback<Void>() {

                @Override
                public void operationComplete(int rc, Void result) {
                    synchronized (removedLedgers) {
                        removedLedgers.notify();
                    }
                }
            });
            removedLedgers.wait();
        }
        removedLedgers.add(ledgerId);
        createdLedgers.remove(ledgerId);
    }
    final CountDownLatch inGcProgress = new CountDownLatch(1);
    final CountDownLatch createLatch = new CountDownLatch(1);
    final CountDownLatch endLatch = new CountDownLatch(2);
    final CompactableLedgerStorage mockLedgerStorage = new MockLedgerStorage();
    TestStatsProvider stats = new TestStatsProvider();
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(), mockLedgerStorage, baseConf, stats.getStatsLogger("gc"));
    Thread gcThread = new Thread() {

        @Override
        public void run() {
            garbageCollector.gc(new GarbageCollector.GarbageCleaner() {

                boolean paused = false;

                @Override
                public void clean(long ledgerId) {
                    try {
                        mockLedgerStorage.deleteLedger(ledgerId);
                    } catch (IOException e) {
                        e.printStackTrace();
                        return;
                    }
                    if (!paused) {
                        inGcProgress.countDown();
                        try {
                            createLatch.await();
                        } catch (InterruptedException ie) {
                            Thread.currentThread().interrupt();
                        }
                        paused = true;
                    }
                    LOG.info("Garbage Collected ledger {}", ledgerId);
                }
            });
            LOG.info("Gc Thread quits.");
            endLatch.countDown();
        }
    };
    Thread createThread = new Thread() {

        @Override
        public void run() {
            try {
                inGcProgress.await();
                // create 10 more ledgers
                createLedgers(10, createdLedgers);
                LOG.info("Finished creating 10 more ledgers.");
                createLatch.countDown();
            } catch (Exception e) {
            }
            LOG.info("Create Thread quits.");
            endLatch.countDown();
        }
    };
    createThread.start();
    gcThread.start();
    endLatch.await();
    // test ledgers
    for (Long ledger : removedLedgers) {
        assertFalse(activeLedgers.containsKey(ledger));
    }
    for (Long ledger : createdLedgers) {
        assertTrue(activeLedgers.containsKey(ledger));
    }
    assertTrue("Wrong DELETED_LEDGER_COUNT", stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == removedLedgers.size());
    assertTrue("Wrong ACTIVE_LEDGER_COUNT", stats.getGauge("gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == createdLedgers.size());
}

Example 4 with ScanAndCompareGarbageCollector

use of org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersOutsideRange.

@Test
public void testGcLedgersOutsideRange() throws Exception {
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final Queue<Long> cleaned = new LinkedList<Long>();
    int numLedgers = 100;
    createLedgers(numLedgers, createdLedgers);
    MockLedgerStorage mockLedgerStorage = new MockLedgerStorage();
    TestStatsProvider stats = new TestStatsProvider();
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(getLedgerManager(), mockLedgerStorage, baseConf, stats.getStatsLogger("gc"));
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
            try {
                mockLedgerStorage.deleteLedger(ledgerId);
            } catch (IOException e) {
                e.printStackTrace();
                fail("Exception from deleteLedger");
            }
        }
    };
    garbageCollector.gc(cleaner);
    assertNull("Should have cleaned nothing", cleaned.poll());
    assertTrue("Wrong DELETED_LEDGER_COUNT", stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 0);
    assertTrue("Wrong ACTIVE_LEDGER_COUNT", stats.getGauge("gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == numLedgers);
    long last = createdLedgers.last();
    removeLedger(last);
    garbageCollector.gc(cleaner);
    assertNotNull("Should have cleaned something", cleaned.peek());
    assertEquals("Should have cleaned last ledger" + last, (long) last, (long) cleaned.poll());
    assertTrue("Wrong DELETED_LEDGER_COUNT", stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 1);
    long first = createdLedgers.first();
    removeLedger(first);
    garbageCollector.gc(cleaner);
    assertNotNull("Should have cleaned something", cleaned.peek());
    assertEquals("Should have cleaned first ledger" + first, (long) first, (long) cleaned.poll());
    assertTrue("Wrong DELETED_LEDGER_COUNT", stats.getCounter("gc." + DELETED_LEDGER_COUNT).get() == 2);
    garbageCollector.gc(cleaner);
    assertTrue("Wrong ACTIVE_LEDGER_COUNT", stats.getGauge("gc." + ACTIVE_LEDGER_COUNT).getSample().intValue() == (numLedgers - 2));
}

Example 5 with ScanAndCompareGarbageCollector

use of org.apache.bookkeeper.bookie.ScanAndCompareGarbageCollector in project bookkeeper by apache.

the class GcLedgersTest method testGcLedgersIfLedgerManagerIteratorFails.

/*
     * in this test scenario no created ledger is deleted, but ledgeriterator is screwed up and returns hasNext to be
     * false and next to be null. So even in this case it is expected not to clean any ledger's data.
     *
     * This testcase is needed for validating fix of bug - W-4292747.
     *
     * ScanAndCompareGarbageCollector/GC should clean data of ledger only if both the LedgerManager.getLedgerRanges says
     * that ledger is not existing and also ledgerManager.readLedgerMetadata fails with error
     * NoSuchLedgerExistsException.
     *
     */
@Test
public void testGcLedgersIfLedgerManagerIteratorFails() throws Exception {
    baseConf.setVerifyMetadataOnGc(true);
    final SortedSet<Long> createdLedgers = Collections.synchronizedSortedSet(new TreeSet<Long>());
    final SortedSet<Long> cleaned = Collections.synchronizedSortedSet(new TreeSet<Long>());
    // Create few ledgers
    final int numLedgers = 5;
    createLedgers(numLedgers, createdLedgers);
    LedgerManager mockLedgerManager = new CleanupLedgerManager(getLedgerManager()) {

        @Override
        public LedgerRangeIterator getLedgerRanges() {
            return new LedgerRangeIterator() {

                @Override
                public LedgerRange next() throws IOException {
                    return null;
                }

                @Override
                public boolean hasNext() throws IOException {
                    return false;
                }
            };
        }
    };
    final GarbageCollector garbageCollector = new ScanAndCompareGarbageCollector(mockLedgerManager, new MockLedgerStorage(), baseConf, NullStatsLogger.INSTANCE);
    GarbageCollector.GarbageCleaner cleaner = new GarbageCollector.GarbageCleaner() {

        @Override
        public void clean(long ledgerId) {
            LOG.info("Cleaned {}", ledgerId);
            cleaned.add(ledgerId);
        }
    };
    validateLedgerRangeIterator(createdLedgers);
    garbageCollector.gc(cleaner);
    assertTrue("Should have cleaned nothing", cleaned.isEmpty());
}