Examples with LedgerManager org.apache.bookkeeper.meta.LedgerManager used on opensource projects

Example 11 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class CompactionTest method testCompactionSafety.

/**
 * Test that compaction doesnt add to index without having persisted
 * entrylog first. This is needed because compaction doesn't go through the journal.
 * {@see https://issues.apache.org/jira/browse/BOOKKEEPER-530}
 * {@see https://issues.apache.org/jira/browse/BOOKKEEPER-664}
 */
@Test
public void testCompactionSafety() throws Exception {
    // I dont want the test infrastructure
    tearDown();
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
    LedgerManager manager = getLedgerManager(ledgers);
    File tmpDir = createTempDir("bkTest", ".dir");
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    conf.setEntryLogSizeLimit(EntryLogger.LOGFILE_HEADER_SIZE + 3 * (4 + ENTRY_SIZE));
    conf.setGcWaitTime(100);
    conf.setMinorCompactionThreshold(0.7f);
    conf.setMajorCompactionThreshold(0.0f);
    conf.setMinorCompactionInterval(1);
    conf.setMajorCompactionInterval(10);
    conf.setPageLimit(1);
    CheckpointSource checkpointSource = new CheckpointSource() {

        AtomicInteger idGen = new AtomicInteger(0);

        class MyCheckpoint implements CheckpointSource.Checkpoint {

            int id = idGen.incrementAndGet();

            @Override
            public int compareTo(CheckpointSource.Checkpoint o) {
                if (o == CheckpointSource.Checkpoint.MAX) {
                    return -1;
                } else if (o == CheckpointSource.Checkpoint.MIN) {
                    return 1;
                }
                return id - ((MyCheckpoint) o).id;
            }
        }

        @Override
        public CheckpointSource.Checkpoint newCheckpoint() {
            return new MyCheckpoint();
        }

        public void checkpointComplete(CheckpointSource.Checkpoint checkpoint, boolean compact) throws IOException {
        }
    };
    final byte[] key = "foobar".getBytes();
    File log0 = new File(curDir, "0.log");
    LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    assertFalse("Log shouldnt exist", log0.exists());
    InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    ledgers.add(1L);
    ledgers.add(2L);
    ledgers.add(3L);
    storage.setMasterKey(1, key);
    storage.setMasterKey(2, key);
    storage.setMasterKey(3, key);
    storage.addEntry(genEntry(1, 1, ENTRY_SIZE));
    storage.addEntry(genEntry(2, 1, ENTRY_SIZE));
    storage.addEntry(genEntry(2, 2, ENTRY_SIZE));
    storage.addEntry(genEntry(3, 2, ENTRY_SIZE));
    storage.flush();
    storage.shutdown();
    assertTrue("Log should exist", log0.exists());
    ledgers.remove(2L);
    ledgers.remove(3L);
    storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    storage.start();
    for (int i = 0; i < 10; i++) {
        if (!log0.exists()) {
            break;
        }
        Thread.sleep(1000);
        // simulate sync thread
        storage.entryLogger.flush();
    }
    assertFalse("Log shouldnt exist", log0.exists());
    ledgers.add(4L);
    storage.setMasterKey(4, key);
    // force ledger 1 page to flush
    storage.addEntry(genEntry(4, 1, ENTRY_SIZE));
    storage.shutdown();
    storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    // entry should exist
    storage.getEntry(1, 1);
}

Example 12 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class CompactionTest method testWhenNoLogsToCompact.

/**
 * Test that compaction should execute silently when there is no entry logs
 * to compact. {@see https://issues.apache.org/jira/browse/BOOKKEEPER-700}
 */
@Test
public void testWhenNoLogsToCompact() throws Exception {
    // I dont want the test infrastructure
    tearDown();
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    File tmpDir = createTempDir("bkTest", ".dir");
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
    LedgerManager manager = getLedgerManager(ledgers);
    CheckpointSource checkpointSource = new CheckpointSource() {

        @Override
        public Checkpoint newCheckpoint() {
            return null;
        }

        @Override
        public void checkpointComplete(Checkpoint checkpoint, boolean compact) throws IOException {
        }
    };
    InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    double threshold = 0.1;
    // shouldn't throw exception
    storage.gcThread.doCompactEntryLogs(threshold);
}

Example 13 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class TestHttpService method testListUnderReplicatedLedgerService.

private void testListUnderReplicatedLedgerService(LedgerManagerFactory mFactory) throws Exception {
    startAuditorElector();
    HttpEndpointService listUnderReplicatedLedgerService = bkHttpServiceProvider.provideHttpEndpointService(HttpServer.ApiType.LIST_UNDER_REPLICATED_LEDGER);
    // 1,  PUT, should return error, because only support GET.
    HttpServiceRequest request1 = new HttpServiceRequest(null, HttpServer.Method.PUT, null);
    HttpServiceResponse response1 = listUnderReplicatedLedgerService.handle(request1);
    assertEquals(HttpServer.StatusCode.NOT_FOUND.getValue(), response1.getStatusCode());
    // 2,  GET, should return success.
    // first put ledger into rereplicate. then use api to list ur ledger.
    @Cleanup LedgerManager ledgerManager = mFactory.newLedgerManager();
    @Cleanup final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes());
    LedgerMetadata md = LedgerHandleAdapter.getLedgerMetadata(lh);
    List<BookieSocketAddress> ensemble = md.getEnsembles().get(0L);
    ensemble.set(0, new BookieSocketAddress("1.1.1.1", 1000));
    TestCallbacks.GenericCallbackFuture<Void> cb = new TestCallbacks.GenericCallbackFuture<Void>();
    ledgerManager.writeLedgerMetadata(lh.getId(), md, cb);
    cb.get();
    long underReplicatedLedger = -1;
    for (int i = 0; i < 10; i++) {
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        if (underReplicatedLedger != -1) {
            break;
        }
        Thread.sleep(1000);
    }
    HttpServiceRequest request2 = new HttpServiceRequest(null, HttpServer.Method.GET, null);
    HttpServiceResponse response2 = listUnderReplicatedLedgerService.handle(request2);
    assertEquals(HttpServer.StatusCode.OK.getValue(), response2.getStatusCode());
    stopAuditorElector();
}

Example 14 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class AuditorLedgerCheckerTest method testTriggerAuditorWithNoPendingAuditTask.

@Test
public void testTriggerAuditorWithNoPendingAuditTask() throws Exception {
    // wait for a second so that the initial periodic check finishes
    Thread.sleep(1000);
    int lostBookieRecoveryDelayConfValue = baseConf.getLostBookieRecoveryDelay();
    Auditor auditorBookiesAuditor = getAuditorBookiesAuditor();
    Future<?> auditTask = auditorBookiesAuditor.getAuditTask();
    int lostBookieRecoveryDelayBeforeChange = auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange();
    Assert.assertEquals("auditTask is supposed to be null", null, auditTask);
    Assert.assertEquals("lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", lostBookieRecoveryDelayConfValue, lostBookieRecoveryDelayBeforeChange);
    @Cleanup("shutdown") OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("test-scheduler").numThreads(1).build();
    @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri()));
    driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.of(zkc));
    // there is no easy way to validate if the Auditor has executed Audit process (Auditor.startAudit),
    // without shuttingdown Bookie. To test if by resetting LostBookieRecoveryDelay it does Auditing
    // even when there is no pending AuditTask, following approach is needed.
    // Here we are creating few ledgers ledgermetadata with non-existing bookies as its ensemble.
    // When Auditor does audit it recognizes these ledgers as underreplicated and mark them as
    // under-replicated, since these bookies are not available.
    int numofledgers = 5;
    Random rand = new Random();
    for (int i = 0; i < numofledgers; i++) {
        LedgerMetadata metadata = new LedgerMetadata(3, 2, 2, DigestType.CRC32, "passwd".getBytes());
        ArrayList<BookieSocketAddress> ensemble = new ArrayList<BookieSocketAddress>();
        ensemble.add(new BookieSocketAddress("99.99.99.99:9999"));
        ensemble.add(new BookieSocketAddress("11.11.11.11:1111"));
        ensemble.add(new BookieSocketAddress("88.88.88.88:8888"));
        metadata.addEnsemble(0, ensemble);
        MutableInt ledgerCreateRC = new MutableInt(-1);
        CountDownLatch latch = new CountDownLatch(1);
        long ledgerId = (Math.abs(rand.nextLong())) % 100000000;
        try (LedgerManager lm = driver.getLedgerManagerFactory().newLedgerManager()) {
            lm.createLedgerMetadata(ledgerId, metadata, (rc, result) -> {
                ledgerCreateRC.setValue(rc);
                latch.countDown();
            });
        }
        Assert.assertTrue("Ledger creation should complete within 2 secs", latch.await(2000, TimeUnit.MILLISECONDS));
        Assert.assertEquals("LedgerCreate should succeed and return OK rc value", BKException.Code.OK, ledgerCreateRC.getValue());
        ledgerList.add(ledgerId);
    }
    final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size());
    urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelayBeforeChange);
    assertTrue("Audit should be triggered and created ledgers should be marked as underreplicated", underReplicaLatch.await(2, TimeUnit.SECONDS));
    assertEquals("All the ledgers should be marked as underreplicated", ledgerList.size(), urLedgerList.size());
    auditTask = auditorBookiesAuditor.getAuditTask();
    Assert.assertEquals("auditTask is supposed to be null", null, auditTask);
    Assert.assertEquals("lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", lostBookieRecoveryDelayBeforeChange, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange());
}

Example 15 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class DLAuditor method collectLedgersFromBK.

/**
 * Find leak ledgers phase 1: collect ledgers set.
 */
private Set<Long> collectLedgersFromBK(BookKeeperClient bkc, final ExecutorService executorService) throws IOException {
    LedgerManager lm = BookKeeperAccessor.getLedgerManager(bkc.get());
    final Set<Long> ledgers = new HashSet<Long>();
    final CompletableFuture<Void> doneFuture = FutureUtils.createFuture();
    BookkeeperInternalCallbacks.Processor<Long> collector = new BookkeeperInternalCallbacks.Processor<Long>() {

        @Override
        public void process(Long lid, final AsyncCallback.VoidCallback cb) {
            synchronized (ledgers) {
                ledgers.add(lid);
                if (0 == ledgers.size() % 1000) {
                    logger.info("Collected {} ledgers", ledgers.size());
                }
            }
            executorService.submit(new Runnable() {

                @Override
                public void run() {
                    cb.processResult(BKException.Code.OK, null, null);
                }
            });
        }
    };
    AsyncCallback.VoidCallback finalCb = new AsyncCallback.VoidCallback() {

        @Override
        public void processResult(int rc, String path, Object ctx) {
            if (BKException.Code.OK == rc) {
                doneFuture.complete(null);
            } else {
                doneFuture.completeExceptionally(BKException.create(rc));
            }
        }
    };
    lm.asyncProcessLedgers(collector, finalCb, null, BKException.Code.OK, BKException.Code.ZKException);
    try {
        doneFuture.get();
        logger.info("Collected total {} ledgers", ledgers.size());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new DLInterruptedException("Interrupted on collecting ledgers : ", e);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof IOException) {
            throw (IOException) (e.getCause());
        } else {
            throw new IOException("Failed to collect ledgers : ", e.getCause());
        }
    }
    return ledgers;
}