Examples with LedgerManager org.apache.bookkeeper.meta.LedgerManager used on opensource projects

Example 6 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class CompactionTest method testExtractMetaFromEntryLogs.

/**
 * Test extractMetaFromEntryLogs optimized method to avoid excess memory usage.
 */
public void testExtractMetaFromEntryLogs() throws Exception {
    // Always run this test with Throttle enabled.
    baseConf.setIsThrottleByBytes(true);
    // restart bookies
    restartBookies(baseConf);
    ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
    File tmpDir = createTempDir("bkTest", ".dir");
    File curDir = Bookie.getCurrentDirectory(tmpDir);
    Bookie.checkDirectoryStructure(curDir);
    conf.setLedgerDirNames(new String[] { tmpDir.toString() });
    LedgerDirsManager dirs = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    final Set<Long> ledgers = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
    LedgerManager manager = getLedgerManager(ledgers);
    CheckpointSource checkpointSource = new CheckpointSource() {

        @Override
        public Checkpoint newCheckpoint() {
            return null;
        }

        @Override
        public void checkpointComplete(Checkpoint checkpoint, boolean compact) throws IOException {
        }
    };
    InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    for (long ledger = 0; ledger <= 10; ledger++) {
        ledgers.add(ledger);
        for (int entry = 1; entry <= 50; entry++) {
            try {
                storage.addEntry(genEntry(ledger, entry, ENTRY_SIZE));
            } catch (IOException e) {
            // ignore exception on failure to add entry.
            }
        }
    }
    storage.flush();
    storage.shutdown();
    storage = new InterleavedLedgerStorage();
    storage.initialize(conf, manager, dirs, dirs, null, checkpointSource, Checkpointer.NULL, NullStatsLogger.INSTANCE);
    long startingEntriesCount = storage.gcThread.entryLogger.getLeastUnflushedLogId() - storage.gcThread.scannedLogId;
    LOG.info("The old Log Entry count is: " + startingEntriesCount);
    Map<Long, EntryLogMetadata> entryLogMetaData = new HashMap<>();
    long finalEntriesCount = storage.gcThread.entryLogger.getLeastUnflushedLogId() - storage.gcThread.scannedLogId;
    LOG.info("The latest Log Entry count is: " + finalEntriesCount);
    assertTrue("The GC did not clean up entries...", startingEntriesCount != finalEntriesCount);
    assertTrue("Entries Count is zero", finalEntriesCount == 0);
}

Example 7 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class CompactionTest method testForceGarbageCollection.

@Test
public void testForceGarbageCollection() throws Exception {
    ServerConfiguration conf = newServerConfiguration();
    conf.setGcWaitTime(60000);
    conf.setMinorCompactionInterval(120000);
    conf.setMajorCompactionInterval(240000);
    LedgerDirsManager dirManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
    CheckpointSource cp = new CheckpointSource() {

        @Override
        public Checkpoint newCheckpoint() {
            // Do nothing.
            return null;
        }

        @Override
        public void checkpointComplete(Checkpoint checkPoint, boolean compact) throws IOException {
        // Do nothing.
        }
    };
    for (File journalDir : conf.getJournalDirs()) {
        Bookie.checkDirectoryStructure(journalDir);
    }
    for (File dir : dirManager.getAllLedgerDirs()) {
        Bookie.checkDirectoryStructure(dir);
    }
    runFunctionWithLedgerManagerFactory(conf, lmf -> {
        try (LedgerManager lm = lmf.newLedgerManager()) {
            InterleavedLedgerStorage storage = new InterleavedLedgerStorage();
            storage.initialize(conf, lm, dirManager, dirManager, null, cp, Checkpointer.NULL, NullStatsLogger.INSTANCE);
            storage.start();
            long startTime = MathUtils.now();
            storage.gcThread.enableForceGC();
            // major
            storage.gcThread.triggerGC().get();
            // minor
            storage.gcThread.triggerGC().get();
            // Minor and Major compaction times should be larger than when we started
            // this test.
            assertTrue("Minor or major compaction did not trigger even on forcing.", storage.gcThread.lastMajorCompactionTime > startTime && storage.gcThread.lastMinorCompactionTime > startTime);
            storage.shutdown();
        } catch (Exception e) {
            throw new UncheckedExecutionException(e.getMessage(), e);
        }
        return null;
    });
}

Example 8 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project pravega by pravega.

the class BookKeeperLogReconcileCommand method execute.

@Override
public void execute() throws Exception {
    ensureArgCount(1);
    int logId = getIntArg(0);
    // Ensure that the Bookkeeper log is disabled; abort otherwise.
    @Cleanup val context = createContext();
    @Cleanup val log = context.logFactory.createDebugLogWrapper(logId);
    // Display a summary of the BookKeeperLog.
    val m = log.fetchMetadata();
    outputLogSummary(logId, m);
    if (m == null || m.isEnabled()) {
        String message = (m == null) ? "BookKeeperLog '%s' does not exist." : "BookKeeperLog '%s' is enabled. Please, disable it before executing this command.";
        output(message, logId);
        return;
    }
    // Once the Bookkeeper log is disabled, list all ledgers from this log. This implies to query all the ledgers
    // in Bookkeeper and filter out the ones related to BookkeeperLog id passed by parameter.
    ClientConfiguration config = new ClientConfiguration().setMetadataServiceUri("zk://" + this.getServiceConfig().getZkURL() + context.bookKeeperConfig.getBkLedgerPath());
    @Cleanup BookKeeper bkClient = BookKeeper.forConfig(config).build();
    @Cleanup LedgerManager manager = bkClient.getLedgerManager();
    LedgerManager.LedgerRangeIterator ledgerRangeIterator = manager.getLedgerRanges(Long.MAX_VALUE);
    List<ReadHandle> candidateLedgers = new ArrayList<>();
    try {
        while (ledgerRangeIterator.hasNext()) {
            LedgerManager.LedgerRange lr = ledgerRangeIterator.next();
            for (long ledgerId : lr.getLedgers()) {
                ReadHandle readHandle = Ledgers.openRead(ledgerId, bkClient, context.bookKeeperConfig);
                if (Ledgers.getBookKeeperLogId(readHandle) == logId) {
                    candidateLedgers.add(readHandle);
                }
            }
        }
        // If there are no candidate ledgers, just return.
        if (candidateLedgers.isEmpty()) {
            output("No candidate ledgers to reconcile.");
            return;
        }
        // Confirm with user prior executing the command.
        output("Candidate ledgers for reconciliation: %s", candidateLedgers.stream().map(String::valueOf).collect(Collectors.joining(",")));
        output("BookKeeperLog '%s' reconciliation is about to be executed.", logId);
        if (!confirmContinue()) {
            output("Not reconciling anything at this time.");
            return;
        }
        // Executing BookkeeperLog reconciliation.
        output("BookKeeperLog '%s': starting ledger reconciliation.", logId);
        log.reconcileLedgers(candidateLedgers);
        output("BookKeeperLog '%s': ledger reconciliation completed.", logId);
    } finally {
        // Closing opened ledgers.
        closeBookkeeperReadHandles(candidateLedgers);
    }
}

Example 9 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class GetLedgerMetaService method handle.

@Override
public HttpServiceResponse handle(HttpServiceRequest request) throws Exception {
    HttpServiceResponse response = new HttpServiceResponse();
    Map<String, String> params = request.getParams();
    if (HttpServer.Method.GET == request.getMethod() && (params != null) && params.containsKey("ledger_id")) {
        Long ledgerId = Long.parseLong(params.get("ledger_id"));
        LedgerManagerFactory mFactory = bookieServer.getBookie().getLedgerManagerFactory();
        LedgerManager manager = mFactory.newLedgerManager();
        // output <ledgerId: ledgerMetadata>
        Map<String, String> output = Maps.newHashMap();
        ListLedgerService.ReadLedgerMetadataCallback cb = new ListLedgerService.ReadLedgerMetadataCallback(ledgerId);
        manager.readLedgerMetadata(ledgerId, cb);
        LedgerMetadata md = cb.get();
        output.put(ledgerId.toString(), new String(md.serialize(), UTF_8));
        manager.close();
        String jsonResponse = JsonUtil.toJson(output);
        LOG.debug("output body:" + jsonResponse);
        response.setBody(jsonResponse);
        response.setCode(HttpServer.StatusCode.OK);
        return response;
    } else {
        response.setCode(HttpServer.StatusCode.NOT_FOUND);
        response.setBody("Not found method. Should be GET method");
        return response;
    }
}

Example 10 with LedgerManager

use of org.apache.bookkeeper.meta.LedgerManager in project bookkeeper by apache.

the class CompactionTest method testSuspendGarbageCollection.

/**
 * Suspend garbage collection when suspendMajor/suspendMinor is set.
 */
@Test
public void testSuspendGarbageCollection() throws Exception {
    ServerConfiguration conf = newServerConfiguration();
    conf.setGcWaitTime(500);
    conf.setMinorCompactionInterval(1);
    conf.setMajorCompactionInterval(2);
    runFunctionWithLedgerManagerFactory(conf, lmf -> {
        try (LedgerManager lm = lmf.newLedgerManager()) {
            testSuspendGarbageCollection(conf, lm);
        } catch (Exception e) {
            throw new UncheckedExecutionException(e.getMessage(), e);
        }
        return null;
    });
}