Examples with SnapshotMap org.apache.bookkeeper.util.SnapshotMap used on opensource projects

Example 1 with SnapshotMap

use of org.apache.bookkeeper.util.SnapshotMap in project bookkeeper by apache.

the class InterleavedStorageRegenerateIndexOp method initiate.

public void initiate(boolean dryRun) throws IOException {
    LOG.info("Starting index rebuilding");
    DiskChecker diskChecker = BookieResources.createDiskChecker(conf);
    LedgerDirsManager ledgerDirsManager = BookieResources.createLedgerDirsManager(conf, diskChecker, NullStatsLogger.INSTANCE);
    LedgerDirsManager indexDirsManager = BookieResources.createIndexDirsManager(conf, diskChecker, NullStatsLogger.INSTANCE, ledgerDirsManager);
    DefaultEntryLogger entryLogger = new DefaultEntryLogger(conf, ledgerDirsManager);
    final LedgerCache ledgerCache;
    if (dryRun) {
        ledgerCache = new DryRunLedgerCache();
    } else {
        ledgerCache = new LedgerCacheImpl(conf, new SnapshotMap<Long, Boolean>(), indexDirsManager, NullStatsLogger.INSTANCE);
    }
    Set<Long> entryLogs = entryLogger.getEntryLogsSet();
    int totalEntryLogs = entryLogs.size();
    int completedEntryLogs = 0;
    long startTime = System.nanoTime();
    LOG.info("Scanning {} entry logs", totalEntryLogs);
    Map<Long, RecoveryStats> stats = new HashMap<>();
    for (long entryLogId : entryLogs) {
        LOG.info("Scanning {}", entryLogId);
        entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {

            @Override
            public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
                long entryId = entry.getLong(8);
                stats.computeIfAbsent(ledgerId, (ignore) -> new RecoveryStats()).registerEntry(entryId);
                // Actual location indexed is pointing past the entry size
                long location = (entryLogId << 32L) | (offset + 4);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1));
                }
                if (!ledgerCache.ledgerExists(ledgerId)) {
                    ledgerCache.setMasterKey(ledgerId, masterKey);
                    ledgerCache.setFenced(ledgerId);
                }
                ledgerCache.putEntryOffset(ledgerId, entryId, location);
            }

            @Override
            public boolean accept(long ledgerId) {
                return ledgerIds.contains(ledgerId);
            }
        });
        ledgerCache.flushLedger(true);
        ++completedEntryLogs;
        LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs);
    }
    LOG.info("Rebuilding indices done");
    for (long ledgerId : ledgerIds) {
        RecoveryStats ledgerStats = stats.get(ledgerId);
        if (ledgerStats == null || ledgerStats.getNumEntries() == 0) {
            LOG.info(" {} - No entries found", ledgerId);
        } else {
            LOG.info(" {} - Found {} entries, from {} to {}", ledgerId, ledgerStats.getNumEntries(), ledgerStats.getFirstEntry(), ledgerStats.getLastEntry());
        }
    }
    LOG.info("Total time: {}", DurationFormatUtils.formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}