Examples with EntryLogScanner org.apache.bookkeeper.bookie.storage.EntryLogScanner used on opensource projects

Example 1 with EntryLogScanner

use of org.apache.bookkeeper.bookie.storage.EntryLogScanner in project bookkeeper by apache.

the class ReadLogCommand method scanEntryLogForPositionRange.

/**
 * Scan over an entry log file for entries in the given position range.
 *
 * @param logId Entry Log File id.
 * @param rangeStartPos Start position of the entry we are looking for
 * @param rangeEndPos End position of the entry we are looking for (-1 for till the end of the entrylog)
 * @param printMsg Whether printing the entry data.
 * @throws Exception
 */
private void scanEntryLogForPositionRange(ServerConfiguration conf, long logId, final long rangeStartPos, final long rangeEndPos, final boolean printMsg) throws Exception {
    LOG.info("Scan entry log {} ({}.log) for PositionRange: {} - {}", logId, Long.toHexString(logId), rangeStartPos, rangeEndPos);
    final MutableBoolean entryFound = new MutableBoolean(false);
    scanEntryLog(conf, logId, new EntryLogScanner() {

        private MutableBoolean stopScanning = new MutableBoolean(false);

        @Override
        public boolean accept(long ledgerId) {
            return !stopScanning.booleanValue();
        }

        @Override
        public void process(long ledgerId, long entryStartPos, ByteBuf entry) throws IOException {
            if (!stopScanning.booleanValue()) {
                if ((rangeEndPos != -1) && (entryStartPos > rangeEndPos)) {
                    stopScanning.setValue(true);
                } else {
                    int entrySize = entry.readableBytes();
                    /**
                     * entrySize of an entry (inclusive of payload and
                     * header) value is stored as int value in log file, but
                     * it is not counted in the entrySize, hence for calculating
                     * the end position of the entry we need to add additional
                     * 4 (intsize of entrySize). Please check
                     * EntryLogger.scanEntryLog.
                     */
                    long entryEndPos = entryStartPos + entrySize + 4 - 1;
                    if (((rangeEndPos == -1) || (entryStartPos <= rangeEndPos)) && (rangeStartPos <= entryEndPos)) {
                        FormatUtil.formatEntry(entryStartPos, entry, printMsg, ledgerIdFormatter, entryFormatter);
                        entryFound.setValue(true);
                    }
                }
            }
        }
    });
    if (!entryFound.booleanValue()) {
        LOG.info("Entry log {} ({}.log) doesn't has any entry in the range {} - {}. " + "Probably the position range, you have provided is lesser than the LOGFILE_HEADER_SIZE (1024) " + "or greater than the current log filesize.", logId, Long.toHexString(logId), rangeStartPos, rangeEndPos);
    }
}

Example 2 with EntryLogScanner

use of org.apache.bookkeeper.bookie.storage.EntryLogScanner in project bookkeeper by apache.

the class ReadLogCommand method scanEntryLogForSpecificEntry.

/**
 * Scan over an entry log file for a particular entry.
 *
 * @param logId Entry Log File id.
 * @param ledgerId id of the ledger
 * @param entryId entryId of the ledger we are looking for (-1 for all of the entries of the ledger)
 * @param printMsg Whether printing the entry data.
 * @throws Exception
 */
private void scanEntryLogForSpecificEntry(ServerConfiguration conf, long logId, final long ledgerId, final long entryId, final boolean printMsg) throws Exception {
    LOG.info("Scan entry log {} ({}.log) for LedgerId {} {}", logId, Long.toHexString(logId), ledgerId, ((entryId == -1) ? "" : " for EntryId " + entryId));
    final MutableBoolean entryFound = new MutableBoolean(false);
    scanEntryLog(conf, logId, new EntryLogScanner() {

        @Override
        public boolean accept(long candidateLedgerId) {
            return ((candidateLedgerId == ledgerId) && ((!entryFound.booleanValue()) || (entryId == -1)));
        }

        @Override
        public void process(long candidateLedgerId, long startPos, ByteBuf entry) {
            long entrysLedgerId = entry.getLong(entry.readerIndex());
            long entrysEntryId = entry.getLong(entry.readerIndex() + 8);
            if ((candidateLedgerId == entrysLedgerId) && (candidateLedgerId == ledgerId) && ((entrysEntryId == entryId) || (entryId == -1))) {
                entryFound.setValue(true);
                FormatUtil.formatEntry(startPos, entry, printMsg, ledgerIdFormatter, entryFormatter);
            }
        }
    });
    if (!entryFound.booleanValue()) {
        LOG.info("LedgerId {} {} is not available in the entry log {} ({}.log)", ledgerId, ((entryId == -1) ? "" : " EntryId " + entryId), logId, Long.toHexString(logId));
    }
}

Example 3 with EntryLogScanner

use of org.apache.bookkeeper.bookie.storage.EntryLogScanner in project bookkeeper by apache.

the class InterleavedStorageRegenerateIndexOp method initiate.

public void initiate(boolean dryRun) throws IOException {
    LOG.info("Starting index rebuilding");
    DiskChecker diskChecker = BookieResources.createDiskChecker(conf);
    LedgerDirsManager ledgerDirsManager = BookieResources.createLedgerDirsManager(conf, diskChecker, NullStatsLogger.INSTANCE);
    LedgerDirsManager indexDirsManager = BookieResources.createIndexDirsManager(conf, diskChecker, NullStatsLogger.INSTANCE, ledgerDirsManager);
    DefaultEntryLogger entryLogger = new DefaultEntryLogger(conf, ledgerDirsManager);
    final LedgerCache ledgerCache;
    if (dryRun) {
        ledgerCache = new DryRunLedgerCache();
    } else {
        ledgerCache = new LedgerCacheImpl(conf, new SnapshotMap<Long, Boolean>(), indexDirsManager, NullStatsLogger.INSTANCE);
    }
    Set<Long> entryLogs = entryLogger.getEntryLogsSet();
    int totalEntryLogs = entryLogs.size();
    int completedEntryLogs = 0;
    long startTime = System.nanoTime();
    LOG.info("Scanning {} entry logs", totalEntryLogs);
    Map<Long, RecoveryStats> stats = new HashMap<>();
    for (long entryLogId : entryLogs) {
        LOG.info("Scanning {}", entryLogId);
        entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {

            @Override
            public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
                long entryId = entry.getLong(8);
                stats.computeIfAbsent(ledgerId, (ignore) -> new RecoveryStats()).registerEntry(entryId);
                // Actual location indexed is pointing past the entry size
                long location = (entryLogId << 32L) | (offset + 4);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1));
                }
                if (!ledgerCache.ledgerExists(ledgerId)) {
                    ledgerCache.setMasterKey(ledgerId, masterKey);
                    ledgerCache.setFenced(ledgerId);
                }
                ledgerCache.putEntryOffset(ledgerId, entryId, location);
            }

            @Override
            public boolean accept(long ledgerId) {
                return ledgerIds.contains(ledgerId);
            }
        });
        ledgerCache.flushLedger(true);
        ++completedEntryLogs;
        LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs);
    }
    LOG.info("Rebuilding indices done");
    for (long ledgerId : ledgerIds) {
        RecoveryStats ledgerStats = stats.get(ledgerId);
        if (ledgerStats == null || ledgerStats.getNumEntries() == 0) {
            LOG.info(" {} - No entries found", ledgerId);
        } else {
            LOG.info(" {} - Found {} entries, from {} to {}", ledgerId, ledgerStats.getNumEntries(), ledgerStats.getFirstEntry(), ledgerStats.getLastEntry());
        }
    }
    LOG.info("Total time: {}", DurationFormatUtils.formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}

Example 4 with EntryLogScanner

use of org.apache.bookkeeper.bookie.storage.EntryLogScanner in project bookkeeper by apache.

the class DefaultEntryLogger method extractEntryLogMetadataByScanning.

private EntryLogMetadata extractEntryLogMetadataByScanning(long entryLogId, AbstractLogCompactor.Throttler throttler) throws IOException {
    final EntryLogMetadata meta = new EntryLogMetadata(entryLogId);
    // Read through the entry log file and extract the entry log meta
    scanEntryLog(entryLogId, new EntryLogScanner() {

        @Override
        public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
            if (throttler != null) {
                throttler.acquire(entry.readableBytes());
            }
            // add new entry size of a ledger to entry log meta
            meta.addLedgerSize(ledgerId, entry.readableBytes() + 4);
        }

        @Override
        public boolean accept(long ledgerId) {
            return ledgerId >= 0;
        }
    });
    if (LOG.isDebugEnabled()) {
        LOG.debug("Retrieved entry log meta data entryLogId: {}, meta: {}", entryLogId, meta);
    }
    return meta;
}

Example 5 with EntryLogScanner

use of org.apache.bookkeeper.bookie.storage.EntryLogScanner in project bookkeeper by apache.

the class LocationsIndexRebuildOp method initiate.

public void initiate() throws IOException {
    LOG.info("Starting locations index rebuilding");
    // Move locations index to a backup directory
    String basePath = BookieImpl.getCurrentDirectory(conf.getLedgerDirs()[0]).toString();
    Path currentPath = FileSystems.getDefault().getPath(basePath, "locations");
    String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(new Date());
    Path backupPath = FileSystems.getDefault().getPath(basePath, "locations.BACKUP-" + timestamp);
    Files.move(currentPath, backupPath);
    LOG.info("Created locations index backup at {}", backupPath);
    long startTime = System.nanoTime();
    DefaultEntryLogger entryLogger = new DefaultEntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold())));
    Set<Long> entryLogs = entryLogger.getEntryLogsSet();
    Set<Long> activeLedgers = getActiveLedgers(conf, KeyValueStorageRocksDB.factory, basePath);
    LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size());
    KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(basePath, "locations", DbConfigType.Default, conf);
    int totalEntryLogs = entryLogs.size();
    int completedEntryLogs = 0;
    LOG.info("Scanning {} entry logs", totalEntryLogs);
    for (long entryLogId : entryLogs) {
        entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {

            @Override
            public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
                long entryId = entry.getLong(8);
                // Actual location indexed is pointing past the entry size
                long location = (entryLogId << 32L) | (offset + 4);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1));
                }
                // Update the ledger index page
                LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId);
                LongWrapper value = LongWrapper.get(location);
                newIndex.put(key.array, value.array);
            }

            @Override
            public boolean accept(long ledgerId) {
                return activeLedgers.contains(ledgerId);
            }
        });
        ++completedEntryLogs;
        LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs);
    }
    newIndex.sync();
    newIndex.close();
    LOG.info("Rebuilding index is done. Total time: {}", DurationFormatUtils.formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}