Examples with EntryLogScanner org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner used on opensource projects

Example 1 with EntryLogScanner

use of org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner in project bookkeeper by apache.

the class LocationsIndexRebuildOp method initiate.

public void initiate() throws IOException {
    LOG.info("Starting index rebuilding");
    // Move locations index to a backup directory
    String basePath = Bookie.getCurrentDirectory(conf.getLedgerDirs()[0]).toString();
    Path currentPath = FileSystems.getDefault().getPath(basePath, "locations");
    String timestamp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").format(new Date());
    Path backupPath = FileSystems.getDefault().getPath(basePath, "locations.BACKUP-" + timestamp);
    Files.move(currentPath, backupPath);
    LOG.info("Created locations index backup at {}", backupPath);
    long startTime = System.nanoTime();
    EntryLogger entryLogger = new EntryLogger(conf, new LedgerDirsManager(conf, conf.getLedgerDirs(), new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold())));
    Set<Long> entryLogs = entryLogger.getEntryLogsSet();
    String locationsDbPath = FileSystems.getDefault().getPath(basePath, "locations").toFile().toString();
    Set<Long> activeLedgers = getActiveLedgers(conf, KeyValueStorageRocksDB.factory, basePath);
    LOG.info("Found {} active ledgers in ledger manager", activeLedgers.size());
    KeyValueStorage newIndex = KeyValueStorageRocksDB.factory.newKeyValueStorage(locationsDbPath, DbConfigType.Huge, conf);
    int totalEntryLogs = entryLogs.size();
    int completedEntryLogs = 0;
    LOG.info("Scanning {} entry logs", totalEntryLogs);
    for (long entryLogId : entryLogs) {
        entryLogger.scanEntryLog(entryLogId, new EntryLogScanner() {

            @Override
            public void process(long ledgerId, long offset, ByteBuf entry) throws IOException {
                long entryId = entry.getLong(8);
                // Actual location indexed is pointing past the entry size
                long location = (entryLogId << 32L) | (offset + 4);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Rebuilding {}:{} at location {} / {}", ledgerId, entryId, location >> 32, location & (Integer.MAX_VALUE - 1));
                }
                // Update the ledger index page
                LongPairWrapper key = LongPairWrapper.get(ledgerId, entryId);
                LongWrapper value = LongWrapper.get(location);
                newIndex.put(key.array, value.array);
            }

            @Override
            public boolean accept(long ledgerId) {
                return activeLedgers.contains(ledgerId);
            }
        });
        ++completedEntryLogs;
        LOG.info("Completed scanning of log {}.log -- {} / {}", Long.toHexString(entryLogId), completedEntryLogs, totalEntryLogs);
    }
    newIndex.sync();
    newIndex.close();
    LOG.info("Rebuilding index is done. Total time: {}", DurationFormatUtils.formatDurationHMS(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime)));
}

Example 2 with EntryLogScanner

use of org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner in project bookkeeper by apache.

the class BookieShell method scanEntryLogForSpecificEntry.

/**
 * Scan over an entry log file for a particular entry.
 *
 * @param logId Entry Log File id.
 * @param ledgerId id of the ledger
 * @param entryId entryId of the ledger we are looking for (-1 for all of the entries of the ledger)
 * @param printMsg Whether printing the entry data.
 * @throws Exception
 */
protected void scanEntryLogForSpecificEntry(long logId, final long ledgerId, final long entryId, final boolean printMsg) throws Exception {
    System.out.println("Scan entry log " + logId + " (" + Long.toHexString(logId) + ".log)" + " for LedgerId " + ledgerId + ((entryId == -1) ? "" : " for EntryId " + entryId));
    final MutableBoolean entryFound = new MutableBoolean(false);
    scanEntryLog(logId, new EntryLogScanner() {

        @Override
        public boolean accept(long candidateLedgerId) {
            return ((candidateLedgerId == ledgerId) && ((!entryFound.booleanValue()) || (entryId == -1)));
        }

        @Override
        public void process(long candidateLedgerId, long startPos, ByteBuf entry) {
            long entrysLedgerId = entry.getLong(entry.readerIndex());
            long entrysEntryId = entry.getLong(entry.readerIndex() + 8);
            if ((candidateLedgerId == entrysLedgerId) && (candidateLedgerId == ledgerId) && ((entrysEntryId == entryId) || (entryId == -1))) {
                entryFound.setValue(true);
                formatEntry(startPos, entry, printMsg);
            }
        }
    });
    if (!entryFound.booleanValue()) {
        System.out.println("LedgerId " + ledgerId + ((entryId == -1) ? "" : " EntryId " + entryId) + " is not available in the entry log " + logId + " (" + Long.toHexString(logId) + ".log)");
    }
}

Example 3 with EntryLogScanner

use of org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner in project bookkeeper by apache.

the class BookieShell method scanEntryLog.

/**
 * Scan over an entry log file.
 *
 * @param logId
 *          Entry Log File id.
 * @param printMsg
 *          Whether printing the entry data.
 */
protected void scanEntryLog(long logId, final boolean printMsg) throws Exception {
    System.out.println("Scan entry log " + logId + " (" + Long.toHexString(logId) + ".log)");
    scanEntryLog(logId, new EntryLogScanner() {

        @Override
        public boolean accept(long ledgerId) {
            return true;
        }

        @Override
        public void process(long ledgerId, long startPos, ByteBuf entry) {
            formatEntry(startPos, entry, printMsg);
        }
    });
}

Example 4 with EntryLogScanner

use of org.apache.bookkeeper.bookie.EntryLogger.EntryLogScanner in project bookkeeper by apache.

the class BookieShell method scanEntryLogForPositionRange.

/**
 * Scan over an entry log file for entries in the given position range.
 *
 * @param logId Entry Log File id.
 * @param rangeStartPos Start position of the entry we are looking for
 * @param rangeEndPos End position of the entry we are looking for (-1 for till the end of the entrylog)
 * @param printMsg Whether printing the entry data.
 * @throws Exception
 */
protected void scanEntryLogForPositionRange(long logId, final long rangeStartPos, final long rangeEndPos, final boolean printMsg) throws Exception {
    System.out.println("Scan entry log " + logId + " (" + Long.toHexString(logId) + ".log)" + " for PositionRange: " + rangeStartPos + " - " + rangeEndPos);
    final MutableBoolean entryFound = new MutableBoolean(false);
    scanEntryLog(logId, new EntryLogScanner() {

        private MutableBoolean stopScanning = new MutableBoolean(false);

        @Override
        public boolean accept(long ledgerId) {
            return !stopScanning.booleanValue();
        }

        @Override
        public void process(long ledgerId, long entryStartPos, ByteBuf entry) {
            if (!stopScanning.booleanValue()) {
                if ((rangeEndPos != -1) && (entryStartPos > rangeEndPos)) {
                    stopScanning.setValue(true);
                } else {
                    int entrySize = entry.readableBytes();
                    /**
                     * entrySize of an entry (inclusive of payload and
                     * header) value is stored as int value in log file, but
                     * it is not counted in the entrySize, hence for calculating
                     * the end position of the entry we need to add additional
                     * 4 (intsize of entrySize). Please check
                     * EntryLogger.scanEntryLog.
                     */
                    long entryEndPos = entryStartPos + entrySize + 4 - 1;
                    if (((rangeEndPos == -1) || (entryStartPos <= rangeEndPos)) && (rangeStartPos <= entryEndPos)) {
                        formatEntry(entryStartPos, entry, printMsg);
                        entryFound.setValue(true);
                    }
                }
            }
        }
    });
    if (!entryFound.booleanValue()) {
        System.out.println("Entry log " + logId + " (" + Long.toHexString(logId) + ".log) doesn't has any entry in the range " + rangeStartPos + " - " + rangeEndPos + ". Probably the position range, you have provided is lesser than the LOGFILE_HEADER_SIZE (1024) " + "or greater than the current log filesize.");
    }
}