Examples with Counter org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter used on opensource projects

Example 1 with Counter

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter in project hadoop by apache.

the class FSEditLogLoader method loadEditRecords.

long loadEditRecords(EditLogInputStream in, boolean closeOnExit, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException {
    FSDirectory fsDir = fsNamesys.dir;
    EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts = new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
    if (LOG.isTraceEnabled()) {
        LOG.trace("Acquiring write lock to replay edit log");
    }
    fsNamesys.writeLock();
    fsDir.writeLock();
    long[] recentOpcodeOffsets = new long[4];
    Arrays.fill(recentOpcodeOffsets, -1);
    long expectedTxId = expectedStartingTxId;
    long numEdits = 0;
    long lastTxId = in.getLastTxId();
    long numTxns = (lastTxId - expectedStartingTxId) + 1;
    StartupProgress prog = NameNode.getStartupProgress();
    Step step = createStartupProgressStep(in);
    prog.setTotal(Phase.LOADING_EDITS, step, numTxns);
    Counter counter = prog.getCounter(Phase.LOADING_EDITS, step);
    long lastLogTime = monotonicNow();
    long lastInodeId = fsNamesys.dir.getLastInodeId();
    try {
        while (true) {
            try {
                FSEditLogOp op;
                try {
                    op = in.readOp();
                    if (op == null) {
                        break;
                    }
                } catch (Throwable e) {
                    // Handle a problem with our input
                    check203UpgradeFailure(in.getVersion(true), e);
                    String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId);
                    FSImage.LOG.error(errorMessage, e);
                    if (recovery == null) {
                        // recovery mode.
                        throw new EditLogInputException(errorMessage, e, numEdits);
                    }
                    MetaRecoveryContext.editLogLoaderPrompt("We failed to read txId " + expectedTxId, recovery, "skipping the bad section in the log");
                    in.resync();
                    continue;
                }
                recentOpcodeOffsets[(int) (numEdits % recentOpcodeOffsets.length)] = in.getPosition();
                if (op.hasTransactionId()) {
                    if (op.getTransactionId() > expectedTxId) {
                        MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be a gap in the edit log.  We expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "ignoring missing " + " transaction IDs");
                    } else if (op.getTransactionId() < expectedTxId) {
                        MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be an out-of-order edit in the edit log.  We " + "expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "skipping the out-of-order edit");
                        continue;
                    }
                }
                try {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("op=" + op + ", startOpt=" + startOpt + ", numEdits=" + numEdits + ", totalEdits=" + totalEdits);
                    }
                    long inodeId = applyEditLogOp(op, fsDir, startOpt, in.getVersion(true), lastInodeId);
                    if (lastInodeId < inodeId) {
                        lastInodeId = inodeId;
                    }
                } catch (RollingUpgradeOp.RollbackException e) {
                    throw e;
                } catch (Throwable e) {
                    LOG.error("Encountered exception on operation " + op, e);
                    if (recovery == null) {
                        throw e instanceof IOException ? (IOException) e : new IOException(e);
                    }
                    MetaRecoveryContext.editLogLoaderPrompt("Failed to " + "apply edit log operation " + op + ": error " + e.getMessage(), recovery, "applying edits");
                }
                // Now that the operation has been successfully decoded and
                // applied, update our bookkeeping.
                incrOpCount(op.opCode, opCounts, step, counter);
                if (op.hasTransactionId()) {
                    lastAppliedTxId = op.getTransactionId();
                    expectedTxId = lastAppliedTxId + 1;
                } else {
                    expectedTxId = lastAppliedTxId = expectedStartingTxId;
                }
                // log progress
                if (op.hasTransactionId()) {
                    long now = monotonicNow();
                    if (now - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
                        long deltaTxId = lastAppliedTxId - expectedStartingTxId + 1;
                        int percent = Math.round((float) deltaTxId / numTxns * 100);
                        LOG.info("replaying edit log: " + deltaTxId + "/" + numTxns + " transactions completed. (" + percent + "%)");
                        lastLogTime = now;
                    }
                }
                numEdits++;
                totalEdits++;
            } catch (RollingUpgradeOp.RollbackException e) {
                LOG.info("Stopped at OP_START_ROLLING_UPGRADE for rollback.");
                break;
            } catch (MetaRecoveryContext.RequestStopException e) {
                MetaRecoveryContext.LOG.warn("Stopped reading edit log at " + in.getPosition() + "/" + in.length());
                break;
            }
        }
    } finally {
        fsNamesys.dir.resetLastInodeId(lastInodeId);
        if (closeOnExit) {
            in.close();
        }
        fsDir.writeUnlock();
        fsNamesys.writeUnlock("loadEditRecords");
        if (LOG.isTraceEnabled()) {
            LOG.trace("replaying edit log finished");
        }
        if (FSImage.LOG.isDebugEnabled()) {
            dumpOpCounts(opCounts);
        }
    }
    return numEdits;
}