Examples with Step org.apache.hadoop.hdfs.server.namenode.startupprogress.Step used on opensource projects

Example 1 with Step

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.Step in project hadoop by apache.

the class FSEditLogLoader method loadFSEdits.

/**
   * Load an edit log, and apply the changes to the in-memory structure
   * This is where we apply edits that we've been writing to disk all
   * along.
   */
long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException {
    StartupProgress prog = NameNode.getStartupProgress();
    Step step = createStartupProgressStep(edits);
    prog.beginStep(Phase.LOADING_EDITS, step);
    fsNamesys.writeLock();
    try {
        long startTime = monotonicNow();
        FSImage.LOG.info("Start loading edits file " + edits.getName());
        long numEdits = loadEditRecords(edits, false, expectedStartingTxId, startOpt, recovery);
        FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow() - startTime) / 1000 + " seconds");
        return numEdits;
    } finally {
        edits.close();
        fsNamesys.writeUnlock("loadFSEdits");
        prog.endStep(Phase.LOADING_EDITS, step);
    }
}

Example 2 with Step

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.Step in project hadoop by apache.

the class StartupProgressServlet method doGet.

@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    resp.setContentType("application/json; charset=UTF-8");
    StartupProgress prog = NameNodeHttpServer.getStartupProgressFromContext(getServletContext());
    StartupProgressView view = prog.createView();
    JsonGenerator json = new JsonFactory().createGenerator(resp.getWriter());
    try {
        json.writeStartObject();
        json.writeNumberField(ELAPSED_TIME, view.getElapsedTime());
        json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete());
        json.writeArrayFieldStart(PHASES);
        for (Phase phase : view.getPhases()) {
            json.writeStartObject();
            json.writeStringField(NAME, phase.getName());
            json.writeStringField(DESC, phase.getDescription());
            json.writeStringField(STATUS, view.getStatus(phase).toString());
            json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete(phase));
            json.writeNumberField(ELAPSED_TIME, view.getElapsedTime(phase));
            writeStringFieldIfNotNull(json, FILE, view.getFile(phase));
            writeNumberFieldIfDefined(json, SIZE, view.getSize(phase));
            json.writeArrayFieldStart(STEPS);
            for (Step step : view.getSteps(phase)) {
                json.writeStartObject();
                StepType type = step.getType();
                if (type != null) {
                    json.writeStringField(NAME, type.getName());
                    json.writeStringField(DESC, type.getDescription());
                }
                json.writeNumberField(COUNT, view.getCount(phase, step));
                writeStringFieldIfNotNull(json, FILE, step.getFile());
                writeNumberFieldIfDefined(json, SIZE, step.getSize());
                json.writeNumberField(TOTAL, view.getTotal(phase, step));
                json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete(phase, step));
                json.writeNumberField(ELAPSED_TIME, view.getElapsedTime(phase, step));
                json.writeEndObject();
            }
            json.writeEndArray();
            json.writeEndObject();
        }
        json.writeEndArray();
        json.writeEndObject();
    } finally {
        IOUtils.cleanup(LOG, json);
    }
}

Example 3 with Step

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.Step in project hadoop by apache.

the class FSEditLogLoader method createStartupProgressStep.

/**
   * Creates a Step used for updating startup progress, populated with
   * information from the given edits.  The step always includes the log's name.
   * If the log has a known length, then the length is included in the step too.
   * 
   * @param edits EditLogInputStream to use for populating step
   * @return Step populated with information from edits
   * @throws IOException thrown if there is an I/O error
   */
private static Step createStartupProgressStep(EditLogInputStream edits) throws IOException {
    long length = edits.length();
    String name = edits.getCurrentStreamName();
    return length != -1 ? new Step(name, length) : new Step(name);
}

Example 4 with Step

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.Step in project hadoop by apache.

the class FSEditLogLoader method loadEditRecords.

long loadEditRecords(EditLogInputStream in, boolean closeOnExit, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException {
    FSDirectory fsDir = fsNamesys.dir;
    EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts = new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
    if (LOG.isTraceEnabled()) {
        LOG.trace("Acquiring write lock to replay edit log");
    }
    fsNamesys.writeLock();
    fsDir.writeLock();
    long[] recentOpcodeOffsets = new long[4];
    Arrays.fill(recentOpcodeOffsets, -1);
    long expectedTxId = expectedStartingTxId;
    long numEdits = 0;
    long lastTxId = in.getLastTxId();
    long numTxns = (lastTxId - expectedStartingTxId) + 1;
    StartupProgress prog = NameNode.getStartupProgress();
    Step step = createStartupProgressStep(in);
    prog.setTotal(Phase.LOADING_EDITS, step, numTxns);
    Counter counter = prog.getCounter(Phase.LOADING_EDITS, step);
    long lastLogTime = monotonicNow();
    long lastInodeId = fsNamesys.dir.getLastInodeId();
    try {
        while (true) {
            try {
                FSEditLogOp op;
                try {
                    op = in.readOp();
                    if (op == null) {
                        break;
                    }
                } catch (Throwable e) {
                    // Handle a problem with our input
                    check203UpgradeFailure(in.getVersion(true), e);
                    String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId);
                    FSImage.LOG.error(errorMessage, e);
                    if (recovery == null) {
                        // recovery mode.
                        throw new EditLogInputException(errorMessage, e, numEdits);
                    }
                    MetaRecoveryContext.editLogLoaderPrompt("We failed to read txId " + expectedTxId, recovery, "skipping the bad section in the log");
                    in.resync();
                    continue;
                }
                recentOpcodeOffsets[(int) (numEdits % recentOpcodeOffsets.length)] = in.getPosition();
                if (op.hasTransactionId()) {
                    if (op.getTransactionId() > expectedTxId) {
                        MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be a gap in the edit log.  We expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "ignoring missing " + " transaction IDs");
                    } else if (op.getTransactionId() < expectedTxId) {
                        MetaRecoveryContext.editLogLoaderPrompt("There appears " + "to be an out-of-order edit in the edit log.  We " + "expected txid " + expectedTxId + ", but got txid " + op.getTransactionId() + ".", recovery, "skipping the out-of-order edit");
                        continue;
                    }
                }
                try {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("op=" + op + ", startOpt=" + startOpt + ", numEdits=" + numEdits + ", totalEdits=" + totalEdits);
                    }
                    long inodeId = applyEditLogOp(op, fsDir, startOpt, in.getVersion(true), lastInodeId);
                    if (lastInodeId < inodeId) {
                        lastInodeId = inodeId;
                    }
                } catch (RollingUpgradeOp.RollbackException e) {
                    throw e;
                } catch (Throwable e) {
                    LOG.error("Encountered exception on operation " + op, e);
                    if (recovery == null) {
                        throw e instanceof IOException ? (IOException) e : new IOException(e);
                    }
                    MetaRecoveryContext.editLogLoaderPrompt("Failed to " + "apply edit log operation " + op + ": error " + e.getMessage(), recovery, "applying edits");
                }
                // Now that the operation has been successfully decoded and
                // applied, update our bookkeeping.
                incrOpCount(op.opCode, opCounts, step, counter);
                if (op.hasTransactionId()) {
                    lastAppliedTxId = op.getTransactionId();
                    expectedTxId = lastAppliedTxId + 1;
                } else {
                    expectedTxId = lastAppliedTxId = expectedStartingTxId;
                }
                // log progress
                if (op.hasTransactionId()) {
                    long now = monotonicNow();
                    if (now - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
                        long deltaTxId = lastAppliedTxId - expectedStartingTxId + 1;
                        int percent = Math.round((float) deltaTxId / numTxns * 100);
                        LOG.info("replaying edit log: " + deltaTxId + "/" + numTxns + " transactions completed. (" + percent + "%)");
                        lastLogTime = now;
                    }
                }
                numEdits++;
                totalEdits++;
            } catch (RollingUpgradeOp.RollbackException e) {
                LOG.info("Stopped at OP_START_ROLLING_UPGRADE for rollback.");
                break;
            } catch (MetaRecoveryContext.RequestStopException e) {
                MetaRecoveryContext.LOG.warn("Stopped reading edit log at " + in.getPosition() + "/" + in.length());
                break;
            }
        }
    } finally {
        fsNamesys.dir.resetLastInodeId(lastInodeId);
        if (closeOnExit) {
            in.close();
        }
        fsDir.writeUnlock();
        fsNamesys.writeUnlock("loadEditRecords");
        if (LOG.isTraceEnabled()) {
            LOG.trace("replaying edit log finished");
        }
        if (FSImage.LOG.isDebugEnabled()) {
            dumpOpCounts(opCounts);
        }
    }
    return numEdits;
}