Examples with StartupProgress org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress used on opensource projects

Example 11 with StartupProgress

use of org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress in project hadoop by apache.

the class FSImage method loadFSImage.

/**
   * Choose latest image from one of the directories,
   * load it and merge with the edits.
   * 
   * Saving and loading fsimage should never trigger symlink resolution. 
   * The paths that are persisted do not have *intermediate* symlinks 
   * because intermediate symlinks are resolved at the time files, 
   * directories, and symlinks are created. All paths accessed while 
   * loading or saving fsimage should therefore only see symlinks as 
   * the final path component, and the functions called below do not
   * resolve symlinks that are the final path component.
   *
   * @return whether the image should be saved
   * @throws IOException
   */
private boolean loadFSImage(FSNamesystem target, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException {
    final boolean rollingRollback = RollingUpgradeStartupOption.ROLLBACK.matches(startOpt);
    final EnumSet<NameNodeFile> nnfs;
    if (rollingRollback) {
        // if it is rollback of rolling upgrade, only load from the rollback image
        nnfs = EnumSet.of(NameNodeFile.IMAGE_ROLLBACK);
    } else {
        // otherwise we can load from both IMAGE and IMAGE_ROLLBACK
        nnfs = EnumSet.of(NameNodeFile.IMAGE, NameNodeFile.IMAGE_ROLLBACK);
    }
    final FSImageStorageInspector inspector = storage.readAndInspectDirs(nnfs, startOpt);
    isUpgradeFinalized = inspector.isUpgradeFinalized();
    List<FSImageFile> imageFiles = inspector.getLatestImages();
    StartupProgress prog = NameNode.getStartupProgress();
    prog.beginPhase(Phase.LOADING_FSIMAGE);
    File phaseFile = imageFiles.get(0).getFile();
    prog.setFile(Phase.LOADING_FSIMAGE, phaseFile.getAbsolutePath());
    prog.setSize(Phase.LOADING_FSIMAGE, phaseFile.length());
    boolean needToSave = inspector.needToSave();
    Iterable<EditLogInputStream> editStreams = null;
    initEditLog(startOpt);
    if (NameNodeLayoutVersion.supports(LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) {
        // If we're open for write, we're either non-HA or we're the active NN, so
        // we better be able to load all the edits. If we're the standby NN, it's
        // OK to not be able to read all of edits right now.
        // In the meanwhile, for HA upgrade, we will still write editlog thus need
        // this toAtLeastTxId to be set to the max-seen txid
        // For rollback in rolling upgrade, we need to set the toAtLeastTxId to
        // the txid right before the upgrade marker.  
        long toAtLeastTxId = editLog.isOpenForWrite() ? inspector.getMaxSeenTxId() : 0;
        if (rollingRollback) {
            // note that the first image in imageFiles is the special checkpoint
            // for the rolling upgrade
            toAtLeastTxId = imageFiles.get(0).getCheckpointTxId() + 2;
        }
        editStreams = editLog.selectInputStreams(imageFiles.get(0).getCheckpointTxId() + 1, toAtLeastTxId, recovery, false);
    } else {
        editStreams = FSImagePreTransactionalStorageInspector.getEditLogStreams(storage);
    }
    int maxOpSize = conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT);
    for (EditLogInputStream elis : editStreams) {
        elis.setMaxOpSize(maxOpSize);
    }
    for (EditLogInputStream l : editStreams) {
        LOG.debug("Planning to load edit log stream: " + l);
    }
    if (!editStreams.iterator().hasNext()) {
        LOG.info("No edit log streams selected.");
    }
    FSImageFile imageFile = null;
    for (int i = 0; i < imageFiles.size(); i++) {
        try {
            imageFile = imageFiles.get(i);
            loadFSImageFile(target, recovery, imageFile, startOpt);
            break;
        } catch (IllegalReservedPathException ie) {
            throw new IOException("Failed to load image from " + imageFile, ie);
        } catch (Exception e) {
            LOG.error("Failed to load image from " + imageFile, e);
            target.clear();
            imageFile = null;
        }
    }
    // Failed to load any images, error out
    if (imageFile == null) {
        FSEditLog.closeAllStreams(editStreams);
        throw new IOException("Failed to load FSImage file, see error(s) " + "above for more info.");
    }
    prog.endPhase(Phase.LOADING_FSIMAGE);
    if (!rollingRollback) {
        long txnsAdvanced = loadEdits(editStreams, target, startOpt, recovery);
        needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(), txnsAdvanced);
    } else {
        // Trigger the rollback for rolling upgrade. Here lastAppliedTxId equals
        // to the last txid in rollback fsimage.
        rollingRollback(lastAppliedTxId + 1, imageFiles.get(0).getCheckpointTxId());
        needToSave = false;
    }
    editLog.setNextTxId(lastAppliedTxId + 1);
    return needToSave;
}