Examples with StorageDirectory org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory used on opensource projects

Example 26 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestCheckpoint method testCheckpointWithFailedStorageDir.

/**
   * Test that, if a storage directory is failed when a checkpoint occurs,
   * the non-failed storage directory receives the checkpoint.
   */
@Test
public void testCheckpointWithFailedStorageDir() throws Exception {
    MiniDFSCluster cluster = null;
    SecondaryNameNode secondary = null;
    File currentDir = null;
    Configuration conf = new HdfsConfiguration();
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
        secondary = startSecondaryNameNode(conf);
        // Checkpoint once
        secondary.doCheckpoint();
        // Now primary NN experiences failure of a volume -- fake by
        // setting its current dir to a-x permissions
        NamenodeProtocols nn = cluster.getNameNodeRpc();
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        StorageDirectory sd0 = storage.getStorageDir(0);
        StorageDirectory sd1 = storage.getStorageDir(1);
        currentDir = sd0.getCurrentDir();
        FileUtil.setExecutable(currentDir, false);
        // Upload checkpoint when NN has a bad storage dir. This should
        // succeed and create the checkpoint in the good dir.
        secondary.doCheckpoint();
        GenericTestUtils.assertExists(new File(sd1.getCurrentDir(), NNStorage.getImageFileName(2)));
        // Restore the good dir
        FileUtil.setExecutable(currentDir, true);
        nn.restoreFailedStorage("true");
        nn.rollEditLog();
        // Checkpoint again -- this should upload to both dirs
        secondary.doCheckpoint();
        assertNNHasCheckpoints(cluster, ImmutableList.of(8));
        assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
    } finally {
        if (currentDir != null) {
            FileUtil.setExecutable(currentDir, true);
        }
        cleanup(secondary);
        secondary = null;
        cleanup(cluster);
        cluster = null;
    }
}

Example 27 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TransferFsImage method downloadEditsToStorage.

static void downloadEditsToStorage(URL fsName, RemoteEditLog log, NNStorage dstStorage) throws IOException {
    assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log;
    String fileid = ImageServlet.getParamStringForLog(log, dstStorage);
    String finalFileName = NNStorage.getFinalizedEditsFileName(log.getStartTxId(), log.getEndTxId());
    List<File> finalFiles = dstStorage.getFiles(NameNodeDirType.EDITS, finalFileName);
    assert !finalFiles.isEmpty() : "No checkpoint targets.";
    for (File f : finalFiles) {
        if (f.exists() && FileUtil.canRead(f)) {
            LOG.info("Skipping download of remote edit log " + log + " since it already is stored locally at " + f);
            return;
        } else if (LOG.isDebugEnabled()) {
            LOG.debug("Dest file: " + f);
        }
    }
    final long milliTime = Time.monotonicNow();
    String tmpFileName = NNStorage.getTemporaryEditsFileName(log.getStartTxId(), log.getEndTxId(), milliTime);
    List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, tmpFileName);
    getFileClient(fsName, fileid, tmpFiles, dstStorage, false);
    LOG.info("Downloaded file " + tmpFiles.get(0).getName() + " size " + finalFiles.get(0).length() + " bytes.");
    CheckpointFaultInjector.getInstance().beforeEditsRename();
    for (StorageDirectory sd : dstStorage.dirIterable(NameNodeDirType.EDITS)) {
        File tmpFile = NNStorage.getTemporaryEditsFile(sd, log.getStartTxId(), log.getEndTxId(), milliTime);
        File finalizedFile = NNStorage.getFinalizedEditsFile(sd, log.getStartTxId(), log.getEndTxId());
        if (LOG.isDebugEnabled()) {
            LOG.debug("Renaming " + tmpFile + " to " + finalizedFile);
        }
        boolean success = tmpFile.renameTo(finalizedFile);
        if (!success) {
            LOG.warn("Unable to rename edits file from " + tmpFile + " to " + finalizedFile);
        }
    }
}

Example 28 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class BootstrapStandby method doPreUpgrade.

/**
   * This is called when using bootstrapStandby for HA upgrade. The SBN should
   * also create previous directory so that later when it starts, it understands
   * that the cluster is in the upgrade state. This function renames the old
   * current directory to previous.tmp.
   */
private boolean doPreUpgrade(NNStorage storage, NamespaceInfo nsInfo) throws IOException {
    boolean isFormatted = false;
    Map<StorageDirectory, StorageState> dataDirStates = new HashMap<>();
    try {
        isFormatted = FSImage.recoverStorageDirs(StartupOption.UPGRADE, storage, dataDirStates);
        if (dataDirStates.values().contains(StorageState.NOT_FORMATTED)) {
            // recoverStorageDirs returns true if there is a formatted directory
            isFormatted = false;
            System.err.println("The original storage directory is not formatted.");
        }
    } catch (InconsistentFSStateException e) {
        // if the storage is in a bad state,
        LOG.warn("The storage directory is in an inconsistent state", e);
    } finally {
        storage.unlockAll();
    }
    // "-bootstrapStandby", we should still be fine.
    if (!isFormatted && !format(storage, nsInfo)) {
        return false;
    }
    // make sure there is no previous directory
    FSImage.checkUpgrade(storage);
    // Do preUpgrade for each directory
    for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        try {
            NNUpgradeUtil.renameCurToTmp(sd);
        } catch (IOException e) {
            LOG.error("Failed to move aside pre-upgrade storage " + "in image directory " + sd.getRoot(), e);
            throw e;
        }
    }
    storage.setStorageInfo(nsInfo);
    storage.setBlockPoolID(nsInfo.getBlockPoolID());
    return true;
}

Example 29 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class FSImage method recoverStorageDirs.

/**
   * For each storage directory, performs recovery of incomplete transitions
   * (eg. upgrade, rollback, checkpoint) and inserts the directory's storage
   * state into the dataDirStates map.
   * @param dataDirStates output of storage directory states
   * @return true if there is at least one valid formatted storage directory
   */
public static boolean recoverStorageDirs(StartupOption startOpt, NNStorage storage, Map<StorageDirectory, StorageState> dataDirStates) throws IOException {
    boolean isFormatted = false;
    // mutate the shared dir below in the actual loop.
    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        StorageState curState;
        if (startOpt == StartupOption.METADATAVERSION) {
            /* All we need is the layout version. */
            storage.readProperties(sd);
            return true;
        }
        try {
            curState = sd.analyzeStorage(startOpt, storage);
            // sd is locked but not opened
            switch(curState) {
                case NON_EXISTENT:
                    // name-node fails if any of the configured storage dirs are missing
                    throw new InconsistentFSStateException(sd.getRoot(), "storage directory does not exist or is not accessible.");
                case NOT_FORMATTED:
                    break;
                case NORMAL:
                    break;
                default:
                    // recovery is possible
                    sd.doRecover(curState);
            }
            if (curState != StorageState.NOT_FORMATTED && startOpt != StartupOption.ROLLBACK) {
                // read and verify consistency with other directories
                storage.readProperties(sd, startOpt);
                isFormatted = true;
            }
            if (startOpt == StartupOption.IMPORT && isFormatted)
                // import of a checkpoint is allowed only into empty image directories
                throw new IOException("Cannot import image from a checkpoint. " + " NameNode already contains an image in " + sd.getRoot());
        } catch (IOException ioe) {
            sd.unlock();
            throw ioe;
        }
        dataDirStates.put(sd, curState);
    }
    return isFormatted;
}

Example 30 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class FSImage method recoverTransitionRead.

/**
   * Analyze storage directories.
   * Recover from previous transitions if required. 
   * Perform fs state transition if necessary depending on the namespace info.
   * Read storage info. 
   * 
   * @throws IOException
   * @return true if the image needs to be saved or false otherwise
   */
boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
    assert startOpt != StartupOption.FORMAT : "NameNode formatting should be performed before reading the image";
    Collection<URI> imageDirs = storage.getImageDirectories();
    Collection<URI> editsDirs = editLog.getEditURIs();
    // none of the data dirs exist
    if ((imageDirs.size() == 0 || editsDirs.size() == 0) && startOpt != StartupOption.IMPORT)
        throw new IOException("All specified directories are not accessible or do not exist.");
    // 1. For each data directory calculate its state and 
    // check whether all is consistent before transitioning.
    Map<StorageDirectory, StorageState> dataDirStates = new HashMap<StorageDirectory, StorageState>();
    boolean isFormatted = recoverStorageDirs(startOpt, storage, dataDirStates);
    if (LOG.isTraceEnabled()) {
        LOG.trace("Data dir states:\n  " + Joiner.on("\n  ").withKeyValueSeparator(": ").join(dataDirStates));
    }
    if (!isFormatted && startOpt != StartupOption.ROLLBACK && startOpt != StartupOption.IMPORT) {
        throw new IOException("NameNode is not formatted.");
    }
    int layoutVersion = storage.getLayoutVersion();
    if (startOpt == StartupOption.METADATAVERSION) {
        System.out.println("HDFS Image Version: " + layoutVersion);
        System.out.println("Software format version: " + HdfsServerConstants.NAMENODE_LAYOUT_VERSION);
        return false;
    }
    if (layoutVersion < Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION) {
        NNStorage.checkVersionUpgradable(storage.getLayoutVersion());
    }
    if (startOpt != StartupOption.UPGRADE && startOpt != StartupOption.UPGRADEONLY && !RollingUpgradeStartupOption.STARTED.matches(startOpt) && layoutVersion < Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION && layoutVersion != HdfsServerConstants.NAMENODE_LAYOUT_VERSION) {
        throw new IOException("\nFile system image contains an old layout version " + storage.getLayoutVersion() + ".\nAn upgrade to version " + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + " is required.\n" + "Please restart NameNode with the \"" + RollingUpgradeStartupOption.STARTED.getOptionString() + "\" option if a rolling upgrade is already started;" + " or restart NameNode with the \"" + StartupOption.UPGRADE.getName() + "\" option to start" + " a new upgrade.");
    }
    storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
    // 2. Format unformatted dirs.
    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        StorageState curState = dataDirStates.get(sd);
        switch(curState) {
            case NON_EXISTENT:
                throw new IOException(StorageState.NON_EXISTENT + " state cannot be here");
            case NOT_FORMATTED:
                LOG.info("Storage directory " + sd.getRoot() + " is not formatted.");
                LOG.info("Formatting ...");
                // create empty currrent dir
                sd.clearDirectory();
                break;
            default:
                break;
        }
    }
    // 3. Do transitions
    switch(startOpt) {
        case UPGRADE:
        case UPGRADEONLY:
            doUpgrade(target);
            // upgrade saved image already
            return false;
        case IMPORT:
            doImportCheckpoint(target);
            // import checkpoint saved image already
            return false;
        case ROLLBACK:
            throw new AssertionError("Rollback is now a standalone command, " + "NameNode should not be starting with this option.");
        case REGULAR:
        default:
    }
    return loadFSImage(target, startOpt, recovery);
}