Examples with StorageDirectory org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory used on opensource projects

Example 31 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class FSImage method saveDigestAndRenameCheckpointImage.

/**
   * This is called by the 2NN after having downloaded an image, and by
   * the NN after having received a new image from the 2NN. It
   * renames the image from fsimage_N.ckpt to fsimage_N and also
   * saves the related .md5 file into place.
   */
public synchronized void saveDigestAndRenameCheckpointImage(NameNodeFile nnf, long txid, MD5Hash digest) throws IOException {
    // Write and rename MD5 file
    List<StorageDirectory> badSds = Lists.newArrayList();
    for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.IMAGE)) {
        File imageFile = NNStorage.getImageFile(sd, nnf, txid);
        try {
            MD5FileUtils.saveMD5File(imageFile, digest);
        } catch (IOException ioe) {
            badSds.add(sd);
        }
    }
    storage.reportErrorsOnDirectories(badSds);
    CheckpointFaultInjector.getInstance().afterMD5Rename();
    // Rename image from tmp file
    renameCheckpoint(txid, NameNodeFile.IMAGE_NEW, nnf, false);
    // from now on
    if (txid > storage.getMostRecentCheckpointTxId()) {
        storage.setMostRecentCheckpointInfo(txid, Time.now());
    }
}

Example 32 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class FSImage method saveFSImageInAllDirs.

private synchronized void saveFSImageInAllDirs(FSNamesystem source, NameNodeFile nnf, long txid, Canceler canceler) throws IOException {
    StartupProgress prog = NameNode.getStartupProgress();
    prog.beginPhase(Phase.SAVING_CHECKPOINT);
    if (storage.getNumStorageDirs(NameNodeDirType.IMAGE) == 0) {
        throw new IOException("No image directories available!");
    }
    if (canceler == null) {
        canceler = new Canceler();
    }
    SaveNamespaceContext ctx = new SaveNamespaceContext(source, txid, canceler);
    try {
        List<Thread> saveThreads = new ArrayList<Thread>();
        // save images into current
        for (Iterator<StorageDirectory> it = storage.dirIterator(NameNodeDirType.IMAGE); it.hasNext(); ) {
            StorageDirectory sd = it.next();
            FSImageSaver saver = new FSImageSaver(ctx, sd, nnf);
            Thread saveThread = new Thread(saver, saver.toString());
            saveThreads.add(saveThread);
            saveThread.start();
        }
        waitForThreads(saveThreads);
        saveThreads.clear();
        storage.reportErrorsOnDirectories(ctx.getErrorSDs());
        if (storage.getNumStorageDirs(NameNodeDirType.IMAGE) == 0) {
            throw new IOException("Failed to save in any storage directories while saving namespace.");
        }
        if (canceler.isCancelled()) {
            deleteCancelledCheckpoint(txid);
            // throws
            ctx.checkCancelled();
            assert false : "should have thrown above!";
        }
        renameCheckpoint(txid, NameNodeFile.IMAGE_NEW, nnf, false);
        // Since we now have a new checkpoint, we can clean up some
        // old edit logs and checkpoints.
        purgeOldStorage(nnf);
        archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
    } finally {
        // Notify any threads waiting on the checkpoint to be canceled
        // that it is complete.
        ctx.markComplete();
        ctx = null;
    }
    prog.endPhase(Phase.SAVING_CHECKPOINT);
}

Example 33 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class FSImage method doUpgrade.

void doUpgrade(FSNamesystem target) throws IOException {
    checkUpgrade();
    // load the latest image
    // Do upgrade for each directory
    this.loadFSImage(target, StartupOption.UPGRADE, null);
    target.checkRollingUpgrade("upgrade namenode");
    long oldCTime = storage.getCTime();
    // generate new cTime for the state
    storage.cTime = now();
    int oldLV = storage.getLayoutVersion();
    storage.layoutVersion = HdfsServerConstants.NAMENODE_LAYOUT_VERSION;
    List<StorageDirectory> errorSDs = Collections.synchronizedList(new ArrayList<StorageDirectory>());
    assert !editLog.isSegmentOpen() : "Edits log must not be open.";
    LOG.info("Starting upgrade of local storage directories." + "\n   old LV = " + oldLV + "; old CTime = " + oldCTime + ".\n   new LV = " + storage.getLayoutVersion() + "; new CTime = " + storage.getCTime());
    // Do upgrade for each directory
    for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        try {
            NNUpgradeUtil.doPreUpgrade(conf, sd);
        } catch (Exception e) {
            LOG.error("Failed to move aside pre-upgrade storage " + "in image directory " + sd.getRoot(), e);
            errorSDs.add(sd);
            continue;
        }
    }
    if (target.isHaEnabled()) {
        editLog.doPreUpgradeOfSharedLog();
    }
    storage.reportErrorsOnDirectories(errorSDs);
    errorSDs.clear();
    saveFSImageInAllDirs(target, editLog.getLastWrittenTxId());
    // upgrade shared edit storage first
    if (target.isHaEnabled()) {
        editLog.doUpgradeOfSharedLog();
    }
    for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        try {
            NNUpgradeUtil.doUpgrade(sd, storage);
        } catch (IOException ioe) {
            errorSDs.add(sd);
            continue;
        }
    }
    storage.reportErrorsOnDirectories(errorSDs);
    isUpgradeFinalized = false;
    if (!storage.getRemovedStorageDirs().isEmpty()) {
        // during upgrade, it's a fatal error to fail any storage directory
        throw new IOException("Upgrade failed in " + storage.getRemovedStorageDirs().size() + " storage directory(ies), previously logged.");
    }
}

Example 34 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestJournalNodeSync method testMultipleJournalsMissingLogs.

@Test(timeout = 30000)
public void testMultipleJournalsMissingLogs() throws Exception {
    File firstJournalDir = jCluster.getJournalDir(0, jid);
    File firstJournalCurrentDir = new StorageDirectory(firstJournalDir).getCurrentDir();
    File secondJournalDir = jCluster.getJournalDir(1, jid);
    StorageDirectory sd = new StorageDirectory(secondJournalDir);
    File secondJournalCurrentDir = sd.getCurrentDir();
    // Generate some edit logs and delete one log from two journals.
    long firstTxId = generateEditLog();
    generateEditLog();
    List<File> missingLogs = Lists.newArrayList();
    missingLogs.add(deleteEditLog(firstJournalCurrentDir, firstTxId));
    missingLogs.add(deleteEditLog(secondJournalCurrentDir, firstTxId));
    GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 10000);
}

Example 35 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestJournalNodeSync method testRandomJournalMissingLogs.

// Test JournalNode Sync by randomly deleting edit logs from one or two of
// the journals.
@Test(timeout = 60000)
public void testRandomJournalMissingLogs() throws Exception {
    Random randomJournal = new Random();
    List<File> journalCurrentDirs = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        journalCurrentDirs.add(new StorageDirectory(jCluster.getJournalDir(i, jid)).getCurrentDir());
    }
    int count = 0;
    long lastStartTxId;
    int journalIndex;
    List<File> missingLogs = Lists.newArrayList();
    while (count < 5) {
        lastStartTxId = generateEditLog();
        // Delete the last edit log segment from randomly selected journal node
        journalIndex = randomJournal.nextInt(3);
        missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex), lastStartTxId));
        // Delete the last edit log segment from two journals for some logs
        if (count % 2 == 0) {
            journalIndex = (journalIndex + 1) % 3;
            missingLogs.add(deleteEditLog(journalCurrentDirs.get(journalIndex), lastStartTxId));
        }
        count++;
    }
    GenericTestUtils.waitFor(editLogExists(missingLogs), 500, 30000);
}