Examples with StorageDirectory org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory used on opensource projects

Example 71 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestCheckpoint method doSecondaryFailsToReturnImage.

private void doSecondaryFailsToReturnImage() throws IOException {
    LOG.info("Starting testSecondaryFailsToReturnImage");
    Configuration conf = new HdfsConfiguration();
    Path file1 = new Path("checkpointRI.dat");
    MiniDFSCluster cluster = null;
    FileSystem fileSys = null;
    FSImage image = null;
    SecondaryNameNode secondary = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
        cluster.waitActive();
        fileSys = cluster.getFileSystem();
        image = cluster.getNameNode().getFSImage();
        assertTrue(!fileSys.exists(file1));
        StorageDirectory sd = image.getStorage().getStorageDir(0);
        File latestImageBeforeCheckpoint = FSImageTestUtil.findLatestImageFile(sd);
        long fsimageLength = latestImageBeforeCheckpoint.length();
        //
        // Make the checkpoint
        //
        secondary = startSecondaryNameNode(conf);
        try {
            // this should fail
            secondary.doCheckpoint();
            fail("Checkpoint succeeded even though we injected an error!");
        } catch (IOException e) {
            // check that it's the injected exception
            GenericTestUtils.assertExceptionContains("If this exception is not caught", e);
        }
        Mockito.reset(faultInjector);
        // Verify that image file sizes did not change.
        for (StorageDirectory sd2 : image.getStorage().dirIterable(NameNodeDirType.IMAGE)) {
            File thisNewestImage = FSImageTestUtil.findLatestImageFile(sd2);
            long len = thisNewestImage.length();
            assertEquals(fsimageLength, len);
        }
    } finally {
        fileSys.close();
        cleanup(secondary);
        secondary = null;
        cleanup(cluster);
        cluster = null;
    }
}

Example 72 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestCheckpoint method testNameDirLocking.

/**
   * Test that the NN locks its storage and edits directories, and won't start up
   * if the directories are already locked
   **/
@Test
public void testNameDirLocking() throws IOException {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    // Start a NN, and verify that lock() fails in all of the configured
    // directories
    StorageDirectory savedSd = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        for (StorageDirectory sd : storage.dirIterable(null)) {
            assertLockFails(sd);
            savedSd = sd;
        }
    } finally {
        cleanup(cluster);
        cluster = null;
    }
    assertNotNull(savedSd);
    // Lock one of the saved directories, then start the NN, and make sure it
    // fails to start
    assertClusterStartFailsWhenDirLocked(conf, savedSd);
}

Example 73 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestCheckpoint method testStorageAlreadyLockedErrorMessage.

/**
   * Test that, an attempt to lock a storage that is already locked by nodename,
   * logs error message that includes JVM name of the namenode that locked it.
   */
@Test
public void testStorageAlreadyLockedErrorMessage() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    StorageDirectory savedSd = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        for (StorageDirectory sd : storage.dirIterable(null)) {
            assertLockFails(sd);
            savedSd = sd;
        }
        LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(LogFactory.getLog(Storage.class));
        try {
            // try to lock the storage that's already locked
            savedSd.lock();
            fail("Namenode should not be able to lock a storage" + " that is already locked");
        } catch (IOException ioe) {
            // cannot read lock file on Windows, so message cannot get JVM name
            String lockingJvmName = Path.WINDOWS ? "" : " " + ManagementFactory.getRuntimeMXBean().getName();
            String expectedLogMessage = "It appears that another node " + lockingJvmName + " has already locked the storage directory";
            assertTrue("Log output does not contain expected log message: " + expectedLogMessage, logs.getOutput().contains(expectedLogMessage));
        }
    } finally {
        cleanup(cluster);
        cluster = null;
    }
}

Example 74 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestBackupNode method testBackupNodeTailsEdits.

/**
   * Ensure that the backupnode will tail edits from the NN
   * and keep in sync, even while the NN rolls, checkpoints
   * occur, etc.
   */
@Test
public void testBackupNodeTailsEdits() throws Exception {
    Configuration conf = new HdfsConfiguration();
    HAUtil.setAllowStandbyReads(conf, true);
    MiniDFSCluster cluster = null;
    FileSystem fileSys = null;
    BackupNode backup = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        fileSys = cluster.getFileSystem();
        backup = startBackupNode(conf, StartupOption.BACKUP, 1);
        BackupImage bnImage = (BackupImage) backup.getFSImage();
        testBNInSync(cluster, backup, 1);
        // Force a roll -- BN should roll with NN.
        NameNode nn = cluster.getNameNode();
        NamenodeProtocols nnRpc = nn.getRpcServer();
        nnRpc.rollEditLog();
        assertEquals(bnImage.getEditLog().getCurSegmentTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId());
        // BN should stay in sync after roll
        testBNInSync(cluster, backup, 2);
        long nnImageBefore = nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
        // BN checkpoint
        backup.doCheckpoint();
        // NN should have received a new image
        long nnImageAfter = nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
        assertTrue("nn should have received new checkpoint. before: " + nnImageBefore + " after: " + nnImageAfter, nnImageAfter > nnImageBefore);
        // BN should stay in sync after checkpoint
        testBNInSync(cluster, backup, 3);
        // Stop BN
        StorageDirectory sd = bnImage.getStorage().getStorageDir(0);
        backup.stop();
        backup = null;
        // When shutting down the BN, it shouldn't finalize logs that are
        // still open on the NN
        EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
        assertEquals(editsLog.getFirstTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId());
        assertTrue("Should not have finalized " + editsLog, editsLog.isInProgress());
        // do some edits
        assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
        // start a new backup node
        backup = startBackupNode(conf, StartupOption.BACKUP, 1);
        testBNInSync(cluster, backup, 4);
        assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down", false));
        // Trigger an unclean shutdown of the backup node. Backup node will not
        // unregister from the active when this is done simulating a node crash.
        backup.stop(false);
        // do some edits on the active. This should go through without failing.
        // This will verify that active is still up and can add entries to
        // master editlog.
        assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down-2")));
    } finally {
        LOG.info("Shutting down...");
        if (backup != null)
            backup.stop();
        if (fileSys != null)
            fileSys.close();
        if (cluster != null)
            cluster.shutdown();
    }
    assertStorageDirsMatch(cluster.getNameNode(), backup);
}

Example 75 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class BackupImage method recoverCreateRead.

/**
   * Analyze backup storage directories for consistency.<br>
   * Recover from incomplete checkpoints if required.<br>
   * Read VERSION and fstime files if exist.<br>
   * Do not load image or edits.
   *
   * @throws IOException if the node should shutdown.
   */
void recoverCreateRead() throws IOException {
    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext(); ) {
        StorageDirectory sd = it.next();
        StorageState curState;
        try {
            curState = sd.analyzeStorage(HdfsServerConstants.StartupOption.REGULAR, storage);
            // sd is locked but not opened
            switch(curState) {
                case NON_EXISTENT:
                    // fail if any of the configured storage dirs are inaccessible
                    throw new InconsistentFSStateException(sd.getRoot(), "checkpoint directory does not exist or is not accessible.");
                case NOT_FORMATTED:
                    // for backup node all directories may be unformatted initially
                    LOG.info("Storage directory " + sd.getRoot() + " is not formatted.");
                    LOG.info("Formatting ...");
                    // create empty current
                    sd.clearDirectory();
                    break;
                case NORMAL:
                    break;
                default:
                    // recovery is possible
                    sd.doRecover(curState);
            }
            if (curState != StorageState.NOT_FORMATTED) {
                // read and verify consistency with other directories
                storage.readProperties(sd);
            }
        } catch (IOException ioe) {
            sd.unlock();
            throw ioe;
        }
    }
}