Examples with HdfsConfiguration org.apache.hadoop.hdfs.HdfsConfiguration used on opensource projects

Example 96 with HdfsConfiguration

use of org.apache.hadoop.hdfs.HdfsConfiguration in project hadoop by apache.

the class TestCheckpoint method testEditFailureOnFirstCheckpoint.

/**
   * Test that a fault while downloading edits the first time after the 2NN
   * starts up does not prevent future checkpointing.
   */
@Test(timeout = 30000)
public void testEditFailureOnFirstCheckpoint() throws IOException {
    Configuration conf = new HdfsConfiguration();
    SecondaryNameNode secondary = null;
    MiniDFSCluster cluster = null;
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        fs.mkdirs(new Path("test-file-1"));
        // Make sure the on-disk fsimage on the NN has txid > 0.
        FSNamesystem fsns = cluster.getNamesystem();
        fsns.enterSafeMode(false);
        fsns.saveNamespace(0, 0);
        fsns.leaveSafeMode(false);
        secondary = startSecondaryNameNode(conf);
        // Cause edit rename to fail during next checkpoint
        Mockito.doThrow(new IOException("Injecting failure before edit rename")).when(faultInjector).beforeEditsRename();
        try {
            secondary.doCheckpoint();
            fail("Fault injection failed.");
        } catch (IOException ioe) {
            GenericTestUtils.assertExceptionContains("Injecting failure before edit rename", ioe);
        }
        Mockito.reset(faultInjector);
        // Next checkpoint should succeed
        secondary.doCheckpoint();
    } finally {
        if (secondary != null) {
            secondary.shutdown();
        }
        if (fs != null) {
            fs.close();
        }
        if (cluster != null) {
            cluster.shutdown();
        }
        Mockito.reset(faultInjector);
    }
}

Example 97 with HdfsConfiguration

use of org.apache.hadoop.hdfs.HdfsConfiguration in project hadoop by apache.

the class TestCheckpoint method testMultipleSecondaryNNsAgainstSameNN.

/**
   * Test case where two secondary namenodes are checkpointing the same
   * NameNode. This differs from {@link #testMultipleSecondaryNamenodes()}
   * since that test runs against two distinct NNs.
   * 
   * This case tests the following interleaving:
   * - 2NN A downloads image (up to txid 2)
   * - 2NN A about to save its own checkpoint
   * - 2NN B downloads image (up to txid 4)
   * - 2NN B uploads checkpoint (txid 4)
   * - 2NN A uploads checkpoint (txid 2)
   * 
   * It verifies that this works even though the earlier-txid checkpoint gets
   * uploaded after the later-txid checkpoint.
   */
@Test
public void testMultipleSecondaryNNsAgainstSameNN() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    SecondaryNameNode secondary1 = null, secondary2 = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
        // Start 2NNs
        secondary1 = startSecondaryNameNode(conf, 1);
        secondary2 = startSecondaryNameNode(conf, 2);
        // Make the first 2NN's checkpoint process delayable - we can pause it
        // right before it saves its checkpoint image.
        CheckpointStorage spyImage1 = spyOnSecondaryImage(secondary1);
        DelayAnswer delayer = new DelayAnswer(LOG);
        Mockito.doAnswer(delayer).when(spyImage1).saveFSImageInAllDirs(Mockito.<FSNamesystem>any(), Mockito.anyLong());
        // Set up a thread to do a checkpoint from the first 2NN
        DoCheckpointThread checkpointThread = new DoCheckpointThread(secondary1);
        checkpointThread.start();
        // Wait for the first checkpointer to get to where it should save its image.
        delayer.waitForCall();
        // Now make the second checkpointer run an entire checkpoint
        secondary2.doCheckpoint();
        // Let the first one finish
        delayer.proceed();
        // It should have succeeded even though another checkpoint raced with it.
        checkpointThread.join();
        checkpointThread.propagateExceptions();
        // primary should record "last checkpoint" as the higher txid (even though
        // a checkpoint with a lower txid finished most recently)
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        assertEquals(4, storage.getMostRecentCheckpointTxId());
        // Should have accepted both checkpoints
        assertNNHasCheckpoints(cluster, ImmutableList.of(2, 4));
        // Now have second one checkpoint one more time just to make sure that
        // the NN isn't left in a broken state
        secondary2.doCheckpoint();
        // NN should have received new checkpoint
        assertEquals(6, storage.getMostRecentCheckpointTxId());
        // Validate invariant that files named the same are the same.
        assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
        // NN should have removed the checkpoint at txid 2 at this point, but has
        // one at txid 6
        assertNNHasCheckpoints(cluster, ImmutableList.of(4, 6));
    } finally {
        cleanup(secondary1);
        secondary1 = null;
        cleanup(secondary2);
        secondary2 = null;
        if (cluster != null) {
            cluster.shutdown();
            cluster = null;
        }
    }
}

Example 98 with HdfsConfiguration

use of org.apache.hadoop.hdfs.HdfsConfiguration in project hadoop by apache.

the class TestCheckpoint method testSaveNamespace.

/**
   * Tests save namespace.
   */
@Test
public void testSaveNamespace() throws IOException {
    MiniDFSCluster cluster = null;
    DistributedFileSystem fs = null;
    FileContext fc;
    try {
        Configuration conf = new HdfsConfiguration();
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
        cluster.waitActive();
        fs = (cluster.getFileSystem());
        fc = FileContext.getFileContext(cluster.getURI(0));
        // Saving image without safe mode should fail
        DFSAdmin admin = new DFSAdmin(conf);
        String[] args = new String[] { "-saveNamespace" };
        try {
            admin.run(args);
        } catch (IOException eIO) {
            assertTrue(eIO.getLocalizedMessage().contains("Safe mode should be turned ON"));
        } catch (Exception e) {
            throw new IOException(e);
        }
        // create new file
        Path file = new Path("namespace.dat");
        DFSTestUtil.createFile(fs, file, fileSize, fileSize, blockSize, replication, seed);
        checkFile(fs, file, replication);
        // create new link
        Path symlink = new Path("file.link");
        fc.createSymlink(file, symlink, false);
        assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
        // verify that the edits file is NOT empty
        Collection<URI> editsDirs = cluster.getNameEditsDirs(0);
        for (URI uri : editsDirs) {
            File ed = new File(uri.getPath());
            assertTrue(new File(ed, "current/" + NNStorage.getInProgressEditsFileName(1)).length() > Integer.SIZE / Byte.SIZE);
        }
        // Saving image in safe mode should succeed
        fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        try {
            admin.run(args);
        } catch (Exception e) {
            throw new IOException(e);
        }
        // TODO: Fix the test to not require a hard-coded transaction count.
        final int EXPECTED_TXNS_FIRST_SEG = 13;
        //
        for (URI uri : editsDirs) {
            File ed = new File(uri.getPath());
            File curDir = new File(ed, "current");
            LOG.info("Files in " + curDir + ":\n  " + Joiner.on("\n  ").join(curDir.list()));
            // Verify that the first edits file got finalized
            File originalEdits = new File(curDir, NNStorage.getInProgressEditsFileName(1));
            assertFalse(originalEdits.exists());
            File finalizedEdits = new File(curDir, NNStorage.getFinalizedEditsFileName(1, EXPECTED_TXNS_FIRST_SEG));
            GenericTestUtils.assertExists(finalizedEdits);
            assertTrue(finalizedEdits.length() > Integer.SIZE / Byte.SIZE);
            GenericTestUtils.assertExists(new File(ed, "current/" + NNStorage.getInProgressEditsFileName(EXPECTED_TXNS_FIRST_SEG + 1)));
        }
        Collection<URI> imageDirs = cluster.getNameDirs(0);
        for (URI uri : imageDirs) {
            File imageDir = new File(uri.getPath());
            File savedImage = new File(imageDir, "current/" + NNStorage.getImageFileName(EXPECTED_TXNS_FIRST_SEG));
            assertTrue("Should have saved image at " + savedImage, savedImage.exists());
        }
        // restart cluster and verify file exists
        cluster.shutdown();
        cluster = null;
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
        cluster.waitActive();
        fs = (cluster.getFileSystem());
        checkFile(fs, file, replication);
        fc = FileContext.getFileContext(cluster.getURI(0));
        assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
    } finally {
        if (fs != null)
            fs.close();
        cleanup(cluster);
        cluster = null;
    }
}

Example 99 with HdfsConfiguration

use of org.apache.hadoop.hdfs.HdfsConfiguration in project hadoop by apache.

the class TestCheckpoint method testCheckpointTriggerOnTxnCount.

/**
   * Test that the 2NN triggers a checkpoint after the configurable interval
   */
@Test(timeout = 30000)
public void testCheckpointTriggerOnTxnCount() throws Exception {
    MiniDFSCluster cluster = null;
    SecondaryNameNode secondary = null;
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 10);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
        FileSystem fs = cluster.getFileSystem();
        secondary = startSecondaryNameNode(conf);
        secondary.startCheckpointThread();
        final NNStorage storage = secondary.getFSImage().getStorage();
        // 2NN should checkpoint at startup
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                LOG.info("Waiting for checkpoint txn id to go to 2");
                return storage.getMostRecentCheckpointTxId() == 2;
            }
        }, 200, 15000);
        // If we make 10 transactions, it should checkpoint again
        for (int i = 0; i < 10; i++) {
            fs.mkdirs(new Path("/test" + i));
        }
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                LOG.info("Waiting for checkpoint txn id to go > 2");
                return storage.getMostRecentCheckpointTxId() > 2;
            }
        }, 200, 15000);
    } finally {
        cleanup(secondary);
        secondary = null;
        cleanup(cluster);
        cluster = null;
    }
}

Example 100 with HdfsConfiguration

use of org.apache.hadoop.hdfs.HdfsConfiguration in project hadoop by apache.

the class TestCheckpoint method testCheckpointSignature.

/* Test case to test CheckpointSignature */
@Test
public void testCheckpointSignature() throws IOException {
    MiniDFSCluster cluster = null;
    Configuration conf = new HdfsConfiguration();
    SecondaryNameNode secondary = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
        NameNode nn = cluster.getNameNode();
        NamenodeProtocols nnRpc = nn.getRpcServer();
        secondary = startSecondaryNameNode(conf);
        // prepare checkpoint image
        secondary.doCheckpoint();
        CheckpointSignature sig = nnRpc.rollEditLog();
        // manipulate the CheckpointSignature fields
        sig.setBlockpoolID("somerandomebpid");
        sig.clusterID = "somerandomcid";
        try {
            // this should fail
            sig.validateStorageInfo(nn.getFSImage());
            assertTrue("This test is expected to fail.", false);
        } catch (Exception ignored) {
        }
    } finally {
        cleanup(secondary);
        secondary = null;
        cleanup(cluster);
        cluster = null;
    }
}