Examples with MiniDFSCluster org.apache.hadoop.hdfs.MiniDFSCluster used on opensource projects

Example 21 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestEditLogTailer method testTriggersLogRollsForAllStandbyNN.

/*
    1. when all NN become standby nn, standby NN execute to roll log,
    it will be failed.
    2. when one NN become active, standby NN roll log success.
   */
@Test
public void testTriggersLogRollsForAllStandbyNN() throws Exception {
    Configuration conf = getConf();
    // Roll every 1s
    conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
    // Have to specify IPC ports so the NNs can talk to each other.
    MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(ServerSocketUtil.getPort(0, 100))).addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(ServerSocketUtil.getPort(0, 100))).addNN(new MiniDFSNNTopology.NNConf("nn3").setIpcPort(ServerSocketUtil.getPort(0, 100))));
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology).numDataNodes(0).build();
    try {
        cluster.transitionToStandby(0);
        cluster.transitionToStandby(1);
        cluster.transitionToStandby(2);
        try {
            waitForLogRollInSharedDir(cluster, 3);
            fail("After all NN become Standby state, Standby NN should roll log, " + "but it will be failed");
        } catch (TimeoutException ignore) {
        }
        cluster.transitionToActive(0);
        waitForLogRollInSharedDir(cluster, 3);
    } finally {
        cluster.shutdown();
    }
}

Example 22 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestEditLogTailer method testRollEditTimeoutForActiveNN.

@Test(timeout = 20000)
public void testRollEditTimeoutForActiveNN() throws IOException {
    Configuration conf = getConf();
    // 5s
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY, 5);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
    HAUtil.setAllowStandbyReads(conf, true);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    cluster.waitActive();
    cluster.transitionToActive(0);
    try {
        EditLogTailer tailer = Mockito.spy(cluster.getNamesystem(1).getEditLogTailer());
        AtomicInteger flag = new AtomicInteger(0);
        // Return a slow roll edit process.
        when(tailer.getNameNodeProxy()).thenReturn(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                // sleep for 30 seconds.
                Thread.sleep(30000);
                assertTrue(Thread.currentThread().isInterrupted());
                flag.addAndGet(1);
                return null;
            }
        });
        tailer.triggerActiveLogRoll();
        assertEquals(0, flag.get());
    } finally {
        cluster.shutdown();
    }
}

Example 23 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestEditLogsDuringFailover method testFailoverFinalizesAndReadsInProgress.

private void testFailoverFinalizesAndReadsInProgress(boolean partialTxAtEnd) throws Exception {
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    try {
        // Create a fake in-progress edit-log in the shared directory
        URI sharedUri = cluster.getSharedEditsDir(0, 1);
        File sharedDir = new File(sharedUri.getPath(), "current");
        FSNamesystem fsn = cluster.getNamesystem(0);
        FSImageTestUtil.createAbortedLogWithMkdirs(sharedDir, NUM_DIRS_IN_LOG, 1, fsn.getFSDirectory().getLastInodeId() + 1);
        assertEditFiles(Collections.singletonList(sharedUri), NNStorage.getInProgressEditsFileName(1));
        if (partialTxAtEnd) {
            FileOutputStream outs = null;
            try {
                File editLogFile = new File(sharedDir, NNStorage.getInProgressEditsFileName(1));
                outs = new FileOutputStream(editLogFile, true);
                outs.write(new byte[] { 0x18, 0x00, 0x00, 0x00 });
                LOG.error("editLogFile = " + editLogFile);
            } finally {
                IOUtils.cleanup(LOG, outs);
            }
        }
        // Transition one of the NNs to active
        cluster.transitionToActive(0);
        // In the transition to active, it should have read the log -- and
        // hence see one of the dirs we made in the fake log.
        String testPath = "/dir" + NUM_DIRS_IN_LOG;
        assertNotNull(cluster.getNameNode(0).getRpcServer().getFileInfo(testPath));
        // It also should have finalized that log in the shared directory and started
        // writing to a new one at the next txid.
        assertEditFiles(Collections.singletonList(sharedUri), NNStorage.getFinalizedEditsFileName(1, NUM_DIRS_IN_LOG + 1), NNStorage.getInProgressEditsFileName(NUM_DIRS_IN_LOG + 2));
    } finally {
        cluster.shutdown();
    }
}

Example 24 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestFailureOfSharedDir method testFailureOfSharedDir.

/**
   * Test that marking the shared edits dir as being "required" causes the NN to
   * fail if that dir can't be accessed.
   */
@Test
public void testFailureOfSharedDir() throws Exception {
    Configuration conf = new Configuration();
    conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, 2000);
    // The shared edits dir will automatically be marked required.
    MiniDFSCluster cluster = null;
    File sharedEditsDir = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).checkExitOnShutdown(false).build();
        cluster.waitActive();
        cluster.transitionToActive(0);
        FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/test1")));
        // Blow away the shared edits dir.
        URI sharedEditsUri = cluster.getSharedEditsDir(0, 1);
        sharedEditsDir = new File(sharedEditsUri);
        assertEquals(0, FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "-w", true));
        Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2);
        NameNode nn1 = cluster.getNameNode(1);
        assertTrue(nn1.isStandbyState());
        assertFalse("StandBy NameNode should not go to SafeMode on resource unavailability", nn1.isInSafeMode());
        NameNode nn0 = cluster.getNameNode(0);
        try {
            // Make sure that subsequent operations on the NN fail.
            nn0.getRpcServer().rollEditLog();
            fail("Succeeded in rolling edit log despite shared dir being deleted");
        } catch (ExitException ee) {
            GenericTestUtils.assertExceptionContains("finalize log segment 1, 3 failed for required journal", ee);
        }
        // dir didn't roll. Regression test for HDFS-2874.
        for (URI editsUri : cluster.getNameEditsDirs(0)) {
            if (editsUri.equals(sharedEditsUri)) {
                continue;
            }
            File editsDir = new File(editsUri.getPath());
            File curDir = new File(editsDir, "current");
            GenericTestUtils.assertGlobEquals(curDir, "edits_.*", NNStorage.getInProgressEditsFileName(1));
        }
    } finally {
        if (sharedEditsDir != null) {
            // without this test cleanup will fail
            FileUtil.chmod(sharedEditsDir.getAbsolutePath(), "+w", true);
        }
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 25 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestDFSUpgradeWithHA method testRollbackWithNfs.

/**
   * Test rollback with NFS shared dir.
   */
@Test
public void testRollbackWithNfs() throws Exception {
    MiniDFSCluster cluster = null;
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
        File sharedDir = new File(cluster.getSharedEditsDir(0, 1));
        // No upgrade is in progress at the moment.
        checkClusterPreviousDirExistence(cluster, false);
        assertCTimesEqual(cluster);
        checkPreviousDirExistence(sharedDir, false);
        // Transition NN0 to active and do some FS ops.
        cluster.transitionToActive(0);
        fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/foo1")));
        // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
        // flag.
        cluster.shutdownNameNode(1);
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
        cluster.restartNameNode(0, false);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, false);
        checkPreviousDirExistence(sharedDir, true);
        // NN0 should come up in the active state when given the -upgrade option,
        // so no need to transition it to active.
        assertTrue(fs.mkdirs(new Path("/foo2")));
        // Now bootstrap the standby with the upgraded info.
        int rc = BootstrapStandby.run(new String[] { "-force" }, cluster.getConfiguration(1));
        assertEquals(0, rc);
        cluster.restartNameNode(1);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, true);
        checkPreviousDirExistence(sharedDir, true);
        assertCTimesEqual(cluster);
        // Now shut down the cluster and do the rollback.
        Collection<URI> nn1NameDirs = cluster.getNameDirs(0);
        cluster.shutdown();
        conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs));
        NameNode.doRollback(conf, false);
        // The rollback operation should have rolled back the first NN's local
        // dirs, and the shared dir, but not the other NN's dirs. Those have to be
        // done by bootstrapping the standby.
        checkNnPreviousDirExistence(cluster, 0, false);
        checkPreviousDirExistence(sharedDir, false);
    } finally {
        if (fs != null) {
            fs.close();
        }
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}