Examples with MiniQJMHACluster org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster used on opensource projects

Example 11 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestDFSUpgradeWithHA method testRollbackWithJournalNodes.

@Test
public void testRollbackWithJournalNodes() throws IOException, URISyntaxException {
    MiniQJMHACluster qjCluster = null;
    FileSystem fs = null;
    try {
        Builder builder = new MiniQJMHACluster.Builder(conf);
        builder.getDfsBuilder().numDataNodes(0);
        qjCluster = builder.build();
        MiniDFSCluster cluster = qjCluster.getDfsCluster();
        // No upgrade is in progress at the moment.
        checkClusterPreviousDirExistence(cluster, false);
        assertCTimesEqual(cluster);
        checkJnPreviousDirExistence(qjCluster, false);
        // Transition NN0 to active and do some FS ops.
        cluster.transitionToActive(0);
        fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/foo1")));
        final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);
        // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
        // flag.
        cluster.shutdownNameNode(1);
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
        cluster.restartNameNode(0, false);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, false);
        checkJnPreviousDirExistence(qjCluster, true);
        // NN0 should come up in the active state when given the -upgrade option,
        // so no need to transition it to active.
        assertTrue(fs.mkdirs(new Path("/foo2")));
        final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster);
        assertTrue(cidDuringUpgrade > cidBeforeUpgrade);
        // Now bootstrap the standby with the upgraded info.
        int rc = BootstrapStandby.run(new String[] { "-force" }, cluster.getConfiguration(1));
        assertEquals(0, rc);
        cluster.restartNameNode(1);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, true);
        checkJnPreviousDirExistence(qjCluster, true);
        assertCTimesEqual(cluster);
        // Shut down the NNs, but deliberately leave the JNs up and running.
        Collection<URI> nn1NameDirs = cluster.getNameDirs(0);
        cluster.shutdown();
        conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, Joiner.on(",").join(nn1NameDirs));
        NameNode.doRollback(conf, false);
        final long cidAfterRollback = getCommittedTxnIdValue(qjCluster);
        assertTrue(cidBeforeUpgrade < cidAfterRollback);
        // make sure the committedTxnId has been reset correctly after rollback
        assertTrue(cidDuringUpgrade > cidAfterRollback);
        // The rollback operation should have rolled back the first NN's local
        // dirs, and the shared dir, but not the other NN's dirs. Those have to be
        // done by bootstrapping the standby.
        checkNnPreviousDirExistence(cluster, 0, false);
        checkJnPreviousDirExistence(qjCluster, false);
    } finally {
        if (fs != null) {
            fs.close();
        }
        if (qjCluster != null) {
            qjCluster.shutdown();
        }
    }
}

Example 12 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestDFSUpgradeWithHA method testUpgradeWithJournalNodes.

/**
   * Make sure that an HA NN can successfully upgrade when configured using
   * JournalNodes.
   */
@Test
public void testUpgradeWithJournalNodes() throws IOException, URISyntaxException {
    MiniQJMHACluster qjCluster = null;
    FileSystem fs = null;
    try {
        Builder builder = new MiniQJMHACluster.Builder(conf);
        builder.getDfsBuilder().numDataNodes(0);
        qjCluster = builder.build();
        MiniDFSCluster cluster = qjCluster.getDfsCluster();
        // No upgrade is in progress at the moment.
        checkJnPreviousDirExistence(qjCluster, false);
        checkClusterPreviousDirExistence(cluster, false);
        assertCTimesEqual(cluster);
        // Transition NN0 to active and do some FS ops.
        cluster.transitionToActive(0);
        fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/foo1")));
        // get the value of the committedTxnId in journal nodes
        final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);
        // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
        // flag.
        cluster.shutdownNameNode(1);
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
        cluster.restartNameNode(0, false);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, false);
        checkJnPreviousDirExistence(qjCluster, true);
        assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster));
        // NN0 should come up in the active state when given the -upgrade option,
        // so no need to transition it to active.
        assertTrue(fs.mkdirs(new Path("/foo2")));
        // Restart NN0 without the -upgrade flag, to make sure that works.
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
        cluster.restartNameNode(0, false);
        // Make sure we can still do FS ops after upgrading.
        cluster.transitionToActive(0);
        assertTrue(fs.mkdirs(new Path("/foo3")));
        assertTrue(getCommittedTxnIdValue(qjCluster) > cidBeforeUpgrade);
        // Now bootstrap the standby with the upgraded info.
        int rc = BootstrapStandby.run(new String[] { "-force" }, cluster.getConfiguration(1));
        assertEquals(0, rc);
        // Now restart NN1 and make sure that we can do ops against that as well.
        cluster.restartNameNode(1);
        cluster.transitionToStandby(0);
        cluster.transitionToActive(1);
        assertTrue(fs.mkdirs(new Path("/foo4")));
        assertCTimesEqual(cluster);
    } finally {
        if (fs != null) {
            fs.close();
        }
        if (qjCluster != null) {
            qjCluster.shutdown();
        }
    }
}

Example 13 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestDFSUpgradeWithHA method testFinalizeFromSecondNameNodeWithJournalNodes.

/**
   * Make sure that even if the NN which initiated the upgrade is in the standby
   * state that we're allowed to finalize.
   */
@Test
public void testFinalizeFromSecondNameNodeWithJournalNodes() throws IOException, URISyntaxException {
    MiniQJMHACluster qjCluster = null;
    FileSystem fs = null;
    try {
        Builder builder = new MiniQJMHACluster.Builder(conf);
        builder.getDfsBuilder().numDataNodes(0);
        qjCluster = builder.build();
        MiniDFSCluster cluster = qjCluster.getDfsCluster();
        // No upgrade is in progress at the moment.
        checkJnPreviousDirExistence(qjCluster, false);
        checkClusterPreviousDirExistence(cluster, false);
        assertCTimesEqual(cluster);
        // Transition NN0 to active and do some FS ops.
        cluster.transitionToActive(0);
        fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/foo1")));
        // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
        // flag.
        cluster.shutdownNameNode(1);
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
        cluster.restartNameNode(0, false);
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, false);
        checkJnPreviousDirExistence(qjCluster, true);
        // Now bootstrap the standby with the upgraded info.
        int rc = BootstrapStandby.run(new String[] { "-force" }, cluster.getConfiguration(1));
        assertEquals(0, rc);
        cluster.restartNameNode(1);
        // Make the second NN (not the one that initiated the upgrade) active when
        // the finalize command is run.
        cluster.transitionToStandby(0);
        cluster.transitionToActive(1);
        runFinalizeCommand(cluster);
        checkClusterPreviousDirExistence(cluster, false);
        checkJnPreviousDirExistence(qjCluster, false);
        assertCTimesEqual(cluster);
    } finally {
        if (fs != null) {
            fs.close();
        }
        if (qjCluster != null) {
            qjCluster.shutdown();
        }
    }
}

Example 14 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestDFSUpgradeWithHA method testFinalizeWithJournalNodes.

@Test
public void testFinalizeWithJournalNodes() throws IOException, URISyntaxException {
    MiniQJMHACluster qjCluster = null;
    FileSystem fs = null;
    try {
        Builder builder = new MiniQJMHACluster.Builder(conf);
        builder.getDfsBuilder().numDataNodes(0);
        qjCluster = builder.build();
        MiniDFSCluster cluster = qjCluster.getDfsCluster();
        // No upgrade is in progress at the moment.
        checkJnPreviousDirExistence(qjCluster, false);
        checkClusterPreviousDirExistence(cluster, false);
        assertCTimesEqual(cluster);
        // Transition NN0 to active and do some FS ops.
        cluster.transitionToActive(0);
        fs = HATestUtil.configureFailoverFs(cluster, conf);
        assertTrue(fs.mkdirs(new Path("/foo1")));
        final long cidBeforeUpgrade = getCommittedTxnIdValue(qjCluster);
        // Do the upgrade. Shut down NN1 and then restart NN0 with the upgrade
        // flag.
        cluster.shutdownNameNode(1);
        cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
        cluster.restartNameNode(0, false);
        assertTrue(cidBeforeUpgrade <= getCommittedTxnIdValue(qjCluster));
        assertTrue(fs.mkdirs(new Path("/foo2")));
        checkNnPreviousDirExistence(cluster, 0, true);
        checkNnPreviousDirExistence(cluster, 1, false);
        checkJnPreviousDirExistence(qjCluster, true);
        // Now bootstrap the standby with the upgraded info.
        int rc = BootstrapStandby.run(new String[] { "-force" }, cluster.getConfiguration(1));
        assertEquals(0, rc);
        cluster.restartNameNode(1);
        final long cidDuringUpgrade = getCommittedTxnIdValue(qjCluster);
        assertTrue(cidDuringUpgrade > cidBeforeUpgrade);
        runFinalizeCommand(cluster);
        assertEquals(cidDuringUpgrade, getCommittedTxnIdValue(qjCluster));
        checkClusterPreviousDirExistence(cluster, false);
        checkJnPreviousDirExistence(qjCluster, false);
        assertCTimesEqual(cluster);
    } finally {
        if (fs != null) {
            fs.close();
        }
        if (qjCluster != null) {
            qjCluster.shutdown();
        }
    }
}