Examples with MiniJournalCluster org.apache.hadoop.hdfs.qjournal.MiniJournalCluster used on opensource projects

Example 1 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestRollingUpgrade method testRollingUpgradeWithQJM.

@Test(timeout = 30000)
public void testRollingUpgradeWithQJM() throws Exception {
    String nnDirPrefix = MiniDFSCluster.getBaseDirectory() + "/nn/";
    final File nn1Dir = new File(nnDirPrefix + "image1");
    final File nn2Dir = new File(nnDirPrefix + "image2");
    LOG.info("nn1Dir=" + nn1Dir);
    LOG.info("nn2Dir=" + nn2Dir);
    final Configuration conf = new HdfsConfiguration();
    final MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build();
    mjc.waitActive();
    setConf(conf, nn1Dir, mjc);
    {
        // Start the cluster once to generate the dfs dirs
        final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).manageNameDfsDirs(false).checkExitOnShutdown(false).build();
        // Shutdown the cluster before making a copy of the namenode dir to release
        // all file locks, otherwise, the copy will fail on some platforms.
        cluster.shutdown();
    }
    MiniDFSCluster cluster2 = null;
    try {
        // Start a second NN pointed to the same quorum.
        // We need to copy the image dir from the first NN -- or else
        // the new NN will just be rejected because of Namespace mismatch.
        FileUtil.fullyDelete(nn2Dir);
        FileUtil.copy(nn1Dir, FileSystem.getLocal(conf).getRaw(), new Path(nn2Dir.getAbsolutePath()), false, conf);
        // Start the cluster again
        final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageNameDfsDirs(false).checkExitOnShutdown(false).build();
        final Path foo = new Path("/foo");
        final Path bar = new Path("/bar");
        final Path baz = new Path("/baz");
        final RollingUpgradeInfo info1;
        {
            final DistributedFileSystem dfs = cluster.getFileSystem();
            dfs.mkdirs(foo);
            //start rolling upgrade
            dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
            info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
            dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
            LOG.info("START\n" + info1);
            //query rolling upgrade
            assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
            dfs.mkdirs(bar);
            cluster.shutdown();
        }
        // cluster2 takes over QJM
        final Configuration conf2 = setConf(new Configuration(), nn2Dir, mjc);
        cluster2 = new MiniDFSCluster.Builder(conf2).numDataNodes(0).format(false).manageNameDfsDirs(false).build();
        final DistributedFileSystem dfs2 = cluster2.getFileSystem();
        // Check that cluster2 sees the edits made on cluster1
        Assert.assertTrue(dfs2.exists(foo));
        Assert.assertTrue(dfs2.exists(bar));
        Assert.assertFalse(dfs2.exists(baz));
        //query rolling upgrade in cluster2
        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
        dfs2.mkdirs(baz);
        LOG.info("RESTART cluster 2");
        cluster2.restartNameNode();
        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
        Assert.assertTrue(dfs2.exists(foo));
        Assert.assertTrue(dfs2.exists(bar));
        Assert.assertTrue(dfs2.exists(baz));
        //restart cluster with -upgrade should fail.
        try {
            cluster2.restartNameNode("-upgrade");
        } catch (IOException e) {
            LOG.info("The exception is expected.", e);
        }
        LOG.info("RESTART cluster 2 again");
        cluster2.restartNameNode();
        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
        Assert.assertTrue(dfs2.exists(foo));
        Assert.assertTrue(dfs2.exists(bar));
        Assert.assertTrue(dfs2.exists(baz));
        //finalize rolling upgrade
        final RollingUpgradeInfo finalize = dfs2.rollingUpgrade(RollingUpgradeAction.FINALIZE);
        Assert.assertTrue(finalize.isFinalized());
        LOG.info("RESTART cluster 2 with regular startup option");
        cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
        cluster2.restartNameNode();
        Assert.assertTrue(dfs2.exists(foo));
        Assert.assertTrue(dfs2.exists(bar));
        Assert.assertTrue(dfs2.exists(baz));
    } finally {
        if (cluster2 != null)
            cluster2.shutdown();
    }
}

Example 2 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestEpochsAreUnique method testSingleThreaded.

@Test
public void testSingleThreaded() throws IOException {
    Configuration conf = new Configuration();
    MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
    cluster.waitActive();
    URI uri = cluster.getQuorumJournalURI(JID);
    QuorumJournalManager qjm = new QuorumJournalManager(conf, uri, FAKE_NSINFO);
    try {
        qjm.format(FAKE_NSINFO);
    } finally {
        qjm.close();
    }
    try {
        // With no failures or contention, epochs should increase one-by-one
        for (int i = 0; i < 5; i++) {
            qjm = new QuorumJournalManager(conf, uri, FAKE_NSINFO);
            try {
                qjm.createNewUniqueEpoch();
                assertEquals(i + 1, qjm.getLoggerSetForTests().getEpoch());
            } finally {
                qjm.close();
            }
        }
        long prevEpoch = 5;
        // skipping some
        for (int i = 0; i < 20; i++) {
            long newEpoch = -1;
            while (true) {
                qjm = new QuorumJournalManager(conf, uri, FAKE_NSINFO, new FaultyLoggerFactory());
                try {
                    qjm.createNewUniqueEpoch();
                    newEpoch = qjm.getLoggerSetForTests().getEpoch();
                    break;
                } catch (IOException ioe) {
                // It's OK to fail to create an epoch, since we randomly inject
                // faults. It's possible we'll inject faults in too many of the
                // underlying nodes, and a failure is expected in that case
                } finally {
                    qjm.close();
                }
            }
            LOG.info("Created epoch " + newEpoch);
            assertTrue("New epoch " + newEpoch + " should be greater than previous " + prevEpoch, newEpoch > prevEpoch);
            prevEpoch = newEpoch;
        }
    } finally {
        cluster.shutdown();
    }
}

Example 3 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestQJMWithFaults method determineMaxIpcNumber.

/**
   * Run through the creation of a log without any faults injected,
   * and count how many RPCs are made to each node. This sets the
   * bounds for the other test cases, so they can exhaustively explore
   * the space of potential failures.
   */
private static long determineMaxIpcNumber() throws Exception {
    Configuration conf = new Configuration();
    MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
    cluster.waitActive();
    QuorumJournalManager qjm = null;
    long ret;
    try {
        qjm = createInjectableQJM(cluster);
        qjm.format(FAKE_NSINFO);
        doWorkload(cluster, qjm);
        SortedSet<Integer> ipcCounts = Sets.newTreeSet();
        for (AsyncLogger l : qjm.getLoggerSetForTests().getLoggersForTests()) {
            InvocationCountingChannel ch = (InvocationCountingChannel) l;
            ch.waitForAllPendingCalls();
            ipcCounts.add(ch.getRpcCount());
        }
        // All of the loggers should have sent the same number of RPCs, since there
        // were no failures.
        assertEquals(1, ipcCounts.size());
        ret = ipcCounts.first();
        LOG.info("Max IPC count = " + ret);
    } finally {
        IOUtils.closeStream(qjm);
        cluster.shutdown();
    }
    return ret;
}

Example 4 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestRollingUpgradeRollback method testRollbackWithQJM.

@Test
public void testRollbackWithQJM() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    MiniJournalCluster mjc = null;
    MiniDFSCluster cluster = null;
    final Path foo = new Path("/foo");
    final Path bar = new Path("/bar");
    try {
        mjc = new MiniJournalCluster.Builder(conf).numJournalNodes(NUM_JOURNAL_NODES).build();
        mjc.waitActive();
        conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc.getQuorumJournalURI(JOURNAL_ID).toString());
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        cluster.waitActive();
        DistributedFileSystem dfs = cluster.getFileSystem();
        final DFSAdmin dfsadmin = new DFSAdmin(conf);
        dfs.mkdirs(foo);
        // start rolling upgrade
        dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        Assert.assertEquals(0, dfsadmin.run(new String[] { "-rollingUpgrade", "prepare" }));
        dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
        // create new directory
        dfs.mkdirs(bar);
        dfs.close();
        // rollback
        cluster.restartNameNode("-rollingUpgrade", "rollback");
        // make sure /foo is still there, but /bar is not
        dfs = cluster.getFileSystem();
        Assert.assertTrue(dfs.exists(foo));
        Assert.assertFalse(dfs.exists(bar));
        // check storage in JNs
        for (int i = 0; i < NUM_JOURNAL_NODES; i++) {
            File dir = mjc.getCurrentDir(0, JOURNAL_ID);
            // segments:(startSegment, mkdir, endSegment), (startSegment, upgrade
            // marker, mkdir, endSegment)
            checkJNStorage(dir, 4, 7);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
        if (mjc != null) {
            mjc.shutdown();
        }
    }
}

Example 5 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestQJMWithFaults method testRandomized.

/**
   * Test case in which three JournalNodes randomly flip flop between
   * up and down states every time they get an RPC.
   * 
   * The writer keeps track of the latest ACKed edit, and on every
   * recovery operation, ensures that it recovers at least to that
   * point or higher. Since at any given point, a majority of JNs
   * may be injecting faults, any writer operation is allowed to fail,
   * so long as the exception message indicates it failed due to injected
   * faults.
   * 
   * Given a random seed, the test should be entirely deterministic.
   */
@Test
public void testRandomized() throws Exception {
    long seed;
    Long userSpecifiedSeed = Long.getLong(RAND_SEED_PROPERTY);
    if (userSpecifiedSeed != null) {
        LOG.info("Using seed specified in system property");
        seed = userSpecifiedSeed;
        // If the user specifies a seed, then we should gather all the
        // IPC trace information so that debugging is easier. This makes
        // the test run about 25% slower otherwise.
        GenericTestUtils.setLogLevel(ProtobufRpcEngine.LOG, Level.ALL);
    } else {
        seed = new Random().nextLong();
    }
    LOG.info("Random seed: " + seed);
    Random r = new Random(seed);
    MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
    cluster.waitActive();
    // Format the cluster using a non-faulty QJM.
    QuorumJournalManager qjmForInitialFormat = createInjectableQJM(cluster);
    qjmForInitialFormat.format(FAKE_NSINFO);
    qjmForInitialFormat.close();
    try {
        long txid = 0;
        long lastAcked = 0;
        for (int i = 0; i < NUM_WRITER_ITERS; i++) {
            LOG.info("Starting writer " + i + "\n-------------------");
            QuorumJournalManager qjm = createRandomFaultyQJM(cluster, r);
            try {
                long recovered;
                try {
                    recovered = QJMTestUtil.recoverAndReturnLastTxn(qjm);
                } catch (Throwable t) {
                    LOG.info("Failed recovery", t);
                    checkException(t);
                    continue;
                }
                assertTrue("Recovered only up to txnid " + recovered + " but had gotten an ack for " + lastAcked, recovered >= lastAcked);
                txid = recovered + 1;
                // at failure cases.
                if (txid > 100 && i % 10 == 1) {
                    qjm.purgeLogsOlderThan(txid - 100);
                }
                Holder<Throwable> thrown = new Holder<Throwable>(null);
                for (int j = 0; j < SEGMENTS_PER_WRITER; j++) {
                    lastAcked = writeSegmentUntilCrash(cluster, qjm, txid, 4, thrown);
                    if (thrown.held != null) {
                        LOG.info("Failed write", thrown.held);
                        checkException(thrown.held);
                        break;
                    }
                    txid += 4;
                }
            } finally {
                qjm.close();
            }
        }
    } finally {
        cluster.shutdown();
    }
}