Examples with MiniJournalCluster org.apache.hadoop.hdfs.qjournal.MiniJournalCluster used on opensource projects

Example 6 with MiniJournalCluster

use of org.apache.hadoop.hdfs.qjournal.MiniJournalCluster in project hadoop by apache.

the class TestQJMWithFaults method testRecoverAfterDoubleFailures.

/**
   * Sets up two of the nodes to each drop a single RPC, at all
   * possible combinations of RPCs. This may result in the
   * active writer failing to write. After this point, a new writer
   * should be able to recover and continue writing without
   * data loss.
   */
@Test
public void testRecoverAfterDoubleFailures() throws Exception {
    final long MAX_IPC_NUMBER = determineMaxIpcNumber();
    for (int failA = 1; failA <= MAX_IPC_NUMBER; failA++) {
        for (int failB = 1; failB <= MAX_IPC_NUMBER; failB++) {
            String injectionStr = "(" + failA + ", " + failB + ")";
            LOG.info("\n\n-------------------------------------------\n" + "Beginning test, failing at " + injectionStr + "\n" + "-------------------------------------------\n\n");
            MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
            cluster.waitActive();
            QuorumJournalManager qjm = null;
            try {
                qjm = createInjectableQJM(cluster);
                qjm.format(FAKE_NSINFO);
                List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
                failIpcNumber(loggers.get(0), failA);
                failIpcNumber(loggers.get(1), failB);
                int lastAckedTxn = doWorkload(cluster, qjm);
                if (lastAckedTxn < 6) {
                    LOG.info("Failed after injecting failures at " + injectionStr + ". This is expected since we injected a failure in the " + "majority.");
                }
                qjm.close();
                qjm = null;
                // Now should be able to recover
                qjm = createInjectableQJM(cluster);
                long lastRecoveredTxn = QJMTestUtil.recoverAndReturnLastTxn(qjm);
                assertTrue(lastRecoveredTxn >= lastAckedTxn);
                writeSegment(cluster, qjm, lastRecoveredTxn + 1, 3, true);
            } catch (Throwable t) {
                // easily triaged.
                throw new RuntimeException("Test failed with injection: " + injectionStr, t);
            } finally {
                cluster.shutdown();
                cluster = null;
                IOUtils.closeStream(qjm);
                qjm = null;
            }
        }
    }
}