Examples with MiniQJMHACluster org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster used on opensource projects

Example 1 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestRollingUpgrade method testCheckpoint.

public void testCheckpoint(int nnCount) throws IOException, InterruptedException {
    final Configuration conf = new Configuration();
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
    MiniQJMHACluster cluster = null;
    final Path foo = new Path("/foo");
    try {
        cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
        MiniDFSCluster dfsCluster = cluster.getDfsCluster();
        dfsCluster.waitActive();
        dfsCluster.transitionToActive(0);
        DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
        // start rolling upgrade
        RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
        Assert.assertTrue(info.isStarted());
        queryForPreparation(dfs);
        dfs.mkdirs(foo);
        long txid = dfs.rollEdits();
        Assert.assertTrue(txid > 0);
        for (int i = 1; i < nnCount; i++) {
            verifyNNCheckpoint(dfsCluster, txid, i);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 2 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestRollingUpgrade method testQuery.

private void testQuery(int nnCount) throws Exception {
    final Configuration conf = new Configuration();
    MiniQJMHACluster cluster = null;
    try {
        cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
        MiniDFSCluster dfsCluster = cluster.getDfsCluster();
        dfsCluster.waitActive();
        dfsCluster.transitionToActive(0);
        DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
        // shutdown other NNs
        for (int i = 1; i < nnCount; i++) {
            dfsCluster.shutdownNameNode(i);
        }
        // start rolling upgrade
        RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
        Assert.assertTrue(info.isStarted());
        info = dfs.rollingUpgrade(RollingUpgradeAction.QUERY);
        Assert.assertFalse(info.createdRollbackImages());
        // restart other NNs
        for (int i = 1; i < nnCount; i++) {
            dfsCluster.restartNameNode(i);
        }
        // check that one of the other NNs has created the rollback image and uploaded it
        queryForPreparation(dfs);
        // The NN should have a copy of the fsimage in case of rollbacks.
        Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage().hasRollbackFSImage());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 3 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestRollingUpgradeDowngrade method testDowngrade.

/**
   * Downgrade option is already obsolete. It should throw exception.
   * @throws Exception
   */
@Test(timeout = 300000, expected = IllegalArgumentException.class)
public void testDowngrade() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    MiniQJMHACluster cluster = null;
    final Path foo = new Path("/foo");
    final Path bar = new Path("/bar");
    try {
        cluster = new MiniQJMHACluster.Builder(conf).build();
        MiniDFSCluster dfsCluster = cluster.getDfsCluster();
        dfsCluster.waitActive();
        // let NN1 tail editlog every 1s
        dfsCluster.getConfiguration(1).setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
        dfsCluster.restartNameNode(1);
        dfsCluster.transitionToActive(0);
        DistributedFileSystem dfs = dfsCluster.getFileSystem(0);
        dfs.mkdirs(foo);
        // start rolling upgrade
        RollingUpgradeInfo info = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
        Assert.assertTrue(info.isStarted());
        dfs.mkdirs(bar);
        TestRollingUpgrade.queryForPreparation(dfs);
        dfs.close();
        dfsCluster.restartNameNode(0, true, "-rollingUpgrade", "downgrade");
        // Once downgraded, there should be no more fsimage for rollbacks.
        Assert.assertFalse(dfsCluster.getNamesystem(0).getFSImage().hasRollbackFSImage());
        // shutdown NN1
        dfsCluster.shutdownNameNode(1);
        dfsCluster.transitionToActive(0);
        dfs = dfsCluster.getFileSystem(0);
        Assert.assertTrue(dfs.exists(foo));
        Assert.assertTrue(dfs.exists(bar));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 4 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestBootstrapStandbyWithQJM method setup.

@Before
public void setup() throws Exception {
    Configuration conf = new Configuration();
    // Turn off IPC client caching, so that the suite can handle
    // the restart of the daemons between test cases.
    conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
    MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
    cluster = miniQjmHaCluster.getDfsCluster();
    jCluster = miniQjmHaCluster.getJournalCluster();
    // make nn0 active
    cluster.transitionToActive(0);
    // do sth to generate in-progress edit log data
    DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil.configureFailoverFs(cluster, conf);
    dfs.mkdirs(new Path("/test2"));
    dfs.close();
}

Example 5 with MiniQJMHACluster

use of org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster in project hadoop by apache.

the class TestDFSInotifyEventInputStream method testReadEventsWithTimeout.

@Test(timeout = 120000)
public void testReadEventsWithTimeout() throws IOException, InterruptedException, MissingEventsException {
    Configuration conf = new HdfsConfiguration();
    MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();
    try {
        cluster.getDfsCluster().waitActive();
        cluster.getDfsCluster().transitionToActive(0);
        final DFSClient client = new DFSClient(cluster.getDfsCluster().getNameNode(0).getNameNodeAddress(), conf);
        DFSInotifyEventInputStream eis = client.getInotifyEventStream();
        ScheduledExecutorService ex = Executors.newSingleThreadScheduledExecutor();
        ex.schedule(new Runnable() {

            @Override
            public void run() {
                try {
                    client.mkdirs("/dir", null, false);
                } catch (IOException e) {
                    // test will fail
                    LOG.error("Unable to create /dir", e);
                }
            }
        }, 1, TimeUnit.SECONDS);
        // a very generous wait period -- the edit will definitely have been
        // processed by the time this is up
        EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
        Assert.assertNotNull(batch);
        Assert.assertEquals(1, batch.getEvents().length);
        Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
        Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
    } finally {
        cluster.shutdown();
    }
}