Examples with MiniDFSNNTopology org.apache.hadoop.hdfs.MiniDFSNNTopology used on opensource projects

Example 11 with MiniDFSNNTopology

use of org.apache.hadoop.hdfs.MiniDFSNNTopology in project hadoop by apache.

the class TestDataNodeHotSwapVolumes method startDFSCluster.

private void startDFSCluster(int numNameNodes, int numDataNodes, int storagePerDataNode) throws IOException {
    shutdown();
    Configuration conf = new Configuration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    /*
     * Lower the DN heartbeat, DF rate, and recheck interval to one second
     * so state about failures and datanode death propagates faster.
     */
    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
    /* Allow 1 volume failure */
    conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1);
    conf.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, 0, TimeUnit.MILLISECONDS);
    MiniDFSNNTopology nnTopology = MiniDFSNNTopology.simpleFederatedTopology(numNameNodes);
    cluster = new MiniDFSCluster.Builder(conf).nnTopology(nnTopology).numDataNodes(numDataNodes).storagesPerDatanode(storagePerDataNode).build();
    cluster.waitActive();
}

Example 12 with MiniDFSNNTopology

use of org.apache.hadoop.hdfs.MiniDFSNNTopology in project hadoop by apache.

the class TestDataNodeReconfiguration method startDFSCluster.

private void startDFSCluster(int numNameNodes, int numDataNodes) throws IOException {
    Configuration conf = new Configuration();
    MiniDFSNNTopology nnTopology = MiniDFSNNTopology.simpleFederatedTopology(numNameNodes);
    cluster = new MiniDFSCluster.Builder(conf).nnTopology(nnTopology).numDataNodes(numDataNodes).build();
    cluster.waitActive();
}

Example 13 with MiniDFSNNTopology

use of org.apache.hadoop.hdfs.MiniDFSNNTopology in project hadoop by apache.

the class TestRefreshNamenodes method testRefreshNamenodes.

@Test
public void testRefreshNamenodes() throws IOException {
    // Start cluster with a single NN and DN
    Configuration conf = new Configuration();
    MiniDFSCluster cluster = null;
    try {
        MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new NSConf("ns1").addNN(new NNConf(null).setIpcPort(nnPort1))).setFederation(true);
        cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology).build();
        DataNode dn = cluster.getDataNodes().get(0);
        assertEquals(1, dn.getAllBpOs().size());
        cluster.addNameNode(conf, nnPort2);
        assertEquals(2, dn.getAllBpOs().size());
        cluster.addNameNode(conf, nnPort3);
        assertEquals(3, dn.getAllBpOs().size());
        cluster.addNameNode(conf, nnPort4);
        // Ensure a BPOfferService in the datanodes corresponds to
        // a namenode in the cluster
        Set<InetSocketAddress> nnAddrsFromCluster = Sets.newHashSet();
        for (int i = 0; i < 4; i++) {
            assertTrue(nnAddrsFromCluster.add(cluster.getNameNode(i).getNameNodeAddress()));
        }
        Set<InetSocketAddress> nnAddrsFromDN = Sets.newHashSet();
        for (BPOfferService bpos : dn.getAllBpOs()) {
            for (BPServiceActor bpsa : bpos.getBPServiceActors()) {
                assertTrue(nnAddrsFromDN.add(bpsa.getNNSocketAddress()));
            }
        }
        assertEquals("", Joiner.on(",").join(Sets.symmetricDifference(nnAddrsFromCluster, nnAddrsFromDN)));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 14 with MiniDFSNNTopology

use of org.apache.hadoop.hdfs.MiniDFSNNTopology in project hadoop by apache.

the class TestEditLogAutoroll method setUp.

@Before
public void setUp() throws Exception {
    conf = new Configuration();
    // Stall the standby checkpointer in two ways
    conf.setLong(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, Long.MAX_VALUE);
    conf.setLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 20);
    // Make it autoroll after 10 edits
    conf.setFloat(DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD, 0.5f);
    conf.setInt(DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS, 100);
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, useAsyncEditLog);
    int retryCount = 0;
    while (true) {
        try {
            int basePort = 10060 + random.nextInt(100) * 2;
            MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)).addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1)));
            cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology).numDataNodes(0).build();
            cluster.waitActive();
            nn0 = cluster.getNameNode(0);
            fs = HATestUtil.configureFailoverFs(cluster, conf);
            cluster.transitionToActive(0);
            fs = cluster.getFileSystem(0);
            editLog = nn0.getNamesystem().getEditLog();
            ++retryCount;
            break;
        } catch (BindException e) {
            LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry " + retryCount + " times");
        }
    }
}

Example 15 with MiniDFSNNTopology

use of org.apache.hadoop.hdfs.MiniDFSNNTopology in project hadoop by apache.

the class TestEditLogTailer method testStandbyTriggersLogRolls.

private static void testStandbyTriggersLogRolls(int activeIndex) throws Exception {
    Configuration conf = getConf();
    // Roll every 1s
    conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
    MiniDFSCluster cluster = null;
    for (int i = 0; i < 5; i++) {
        try {
            // Have to specify IPC ports so the NNs can talk to each other.
            int[] ports = ServerSocketUtil.getPorts(3);
            MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(ports[0])).addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(ports[1])).addNN(new MiniDFSNNTopology.NNConf("nn3").setIpcPort(ports[2])));
            cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology).numDataNodes(0).build();
            break;
        } catch (BindException e) {
            // retry if race on ports given by ServerSocketUtil#getPorts
            continue;
        }
    }
    if (cluster == null) {
        fail("failed to start mini cluster.");
    }
    try {
        cluster.transitionToActive(activeIndex);
        waitForLogRollInSharedDir(cluster, 3);
    } finally {
        cluster.shutdown();
    }
}