Examples with NameNode org.apache.hadoop.hdfs.server.namenode.NameNode used on opensource projects

Example 26 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestNameNodeMetrics method testTransactionSinceLastCheckpointMetrics.

/**
   * Testing TransactionsSinceLastCheckpoint. Need a new cluster as
   * the other tests in here don't use HA. See HDFS-7501.
   */
@Test(timeout = 300000)
public void testTransactionSinceLastCheckpointMetrics() throws Exception {
    Random random = new Random();
    int retryCount = 0;
    while (retryCount < 5) {
        try {
            int basePort = 10060 + random.nextInt(100) * 2;
            MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)).addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1)));
            HdfsConfiguration conf2 = new HdfsConfiguration();
            // Lower the checkpoint condition for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 100);
            // Check for checkpoint condition very often, for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
            // Poll and follow ANN txns very often, for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
            MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf2).nnTopology(topology).numDataNodes(1).build();
            cluster2.waitActive();
            DistributedFileSystem fs2 = cluster2.getFileSystem(0);
            NameNode nn0 = cluster2.getNameNode(0);
            NameNode nn1 = cluster2.getNameNode(1);
            cluster2.transitionToActive(0);
            fs2.mkdirs(new Path("/tmp-t1"));
            fs2.mkdirs(new Path("/tmp-t2"));
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Test to ensure tracking works before the first-ever
            // checkpoint.
            assertEquals("SBN failed to track 2 transactions pre-checkpoint.", // 2 txns added further when catch-up is called.
            4L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            // rounded at 100, as 4 + 94 + 2 (catch-up call) = 100.
            for (int i = 1; i <= 94; i++) {
                fs2.mkdirs(new Path("/tmp-" + i));
            }
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Assert 100 transactions in checkpoint.
            HATestUtil.waitForCheckpoint(cluster2, 1, ImmutableList.of(100));
            // Test to ensure number tracks the right state of
            // uncheckpointed edits, and does not go negative
            // (as fixed in HDFS-7501).
            assertEquals("Should be zero right after the checkpoint.", 0L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            fs2.mkdirs(new Path("/tmp-t3"));
            fs2.mkdirs(new Path("/tmp-t4"));
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Test to ensure we track the right numbers after
            // the checkpoint resets it to zero again.
            assertEquals("SBN failed to track 2 added txns after the ckpt.", 4L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            cluster2.shutdown();
            break;
        } catch (Exception e) {
            LOG.warn("Unable to set up HA cluster, exception thrown: " + e);
            retryCount++;
        }
    }
}

Example 27 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestDFSStripedOutputStreamWithFailure method runTest.

/**
   * runTest implementation.
   * @param length file length
   * @param killPos killing positions in ascending order
   * @param dnIndex DN index to kill when meets killing positions
   * @param tokenExpire wait token to expire when kill a DN
   * @throws Exception
   */
private void runTest(final int length, final int[] killPos, final int[] dnIndex, final boolean tokenExpire) throws Exception {
    if (killPos[0] <= FLUSH_POS) {
        LOG.warn("killPos=" + Arrays.toString(killPos) + " <= FLUSH_POS=" + FLUSH_POS + ", length=" + length + ", dnIndex=" + Arrays.toString(dnIndex));
        //skip test
        return;
    }
    Preconditions.checkArgument(length > killPos[0], "length=%s <= killPos=%s", length, killPos);
    Preconditions.checkArgument(killPos.length == dnIndex.length);
    final Path p = new Path(dir, "dn" + Arrays.toString(dnIndex) + "len" + length + "kill" + Arrays.toString(killPos));
    final String fullPath = p.toString();
    LOG.info("fullPath=" + fullPath);
    if (tokenExpire) {
        final NameNode nn = cluster.getNameNode();
        final BlockManager bm = nn.getNamesystem().getBlockManager();
        final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
        // set a short token lifetime (1 second)
        SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    }
    final AtomicInteger pos = new AtomicInteger();
    final FSDataOutputStream out = dfs.create(p);
    final DFSStripedOutputStream stripedOut = (DFSStripedOutputStream) out.getWrappedStream();
    // first GS of this block group which never proceeds blockRecovery
    long firstGS = -1;
    // the old GS before bumping
    long oldGS = -1;
    List<Long> gsList = new ArrayList<>();
    final List<DatanodeInfo> killedDN = new ArrayList<>();
    int numKilled = 0;
    for (; pos.get() < length; ) {
        final int i = pos.getAndIncrement();
        if (numKilled < killPos.length && i == killPos[numKilled]) {
            assertTrue(firstGS != -1);
            final long gs = getGenerationStamp(stripedOut);
            if (numKilled == 0) {
                assertEquals(firstGS, gs);
            } else {
                //TODO: implement hflush/hsync and verify gs strict greater than oldGS
                assertTrue(gs >= oldGS);
            }
            oldGS = gs;
            if (tokenExpire) {
                DFSTestUtil.flushInternal(stripedOut);
                waitTokenExpires(out);
            }
            killedDN.add(killDatanode(cluster, stripedOut, dnIndex[numKilled], pos));
            numKilled++;
        }
        write(out, i);
        if (i % blockGroupSize == FLUSH_POS) {
            firstGS = getGenerationStamp(stripedOut);
            oldGS = firstGS;
        }
        if (i > 0 && (i + 1) % blockGroupSize == 0) {
            gsList.add(oldGS);
        }
    }
    gsList.add(oldGS);
    out.close();
    assertEquals(dnIndex.length, numKilled);
    StripedFileTestUtil.waitBlockGroupsReported(dfs, fullPath, numKilled);
    cluster.triggerBlockReports();
    StripedFileTestUtil.checkData(dfs, p, length, killedDN, gsList, blockGroupSize);
}

Example 28 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestDataNodeMultipleRegistrations method testClusterIdMismatch.

@Test
public void testClusterIdMismatch() throws Exception {
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)).build();
    try {
        cluster.waitActive();
        DataNode dn = cluster.getDataNodes().get(0);
        List<BPOfferService> bposs = dn.getAllBpOs();
        LOG.info("dn bpos len (should be 2):" + bposs.size());
        Assert.assertEquals("should've registered with two namenodes", bposs.size(), 2);
        // add another namenode
        cluster.addNameNode(conf, 9938);
        // lets wait for the registration to happen
        Thread.sleep(500);
        bposs = dn.getAllBpOs();
        LOG.info("dn bpos len (should be 3):" + bposs.size());
        Assert.assertEquals("should've registered with three namenodes", bposs.size(), 3);
        // change cluster id and another Namenode
        StartupOption.FORMAT.setClusterId("DifferentCID");
        cluster.addNameNode(conf, 9948);
        NameNode nn4 = cluster.getNameNode(3);
        assertNotNull("cannot create nn4", nn4);
        // lets wait for the registration to happen
        Thread.sleep(500);
        bposs = dn.getAllBpOs();
        LOG.info("dn bpos len (still should be 3):" + bposs.size());
        Assert.assertEquals("should've registered with three namenodes", 3, bposs.size());
    } finally {
        cluster.shutdown();
    }
}

Example 29 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestAclConfigFlag method restart.

/**
   * Restart the cluster, optionally saving a new checkpoint.
   *
   * @param checkpoint boolean true to save a new checkpoint
   * @param aclsEnabled if true, ACL support is enabled
   * @throws Exception if restart fails
   */
private void restart(boolean checkpoint, boolean aclsEnabled) throws Exception {
    NameNode nameNode = cluster.getNameNode();
    if (checkpoint) {
        NameNodeAdapter.enterSafeMode(nameNode, false);
        NameNodeAdapter.saveNamespace(nameNode);
    }
    shutdown();
    initCluster(false, aclsEnabled);
}

Example 30 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestXAttrConfigFlag method restart.

/**
   * Restart the cluster, optionally saving a new checkpoint.
   *
   * @param checkpoint boolean true to save a new checkpoint
   * @param xattrsEnabled if true, XAttr support is enabled
   * @throws Exception if restart fails
   */
private void restart(boolean checkpoint, boolean xattrsEnabled) throws Exception {
    NameNode nameNode = cluster.getNameNode();
    if (checkpoint) {
        NameNodeAdapter.enterSafeMode(nameNode, false);
        NameNodeAdapter.saveNamespace(nameNode);
    }
    shutdown();
    initCluster(false, xattrsEnabled);
}