Examples with NameNode org.apache.hadoop.hdfs.server.namenode.NameNode used on opensource projects

Example 1 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestDecommission method testClusterStats.

public void testClusterStats(int numNameNodes) throws IOException, InterruptedException {
    LOG.info("Starting test testClusterStats");
    int numDatanodes = 1;
    startCluster(numNameNodes, numDatanodes);
    for (int i = 0; i < numNameNodes; i++) {
        FileSystem fileSys = getCluster().getFileSystem(i);
        Path file = new Path("testClusterStats.dat");
        writeFile(fileSys, file, 1);
        FSNamesystem fsn = getCluster().getNamesystem(i);
        NameNode namenode = getCluster().getNameNode(i);
        DatanodeInfo decomInfo = takeNodeOutofService(i, null, 0, null, AdminStates.DECOMMISSION_INPROGRESS);
        DataNode decomNode = getDataNode(decomInfo);
        // Check namenode stats for multiple datanode heartbeats
        verifyStats(namenode, fsn, decomInfo, decomNode, true);
        // Stop decommissioning and verify stats
        DatanodeInfo retInfo = NameNodeAdapter.getDatanode(fsn, decomInfo);
        putNodeInService(i, retInfo);
        DataNode retNode = getDataNode(decomInfo);
        verifyStats(namenode, fsn, retInfo, retNode, false);
    }
}

Example 2 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestOpenFilesWithSnapshot method doTestMultipleSnapshots.

private void doTestMultipleSnapshots(boolean saveNamespace) throws IOException {
    Path path = new Path("/test");
    doWriteAndAbort(fs, path);
    fs.createSnapshot(path, "s2");
    fs.delete(new Path("/test/test"), true);
    fs.deleteSnapshot(path, "s2");
    cluster.triggerBlockReports();
    if (saveNamespace) {
        NameNode nameNode = cluster.getNameNode();
        NameNodeAdapter.enterSafeMode(nameNode, false);
        NameNodeAdapter.saveNamespace(nameNode);
        NameNodeAdapter.leaveSafeMode(nameNode);
    }
    cluster.restartNameNode(true);
}

Example 3 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestOpenFilesWithSnapshot method restartNameNode.

private void restartNameNode() throws Exception {
    cluster.triggerBlockReports();
    NameNode nameNode = cluster.getNameNode();
    NameNodeAdapter.enterSafeMode(nameNode, false);
    NameNodeAdapter.saveNamespace(nameNode);
    NameNodeAdapter.leaveSafeMode(nameNode);
    cluster.restartNameNode(true);
}

Example 4 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestNameNodeMetrics method testTransactionSinceLastCheckpointMetrics.

/**
   * Testing TransactionsSinceLastCheckpoint. Need a new cluster as
   * the other tests in here don't use HA. See HDFS-7501.
   */
@Test(timeout = 300000)
public void testTransactionSinceLastCheckpointMetrics() throws Exception {
    Random random = new Random();
    int retryCount = 0;
    while (retryCount < 5) {
        try {
            int basePort = 10060 + random.nextInt(100) * 2;
            MiniDFSNNTopology topology = new MiniDFSNNTopology().addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort)).addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1)));
            HdfsConfiguration conf2 = new HdfsConfiguration();
            // Lower the checkpoint condition for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 100);
            // Check for checkpoint condition very often, for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
            // Poll and follow ANN txns very often, for purpose of testing.
            conf2.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
            MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf2).nnTopology(topology).numDataNodes(1).build();
            cluster2.waitActive();
            DistributedFileSystem fs2 = cluster2.getFileSystem(0);
            NameNode nn0 = cluster2.getNameNode(0);
            NameNode nn1 = cluster2.getNameNode(1);
            cluster2.transitionToActive(0);
            fs2.mkdirs(new Path("/tmp-t1"));
            fs2.mkdirs(new Path("/tmp-t2"));
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Test to ensure tracking works before the first-ever
            // checkpoint.
            assertEquals("SBN failed to track 2 transactions pre-checkpoint.", // 2 txns added further when catch-up is called.
            4L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            // rounded at 100, as 4 + 94 + 2 (catch-up call) = 100.
            for (int i = 1; i <= 94; i++) {
                fs2.mkdirs(new Path("/tmp-" + i));
            }
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Assert 100 transactions in checkpoint.
            HATestUtil.waitForCheckpoint(cluster2, 1, ImmutableList.of(100));
            // Test to ensure number tracks the right state of
            // uncheckpointed edits, and does not go negative
            // (as fixed in HDFS-7501).
            assertEquals("Should be zero right after the checkpoint.", 0L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            fs2.mkdirs(new Path("/tmp-t3"));
            fs2.mkdirs(new Path("/tmp-t4"));
            HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
            // Test to ensure we track the right numbers after
            // the checkpoint resets it to zero again.
            assertEquals("SBN failed to track 2 added txns after the ckpt.", 4L, cluster2.getNameNode(1).getNamesystem().getTransactionsSinceLastCheckpoint());
            cluster2.shutdown();
            break;
        } catch (Exception e) {
            LOG.warn("Unable to set up HA cluster, exception thrown: " + e);
            retryCount++;
        }
    }
}

Example 5 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestDFSStripedOutputStreamWithFailure method runTest.

/**
   * runTest implementation.
   * @param length file length
   * @param killPos killing positions in ascending order
   * @param dnIndex DN index to kill when meets killing positions
   * @param tokenExpire wait token to expire when kill a DN
   * @throws Exception
   */
private void runTest(final int length, final int[] killPos, final int[] dnIndex, final boolean tokenExpire) throws Exception {
    if (killPos[0] <= FLUSH_POS) {
        LOG.warn("killPos=" + Arrays.toString(killPos) + " <= FLUSH_POS=" + FLUSH_POS + ", length=" + length + ", dnIndex=" + Arrays.toString(dnIndex));
        //skip test
        return;
    }
    Preconditions.checkArgument(length > killPos[0], "length=%s <= killPos=%s", length, killPos);
    Preconditions.checkArgument(killPos.length == dnIndex.length);
    final Path p = new Path(dir, "dn" + Arrays.toString(dnIndex) + "len" + length + "kill" + Arrays.toString(killPos));
    final String fullPath = p.toString();
    LOG.info("fullPath=" + fullPath);
    if (tokenExpire) {
        final NameNode nn = cluster.getNameNode();
        final BlockManager bm = nn.getNamesystem().getBlockManager();
        final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
        // set a short token lifetime (1 second)
        SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    }
    final AtomicInteger pos = new AtomicInteger();
    final FSDataOutputStream out = dfs.create(p);
    final DFSStripedOutputStream stripedOut = (DFSStripedOutputStream) out.getWrappedStream();
    // first GS of this block group which never proceeds blockRecovery
    long firstGS = -1;
    // the old GS before bumping
    long oldGS = -1;
    List<Long> gsList = new ArrayList<>();
    final List<DatanodeInfo> killedDN = new ArrayList<>();
    int numKilled = 0;
    for (; pos.get() < length; ) {
        final int i = pos.getAndIncrement();
        if (numKilled < killPos.length && i == killPos[numKilled]) {
            assertTrue(firstGS != -1);
            final long gs = getGenerationStamp(stripedOut);
            if (numKilled == 0) {
                assertEquals(firstGS, gs);
            } else {
                //TODO: implement hflush/hsync and verify gs strict greater than oldGS
                assertTrue(gs >= oldGS);
            }
            oldGS = gs;
            if (tokenExpire) {
                DFSTestUtil.flushInternal(stripedOut);
                waitTokenExpires(out);
            }
            killedDN.add(killDatanode(cluster, stripedOut, dnIndex[numKilled], pos));
            numKilled++;
        }
        write(out, i);
        if (i % blockGroupSize == FLUSH_POS) {
            firstGS = getGenerationStamp(stripedOut);
            oldGS = firstGS;
        }
        if (i > 0 && (i + 1) % blockGroupSize == 0) {
            gsList.add(oldGS);
        }
    }
    gsList.add(oldGS);
    out.close();
    assertEquals(dnIndex.length, numKilled);
    StripedFileTestUtil.waitBlockGroupsReported(dfs, fullPath, numKilled);
    cluster.triggerBlockReports();
    StripedFileTestUtil.checkData(dfs, p, length, killedDN, gsList, blockGroupSize);
}