Examples with DataNode org.apache.hadoop.hdfs.server.datanode.DataNode used on opensource projects

Example 91 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class MiniDFSCluster method addNameNode.

/**
   * Add a namenode to a federated cluster and start it. Configuration of
   * datanodes in the cluster is refreshed to register with the new namenode.
   * 
   * @return newly started namenode
   */
public void addNameNode(Configuration conf, int namenodePort) throws IOException {
    if (!federation)
        throw new IOException("cannot add namenode to non-federated cluster");
    int nameServiceIndex = namenodes.keys().size();
    String nameserviceId = NAMESERVICE_ID_PREFIX + (namenodes.keys().size() + 1);
    String nameserviceIds = conf.get(DFS_NAMESERVICES);
    nameserviceIds += "," + nameserviceId;
    conf.set(DFS_NAMESERVICES, nameserviceIds);
    String nnId = null;
    initNameNodeAddress(conf, nameserviceId, new NNConf(nnId).setIpcPort(namenodePort));
    // figure out the current number of NNs
    NameNodeInfo[] infos = this.getNameNodeInfos(nameserviceId);
    int nnIndex = infos == null ? 0 : infos.length;
    initNameNodeConf(conf, nameserviceId, nameServiceIndex, nnId, true, true, nnIndex);
    createNameNode(conf, true, null, null, nameserviceId, nnId);
    // Refresh datanodes with the newly started namenode
    for (DataNodeProperties dn : dataNodes) {
        DataNode datanode = dn.datanode;
        datanode.refreshNamenodes(conf);
    }
    // Wait for new namenode to get registrations from all the datanodes
    waitActive(nnIndex);
}

Example 92 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestBlockStoragePolicy method testChooseSsdOverDisk.

@Test
public void testChooseSsdOverDisk() throws Exception {
    BlockStoragePolicy policy = new BlockStoragePolicy((byte) 9, "TEST1", new StorageType[] { StorageType.SSD, StorageType.DISK, StorageType.ARCHIVE }, new StorageType[] {}, new StorageType[] {});
    final String[] racks = { "/d1/r1", "/d1/r1", "/d1/r1" };
    final String[] hosts = { "host1", "host2", "host3" };
    final StorageType[] disks = { StorageType.DISK, StorageType.DISK, StorageType.DISK };
    final DatanodeStorageInfo[] diskStorages = DFSTestUtil.createDatanodeStorageInfos(3, racks, hosts, disks);
    final DatanodeDescriptor[] dataNodes = DFSTestUtil.toDatanodeDescriptor(diskStorages);
    for (int i = 0; i < dataNodes.length; i++) {
        BlockManagerTestUtil.updateStorage(dataNodes[i], new DatanodeStorage("ssd" + i, DatanodeStorage.State.NORMAL, StorageType.SSD));
    }
    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
    File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath());
    DFSTestUtil.formatNameNode(conf);
    NameNode namenode = new NameNode(conf);
    final BlockManager bm = namenode.getNamesystem().getBlockManager();
    BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
    for (DatanodeDescriptor datanode : dataNodes) {
        cluster.add(datanode);
    }
    DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3, dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false, new HashSet<Node>(), 0, policy, null);
    System.out.println(policy.getName() + ": " + Arrays.asList(targets));
    Assert.assertEquals(2, targets.length);
    Assert.assertEquals(StorageType.SSD, targets[0].getStorageType());
    Assert.assertEquals(StorageType.DISK, targets[1].getStorageType());
}

Example 93 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryOnOOB.

/**
   * Test recovery on restart OOB message. It also tests the delivery of 
   * OOB ack originating from the primary datanode. Since there is only
   * one node in the cluster, failure of restart-recovery will fail the
   * test.
   */
@Test
public void testPipelineRecoveryOnOOB() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.set(HdfsClientConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, "15");
    MiniDFSCluster cluster = null;
    try {
        int numDataNodes = 1;
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
        cluster.waitActive();
        FileSystem fileSys = cluster.getFileSystem();
        Path file = new Path("dataprotocol2.dat");
        DFSTestUtil.createFile(fileSys, file, 10240L, (short) 1, 0L);
        DFSOutputStream out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
        out.write(1);
        out.hflush();
        DFSAdmin dfsadmin = new DFSAdmin(conf);
        DataNode dn = cluster.getDataNodes().get(0);
        final String dnAddr = dn.getDatanodeId().getIpcAddr(false);
        // issue shutdown to the datanode.
        final String[] args1 = { "-shutdownDatanode", dnAddr, "upgrade" };
        Assert.assertEquals(0, dfsadmin.run(args1));
        // Wait long enough to receive an OOB ack before closing the file.
        GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
        // Retart the datanode 
        cluster.restartDataNode(0, true);
        // The following forces a data packet and end of block packets to be sent. 
        out.close();
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 94 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryOnRestartFailure.

/** Test restart timeout */
@Test
public void testPipelineRecoveryOnRestartFailure() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.set(HdfsClientConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, "5");
    MiniDFSCluster cluster = null;
    try {
        int numDataNodes = 2;
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
        cluster.waitActive();
        FileSystem fileSys = cluster.getFileSystem();
        Path file = new Path("dataprotocol3.dat");
        DFSTestUtil.createFile(fileSys, file, 10240L, (short) 2, 0L);
        DFSOutputStream out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
        out.write(1);
        out.hflush();
        DFSAdmin dfsadmin = new DFSAdmin(conf);
        DataNode dn = cluster.getDataNodes().get(0);
        final String dnAddr1 = dn.getDatanodeId().getIpcAddr(false);
        // issue shutdown to the datanode.
        final String[] args1 = { "-shutdownDatanode", dnAddr1, "upgrade" };
        Assert.assertEquals(0, dfsadmin.run(args1));
        GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
        // This should succeed without restarting the node. The restart will
        // expire and regular pipeline recovery will kick in. 
        out.close();
        // At this point there is only one node in the cluster. 
        out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
        out.write(1);
        out.hflush();
        dn = cluster.getDataNodes().get(1);
        final String dnAddr2 = dn.getDatanodeId().getIpcAddr(false);
        // issue shutdown to the datanode.
        final String[] args2 = { "-shutdownDatanode", dnAddr2, "upgrade" };
        Assert.assertEquals(0, dfsadmin.run(args2));
        GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
        try {
            // close should fail
            out.close();
            assert false;
        } catch (IOException ioe) {
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 95 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryWithTransferBlock.

// Test to verify that blocks are no longer corrupted after HDFS-4660.
// Revert HDFS-4660 and the other related ones (HDFS-9220, HDFS-8722), this
// test would fail.
// Scenario: Prior to the fix, block get corrupted when the transferBlock
// happens during pipeline recovery with extra bytes to make up the end of
// chunk.
// For verification, Need to fail the pipeline for last datanode when the
// second datanode have more bytes on disk than already acked bytes.
// This will enable to transfer extra bytes to the newNode to makeup
// end-of-chunk during pipeline recovery. This is achieved by the customized
// DataNodeFaultInjector class in this test.
// For detailed info, please refer to HDFS-4660 and HDFS-10587. HDFS-9220
// fixes an issue in HDFS-4660 patch, and HDFS-8722 is an optimization.
@Test
public void testPipelineRecoveryWithTransferBlock() throws Exception {
    final int chunkSize = 512;
    final int oneWriteSize = 5000;
    final int totalSize = 1024 * 1024;
    final int errorInjectionPos = 512;
    Configuration conf = new HdfsConfiguration();
    // Need 4 datanodes to verify the replaceDatanode during pipeline recovery
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    DataNodeFaultInjector old = DataNodeFaultInjector.get();
    try {
        DistributedFileSystem fs = cluster.getFileSystem();
        Path fileName = new Path("/f");
        FSDataOutputStream o = fs.create(fileName);
        int count = 0;
        // Flush to get the pipeline created.
        o.writeBytes("hello");
        o.hflush();
        DFSOutputStream dfsO = (DFSOutputStream) o.getWrappedStream();
        final DatanodeInfo[] pipeline = dfsO.getStreamer().getNodes();
        final String lastDn = pipeline[2].getXferAddr(false);
        final AtomicBoolean failed = new AtomicBoolean(false);
        DataNodeFaultInjector.set(new DataNodeFaultInjector() {

            @Override
            public void failPipeline(ReplicaInPipeline replicaInfo, String mirror) throws IOException {
                if (!lastDn.equals(mirror)) {
                    // Only fail for second DN
                    return;
                }
                if (!failed.get() && (replicaInfo.getBytesAcked() > errorInjectionPos) && (replicaInfo.getBytesAcked() % chunkSize != 0)) {
                    int count = 0;
                    while (count < 10) {
                        // described in HDFS-4660 would occur.
                        if ((replicaInfo.getBytesOnDisk() / chunkSize) - (replicaInfo.getBytesAcked() / chunkSize) >= 1) {
                            failed.set(true);
                            throw new IOException("Failing Pipeline " + replicaInfo.getBytesAcked() + " : " + replicaInfo.getBytesOnDisk());
                        }
                        try {
                            Thread.sleep(200);
                        } catch (InterruptedException e) {
                        }
                        count++;
                    }
                }
            }
        });
        Random r = new Random();
        byte[] b = new byte[oneWriteSize];
        while (count < totalSize) {
            r.nextBytes(b);
            o.write(b);
            count += oneWriteSize;
            o.hflush();
        }
        assertTrue("Expected a failure in the pipeline", failed.get());
        DatanodeInfo[] newNodes = dfsO.getStreamer().getNodes();
        o.close();
        // Trigger block report to NN
        for (DataNode d : cluster.getDataNodes()) {
            DataNodeTestUtils.triggerBlockReport(d);
        }
        // Read from the replaced datanode to verify the corruption. So shutdown
        // all other nodes in the pipeline.
        List<DatanodeInfo> pipelineList = Arrays.asList(pipeline);
        DatanodeInfo newNode = null;
        for (DatanodeInfo node : newNodes) {
            if (!pipelineList.contains(node)) {
                newNode = node;
                break;
            }
        }
        LOG.info("Number of nodes in pipeline: {} newNode {}", newNodes.length, newNode.getName());
        // shutdown old 2 nodes
        for (int i = 0; i < newNodes.length; i++) {
            if (newNodes[i].getName().equals(newNode.getName())) {
                continue;
            }
            LOG.info("shutdown {}", newNodes[i].getName());
            cluster.stopDataNode(newNodes[i].getName());
        }
        // Read should be successfull from only the newNode. There should not be
        // any corruption reported.
        DFSTestUtil.readFile(fs, fileName);
    } finally {
        DataNodeFaultInjector.set(old);
        cluster.shutdown();
    }
}