Examples with DatanodeInfo org.apache.hadoop.hdfs.protocol.DatanodeInfo used on opensource projects

Example 46 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestBlockPlacementPolicyRackFaultTolerant method doTestLocatedBlock.

private void doTestLocatedBlock(int replication, LocatedBlock locatedBlock) {
    assertEquals(replication, locatedBlock.getLocations().length);
    HashMap<String, Integer> racksCount = new HashMap<String, Integer>();
    for (DatanodeInfo node : locatedBlock.getLocations()) {
        addToRacksCount(node.getNetworkLocation(), racksCount);
    }
    int minCount = Integer.MAX_VALUE;
    int maxCount = Integer.MIN_VALUE;
    for (Integer rackCount : racksCount.values()) {
        minCount = Math.min(minCount, rackCount);
        maxCount = Math.max(maxCount, rackCount);
    }
    assertTrue(maxCount - minCount <= 1);
}

Example 47 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestBlockPlacementPolicyRackFaultTolerant method doTestChooseTargetSpecialCase.

/**
   * Test more randomly. So it covers some special cases.
   * Like when some racks already have 2 replicas, while some racks have none,
   * we should choose the racks that have none.
   */
private void doTestChooseTargetSpecialCase() throws Exception {
    String clientMachine = "client.foo.com";
    // Test 5 files
    String src = "/testfile_1_";
    // Create the file with client machine
    HdfsFileStatus fileStatus = namesystem.startFile(src, perm, clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true, (short) 20, DEFAULT_BLOCK_SIZE, null, false);
    //test chooseTarget for new file
    LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine, null, null, fileStatus.getFileId(), null, null);
    doTestLocatedBlock(20, locatedBlock);
    DatanodeInfo[] locs = locatedBlock.getLocations();
    String[] storageIDs = locatedBlock.getStorageIDs();
    for (int time = 0; time < 5; time++) {
        shuffle(locs, storageIDs);
        for (int i = 1; i < locs.length; i++) {
            DatanodeInfo[] partLocs = new DatanodeInfo[i];
            String[] partStorageIDs = new String[i];
            System.arraycopy(locs, 0, partLocs, 0, i);
            System.arraycopy(storageIDs, 0, partStorageIDs, 0, i);
            for (int j = 1; j < 20 - i; j++) {
                LocatedBlock additionalLocatedBlock = nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(), locatedBlock.getBlock(), partLocs, partStorageIDs, new DatanodeInfo[0], j, clientMachine);
                doTestLocatedBlock(i + j, additionalLocatedBlock);
            }
        }
    }
}

Example 48 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestWebHdfsDataLocality method testExcludeDataNodes.

@Test
public void testExcludeDataNodes() throws Exception {
    final Configuration conf = WebHdfsTestUtil.createConf();
    final String[] racks = { RACK0, RACK0, RACK1, RACK1, RACK2, RACK2 };
    final String[] hosts = { "DataNode1", "DataNode2", "DataNode3", "DataNode4", "DataNode5", "DataNode6" };
    final int nDataNodes = hosts.length;
    LOG.info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.asList(racks) + ", hosts=" + Arrays.asList(hosts));
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).hosts(hosts).numDataNodes(nDataNodes).racks(racks).build();
    try {
        cluster.waitActive();
        final DistributedFileSystem dfs = cluster.getFileSystem();
        final NameNode namenode = cluster.getNameNode();
        final DatanodeManager dm = namenode.getNamesystem().getBlockManager().getDatanodeManager();
        LOG.info("dm=" + dm);
        final long blocksize = DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
        final String f = "/foo";
        //create a file with three replica.
        final Path p = new Path(f);
        final FSDataOutputStream out = dfs.create(p, (short) 3);
        out.write(1);
        out.close();
        //get replica location.
        final LocatedBlocks locatedblocks = NameNodeAdapter.getBlockLocations(namenode, f, 0, 1);
        final List<LocatedBlock> lb = locatedblocks.getLocatedBlocks();
        Assert.assertEquals(1, lb.size());
        final DatanodeInfo[] locations = lb.get(0).getLocations();
        Assert.assertEquals(3, locations.length);
        //For GETFILECHECKSUM, OPEN and APPEND,
        //the chosen datanode must be different with exclude nodes.
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < 2; i++) {
            sb.append(locations[i].getXferAddr());
            {
                // test GETFILECHECKSUM
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            {
                // test OPEN
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, GetOpParam.Op.OPEN, 0, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            {
                // test APPEND
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            sb.append(",");
        }
    } finally {
        cluster.shutdown();
    }
}

Example 49 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestShortCircuitCache method testAllocShm.

@Test(timeout = 60000)
public void testAllocShm() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(0, info.size());
        }
    });
    DomainPeer peer = getDomainPeerToDn(conf);
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty
            Assert.assertEquals(1, info.size());
            PerDatanodeVisitorInfo vinfo = info.get(datanode);
            Assert.assertFalse(vinfo.disabled);
            Assert.assertEquals(0, vinfo.full.size());
            Assert.assertEquals(1, vinfo.notFull.size());
        }
    });
    cache.scheduleSlotReleaser(slot);
    // Wait for the slot to be released, and the shared memory area to be
    // closed.  Since we didn't register this shared memory segment on the
    // server, it will also be a test of how well the server deals with
    // bogus client behavior.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            final MutableBoolean done = new MutableBoolean(false);
            try {
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}

Example 50 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestShortCircuitCache method testUnlinkingReplicasInFileDescriptorCache.

/**
   * Test unlinking a file whose blocks we are caching in the DFSClient.
   * The DataNode will notify the DFSClient that the replica is stale via the
   * ShortCircuitShm.
   */
@Test(timeout = 60000)
public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
    BlockReaderTestUtil.enableShortCircuitShmTracing();
    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
    Configuration conf = createShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache", sockDir);
    // We don't want the CacheCleaner to time out short-circuit shared memory
    // segments during the test, so set the timeout really high.
    conf.setLong(HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY, 1000000000L);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs = cluster.getFileSystem();
    final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            // The ClientShmManager starts off empty.
            Assert.assertEquals(0, info.size());
        }
    });
    final Path TEST_PATH = new Path("/test_file");
    final int TEST_FILE_LEN = 8193;
    final int SEED = 0xFADE0;
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, (short) 1, SEED);
    byte[] contents = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
    byte[] expected = DFSTestUtil.calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    // Loading this file brought the ShortCircuitReplica into our local
    // replica cache.
    final DatanodeInfo datanode = new DatanodeInfoBuilder().setNodeID(cluster.getDataNodes().get(0).getDatanodeId()).build();
    cache.getDfsClientShmManager().visit(new Visitor() {

        @Override
        public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
            Assert.assertTrue(info.get(datanode).full.isEmpty());
            Assert.assertFalse(info.get(datanode).disabled);
            Assert.assertEquals(1, info.get(datanode).notFull.values().size());
            DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
            Assert.assertFalse(shm.isDisconnected());
        }
    });
    // Remove the file whose blocks we just read.
    fs.delete(TEST_PATH, false);
    // Wait for the replica to be purged from the DFSClient's cache.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        MutableBoolean done = new MutableBoolean(true);

        @Override
        public Boolean get() {
            try {
                done.setValue(true);
                cache.getDfsClientShmManager().visit(new Visitor() {

                    @Override
                    public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) throws IOException {
                        Assert.assertTrue(info.get(datanode).full.isEmpty());
                        Assert.assertFalse(info.get(datanode).disabled);
                        Assert.assertEquals(1, info.get(datanode).notFull.values().size());
                        DfsClientShm shm = info.get(datanode).notFull.values().iterator().next();
                        // Check that all slots have been invalidated.
                        for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
                            Slot slot = iter.next();
                            if (slot.isValid()) {
                                done.setValue(false);
                            }
                        }
                    }
                });
            } catch (IOException e) {
                LOG.error("error running visitor", e);
            }
            return done.booleanValue();
        }
    }, 10, 60000);
    cluster.shutdown();
    sockDir.close();
}