Examples with DatanodeID org.apache.hadoop.hdfs.protocol.DatanodeID used on opensource projects

Example 46 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestHostFileManager method testIncludeExcludeLists.

@Test
@SuppressWarnings("unchecked")
public void testIncludeExcludeLists() throws IOException {
    BlockManager bm = mock(BlockManager.class);
    FSNamesystem fsn = mock(FSNamesystem.class);
    Configuration conf = new Configuration();
    HostFileManager hm = new HostFileManager();
    HostSet includedNodes = new HostSet();
    HostSet excludedNodes = new HostSet();
    includedNodes.add(entry("127.0.0.1:12345"));
    includedNodes.add(entry("localhost:12345"));
    includedNodes.add(entry("127.0.0.1:12345"));
    includedNodes.add(entry("127.0.0.2"));
    excludedNodes.add(entry("127.0.0.1:12346"));
    excludedNodes.add(entry("127.0.30.1:12346"));
    Assert.assertEquals(2, includedNodes.size());
    Assert.assertEquals(2, excludedNodes.size());
    hm.refresh(includedNodes, excludedNodes);
    DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
    Whitebox.setInternalState(dm, "hostConfigManager", hm);
    Map<String, DatanodeDescriptor> dnMap = (Map<String, DatanodeDescriptor>) Whitebox.getInternalState(dm, "datanodeMap");
    // After the de-duplication, there should be only one DN from the included
    // nodes declared as dead.
    Assert.assertEquals(2, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.ALL).size());
    Assert.assertEquals(2, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
    dnMap.put("uuid-foo", new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost", "uuid-foo", 12345, 1020, 1021, 1022)));
    Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
    dnMap.put("uuid-bar", new DatanodeDescriptor(new DatanodeID("127.0.0.2", "127.0.0.2", "uuid-bar", 12345, 1020, 1021, 1022)));
    Assert.assertEquals(0, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
    DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3", "127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022));
    DFSTestUtil.setDatanodeDead(spam);
    includedNodes.add(entry("127.0.0.3:12345"));
    dnMap.put("uuid-spam", spam);
    Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
    dnMap.remove("uuid-spam");
    Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
    excludedNodes.add(entry("127.0.0.3"));
    Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD).size());
}

Example 47 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestBlockReportRateLimiting method testRateLimitingDuringDataNodeStartup.

@Test(timeout = 180000)
public void testRateLimitingDuringDataNodeStartup() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(DFS_NAMENODE_MAX_FULL_BLOCK_REPORT_LEASES, 1);
    conf.setLong(DFS_NAMENODE_FULL_BLOCK_REPORT_LEASE_LENGTH_MS, 20L * 60L * 1000L);
    final Semaphore fbrSem = new Semaphore(0);
    final HashSet<DatanodeID> expectedFbrDns = new HashSet<>();
    final HashSet<DatanodeID> fbrDns = new HashSet<>();
    final AtomicReference<String> failure = new AtomicReference<String>("");
    final BlockManagerFaultInjector injector = new BlockManagerFaultInjector() {

        private int numLeases = 0;

        @Override
        public void incomingBlockReportRpc(DatanodeID nodeID, BlockReportContext context) throws IOException {
            LOG.info("Incoming full block report from " + nodeID + ".  Lease ID = 0x" + Long.toHexString(context.getLeaseId()));
            if (context.getLeaseId() == 0) {
                setFailure(failure, "Got unexpected rate-limiting-" + "bypassing full block report RPC from " + nodeID);
            }
            fbrSem.acquireUninterruptibly();
            synchronized (this) {
                fbrDns.add(nodeID);
                if (!expectedFbrDns.remove(nodeID)) {
                    setFailure(failure, "Got unexpected full block report " + "RPC from " + nodeID + ".  expectedFbrDns = " + Joiner.on(", ").join(expectedFbrDns));
                }
                LOG.info("Proceeding with full block report from " + nodeID + ".  Lease ID = 0x" + Long.toHexString(context.getLeaseId()));
            }
        }

        @Override
        public void requestBlockReportLease(DatanodeDescriptor node, long leaseId) {
            if (leaseId == 0) {
                return;
            }
            synchronized (this) {
                numLeases++;
                expectedFbrDns.add(node);
                LOG.info("requestBlockReportLease(node=" + node + ", leaseId=0x" + Long.toHexString(leaseId) + ").  " + "expectedFbrDns = " + Joiner.on(", ").join(expectedFbrDns));
                if (numLeases > 1) {
                    setFailure(failure, "More than 1 lease was issued at once.");
                }
            }
        }

        @Override
        public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) {
            LOG.info("removeBlockReportLease(node=" + node + ", leaseId=0x" + Long.toHexString(leaseId) + ")");
            synchronized (this) {
                numLeases--;
            }
        }
    };
    BlockManagerFaultInjector.instance = injector;
    final int NUM_DATANODES = 5;
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
    cluster.waitActive();
    for (int n = 1; n <= NUM_DATANODES; n++) {
        LOG.info("Waiting for " + n + " datanode(s) to report in.");
        fbrSem.release();
        Uninterruptibles.sleepUninterruptibly(20, TimeUnit.MILLISECONDS);
        final int currentN = n;
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                synchronized (injector) {
                    if (fbrDns.size() > currentN) {
                        setFailure(failure, "Expected at most " + currentN + " datanodes to have sent a block report, but actually " + fbrDns.size() + " have.");
                    }
                    return (fbrDns.size() >= currentN);
                }
            }
        }, 25, 50000);
    }
    cluster.shutdown();
    Assert.assertEquals("", failure.get());
}

Example 48 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestBlockTokenWithDFS method tryRead.

// try reading a block using a BlockReader directly
protected void tryRead(final Configuration conf, LocatedBlock lblock, boolean shouldSucceed) {
    InetSocketAddress targetAddr = null;
    IOException ioe = null;
    BlockReader blockReader = null;
    ExtendedBlock block = lblock.getBlock();
    try {
        DatanodeInfo[] nodes = lblock.getLocations();
        targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());
        blockReader = new BlockReaderFactory(new DfsClientConf(conf)).setFileName(BlockReaderFactory.getFileName(targetAddr, "test-blockpoolid", block.getBlockId())).setBlock(block).setBlockToken(lblock.getBlockToken()).setInetSocketAddress(targetAddr).setStartOffset(0).setLength(0).setVerifyChecksum(true).setClientName("TestBlockTokenWithDFS").setDatanodeInfo(nodes[0]).setCachingStrategy(CachingStrategy.newDefaultStrategy()).setClientCacheContext(ClientContext.getFromConf(conf)).setConfiguration(conf).setTracer(FsTracer.get(conf)).setRemotePeerFactory(new RemotePeerFactory() {

            @Override
            public Peer newConnectedPeer(InetSocketAddress addr, Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId) throws IOException {
                Peer peer = null;
                Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
                try {
                    sock.connect(addr, HdfsConstants.READ_TIMEOUT);
                    sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
                    peer = DFSUtilClient.peerFromSocket(sock);
                } finally {
                    if (peer == null) {
                        IOUtils.closeSocket(sock);
                    }
                }
                return peer;
            }
        }).build();
    } catch (IOException ex) {
        ioe = ex;
    } finally {
        if (blockReader != null) {
            try {
                blockReader.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    if (shouldSucceed) {
        Assert.assertNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid", blockReader);
    } else {
        Assert.assertNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid", ioe);
        Assert.assertTrue("OP_READ_BLOCK failed due to reasons other than access token: ", ioe instanceof InvalidBlockTokenException);
    }
}

Example 49 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testReduceReplFactorDueToRejoinRespectsRackPolicy.

/*
   * Test that when the excess replicas of a block are reduced due to
   * a node re-joining the cluster the rack policy is not violated.
   */
@Test
public void testReduceReplFactorDueToRejoinRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 2;
    final Path filePath = new Path("/testFile");
    // Last datanode is on a different rack
    String[] racks = { "/rack1", "/rack1", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
    try {
        // Create a file with one block
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Make the last (cross rack) datanode look like it failed
        // to heartbeat by stopping it and calling removeDatanode.
        ArrayList<DataNode> datanodes = cluster.getDataNodes();
        assertEquals(3, datanodes.size());
        DataNode dataNode = datanodes.get(2);
        DatanodeID dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(2);
        dm.removeDatanode(dnId);
        // The block gets re-replicated to another datanode so it has a 
        // sufficient # replicas, but not across racks, so there should
        // be 1 rack.
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
        // Start the "failed" datanode, which has a replica so the block is
        // now over-replicated and therefore a replica should be removed but
        // not on the restarted datanode as that would violate the rack policy.
        String[] rack2 = { "/rack2" };
        cluster.startDataNodes(conf, 1, true, null, rack2);
        cluster.waitActive();
        // The block now has sufficient # replicas, across racks
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}

Example 50 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestDataNodeVolumeFailure method accessBlock.

/**
   * try to access a block on a data node. If fails - throws exception
   * @param datanode
   * @param lblock
   * @throws IOException
   */
private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException {
    InetSocketAddress targetAddr = null;
    ExtendedBlock block = lblock.getBlock();
    targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
    BlockReader blockReader = new BlockReaderFactory(new DfsClientConf(conf)).setInetSocketAddress(targetAddr).setBlock(block).setFileName(BlockReaderFactory.getFileName(targetAddr, "test-blockpoolid", block.getBlockId())).setBlockToken(lblock.getBlockToken()).setStartOffset(0).setLength(0).setVerifyChecksum(true).setClientName("TestDataNodeVolumeFailure").setDatanodeInfo(datanode).setCachingStrategy(CachingStrategy.newDefaultStrategy()).setClientCacheContext(ClientContext.getFromConf(conf)).setConfiguration(conf).setTracer(FsTracer.get(conf)).setRemotePeerFactory(new RemotePeerFactory() {

        @Override
        public Peer newConnectedPeer(InetSocketAddress addr, Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId) throws IOException {
            Peer peer = null;
            Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
            try {
                sock.connect(addr, HdfsConstants.READ_TIMEOUT);
                sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
                peer = DFSUtilClient.peerFromSocket(sock);
            } finally {
                if (peer == null) {
                    IOUtils.closeSocket(sock);
                }
            }
            return peer;
        }
    }).build();
    blockReader.close();
}