Examples with DatanodeID org.apache.hadoop.hdfs.protocol.DatanodeID used on opensource projects

Example 16 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testReplDueToNodeFailRespectsRackPolicy.

/*
   * Test that when a block is replicated because a replica is lost due
   * to host failure the the rack policy is preserved.
   */
@Test
public void testReplDueToNodeFailRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 3;
    final Path filePath = new Path("/testFile");
    // Last datanode is on a different rack
    String[] racks = { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
    try {
        // Create a file with one block with a replication factor of 3
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Make the last datanode look like it failed to heartbeat by 
        // calling removeDatanode and stopping it.
        ArrayList<DataNode> datanodes = cluster.getDataNodes();
        int idx = datanodes.size() - 1;
        DataNode dataNode = datanodes.get(idx);
        DatanodeID dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // The block should still have sufficient # replicas, across racks.
        // The last node may not have contained a replica, but if it did
        // it should have been replicated within the same rack.
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Fail the last datanode again, it's also on rack2 so there is
        // only 1 rack for all the replicas
        datanodes = cluster.getDataNodes();
        idx = datanodes.size() - 1;
        dataNode = datanodes.get(idx);
        dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // Make sure we have enough live replicas even though we are
        // short one rack. The cluster now has only 1 rack thus we just make sure
        // we still have 3 replicas.
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}

Example 17 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestCachedBlocksList method testSingleList.

@Test(timeout = 60000)
public void testSingleList() {
    DatanodeDescriptor dn = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002, 5003));
    CachedBlock[] blocks = new CachedBlock[] { new CachedBlock(0L, (short) 1, true), new CachedBlock(1L, (short) 1, true), new CachedBlock(2L, (short) 1, true) };
    // check that lists are empty
    Assert.assertTrue("expected pending cached list to start off empty.", !dn.getPendingCached().iterator().hasNext());
    Assert.assertTrue("expected cached list to start off empty.", !dn.getCached().iterator().hasNext());
    Assert.assertTrue("expected pending uncached list to start off empty.", !dn.getPendingUncached().iterator().hasNext());
    // add a block to the back
    Assert.assertTrue(dn.getCached().add(blocks[0]));
    Assert.assertTrue("expected pending cached list to still be empty.", !dn.getPendingCached().iterator().hasNext());
    Assert.assertEquals("failed to insert blocks[0]", blocks[0], dn.getCached().iterator().next());
    Assert.assertTrue("expected pending uncached list to still be empty.", !dn.getPendingUncached().iterator().hasNext());
    // add another block to the back
    Assert.assertTrue(dn.getCached().add(blocks[1]));
    Iterator<CachedBlock> iter = dn.getCached().iterator();
    Assert.assertEquals(blocks[0], iter.next());
    Assert.assertEquals(blocks[1], iter.next());
    Assert.assertTrue(!iter.hasNext());
    // add a block to the front
    Assert.assertTrue(dn.getCached().addFirst(blocks[2]));
    iter = dn.getCached().iterator();
    Assert.assertEquals(blocks[2], iter.next());
    Assert.assertEquals(blocks[0], iter.next());
    Assert.assertEquals(blocks[1], iter.next());
    Assert.assertTrue(!iter.hasNext());
    // remove a block from the middle
    Assert.assertTrue(dn.getCached().remove(blocks[0]));
    iter = dn.getCached().iterator();
    Assert.assertEquals(blocks[2], iter.next());
    Assert.assertEquals(blocks[1], iter.next());
    Assert.assertTrue(!iter.hasNext());
    // remove all blocks
    dn.getCached().clear();
    Assert.assertTrue("expected cached list to be empty after clear.", !dn.getPendingCached().iterator().hasNext());
}

Example 18 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestCachedBlocksList method testMultipleLists.

@Test(timeout = 60000)
public void testMultipleLists() {
    DatanodeDescriptor[] datanodes = new DatanodeDescriptor[] { new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002, 5003)), new DatanodeDescriptor(new DatanodeID("127.0.1.1", "localhost", "efgh", 6000, 6001, 6002, 6003)) };
    CachedBlocksList[] lists = new CachedBlocksList[] { datanodes[0].getPendingCached(), datanodes[0].getCached(), datanodes[1].getPendingCached(), datanodes[1].getCached(), datanodes[1].getPendingUncached() };
    final int NUM_BLOCKS = 8000;
    CachedBlock[] blocks = new CachedBlock[NUM_BLOCKS];
    for (int i = 0; i < NUM_BLOCKS; i++) {
        blocks[i] = new CachedBlock(i, (short) i, true);
    }
    Random r = new Random(654);
    for (CachedBlocksList list : lists) {
        testAddElementsToList(list, blocks);
    }
    for (CachedBlocksList list : lists) {
        testRemoveElementsFromList(r, list, blocks);
    }
}

Example 19 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestBlockReportRateLimiting method testLeaseExpiration.

/**
   * Start a 2-node cluster with only one block report lease.  When the
   * first datanode gets a lease, kill it.  Then wait for the lease to
   * expire, and the second datanode to send a full block report.
   */
@Test(timeout = 180000)
public void testLeaseExpiration() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(DFS_NAMENODE_MAX_FULL_BLOCK_REPORT_LEASES, 1);
    conf.setLong(DFS_NAMENODE_FULL_BLOCK_REPORT_LEASE_LENGTH_MS, 100L);
    final Semaphore gotFbrSem = new Semaphore(0);
    final AtomicReference<String> failure = new AtomicReference<>();
    final AtomicReference<MiniDFSCluster> cluster = new AtomicReference<>();
    final AtomicReference<String> datanodeToStop = new AtomicReference<>();
    final BlockManagerFaultInjector injector = new BlockManagerFaultInjector() {

        @Override
        public void incomingBlockReportRpc(DatanodeID nodeID, BlockReportContext context) throws IOException {
            if (context.getLeaseId() == 0) {
                setFailure(failure, "Got unexpected rate-limiting-" + "bypassing full block report RPC from " + nodeID);
            }
            if (nodeID.getXferAddr().equals(datanodeToStop.get())) {
                throw new IOException("Injecting failure into block " + "report RPC for " + nodeID);
            }
            gotFbrSem.release();
        }

        @Override
        public void requestBlockReportLease(DatanodeDescriptor node, long leaseId) {
            if (leaseId == 0) {
                return;
            }
            datanodeToStop.compareAndSet(null, node.getXferAddr());
        }

        @Override
        public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) {
        }
    };
    try {
        BlockManagerFaultInjector.instance = injector;
        cluster.set(new MiniDFSCluster.Builder(conf).numDataNodes(2).build());
        cluster.get().waitActive();
        Assert.assertNotNull(cluster.get().stopDataNode(datanodeToStop.get()));
        gotFbrSem.acquire();
        Assert.assertNull(failure.get());
    } finally {
        if (cluster.get() != null) {
            cluster.get().shutdown();
        }
    }
}

Example 20 with DatanodeID

use of org.apache.hadoop.hdfs.protocol.DatanodeID in project hadoop by apache.

the class TestDatanodeManager method testNumVersionsCorrectAfterReregister.

/**
   * This test checks that if a node is re-registered with a new software
   * version after the heartbeat expiry interval but before the HeartbeatManager
   * has a chance to detect this and remove it, the node's version will still
   * be correctly decremented.
   */
@Test
public void testNumVersionsCorrectAfterReregister() throws IOException, InterruptedException {
    //Create the DatanodeManager which will be tested
    FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
    Mockito.when(fsn.hasWriteLock()).thenReturn(true);
    Configuration conf = new Configuration();
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 0);
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 10);
    DatanodeManager dm = mockDatanodeManager(fsn, conf);
    String storageID = "someStorageID1";
    String ip = "someIP" + storageID;
    // Register then reregister the same node but with a different version
    for (int i = 0; i <= 1; i++) {
        dm.registerDatanode(new DatanodeRegistration(new DatanodeID(ip, "", storageID, 9000, 0, 0, 0), null, null, "version" + i));
        if (i == 0) {
            Thread.sleep(25);
        }
    }
    //Verify DatanodeManager has the correct count
    Map<String, Integer> mapToCheck = dm.getDatanodesSoftwareVersions();
    assertNull("should be no more version0 nodes", mapToCheck.get("version0"));
    assertEquals("should be one version1 node", mapToCheck.get("version1").intValue(), 1);
}