Examples with DatanodeInfo org.apache.hadoop.hdfs.protocol.DatanodeInfo used on opensource projects

Example 41 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestMaintenanceState method testMultipleNodesMaintenance.

/**
   * Verify if multiple DataNodes can transition to maintenance state
   * at the same time.
   */
@Test(timeout = 360000)
public void testMultipleNodesMaintenance() throws Exception {
    startCluster(1, 5);
    final Path file = new Path("/testMultipleNodesMaintenance.dat");
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    int repl = 3;
    writeFile(fileSys, file, repl, 1);
    final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys, file);
    // Request maintenance for DataNodes 1 and 2 which has the file blocks.
    List<DatanodeInfo> maintenanceDN = takeNodeOutofService(0, Lists.newArrayList(nodes[0].getDatanodeUuid(), nodes[1].getDatanodeUuid()), Long.MAX_VALUE, null, null, AdminStates.IN_MAINTENANCE);
    // Verify file replication matches maintenance state min replication
    assertNull(checkWithRetry(ns, fileSys, file, 1, null, nodes[0]));
    // Put the maintenance nodes back in service
    for (DatanodeInfo datanodeInfo : maintenanceDN) {
        putNodeInService(0, datanodeInfo);
    }
    // Verify file replication catching up to the old state
    assertNull(checkWithRetry(ns, fileSys, file, repl, null));
    cleanupFile(fileSys, file);
}

Example 42 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestMaintenanceState method testDecommissionDifferentNodeAfterMaintenance.

private void testDecommissionDifferentNodeAfterMaintenance(int repl) throws Exception {
    setup();
    startCluster(1, 5);
    final Path file = new Path("/testDecommissionDifferentNodeAfterMaintenance.dat");
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, repl, 1);
    final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys, file);
    String maintenanceDNUuid = nodes[0].getDatanodeUuid();
    String decommissionDNUuid = nodes[1].getDatanodeUuid();
    DatanodeInfo maintenanceDN = takeNodeOutofService(0, maintenanceDNUuid, Long.MAX_VALUE, null, null, AdminStates.IN_MAINTENANCE);
    Map<DatanodeInfo, Long> maintenanceNodes = new HashMap<>();
    maintenanceNodes.put(nodes[0], Long.MAX_VALUE);
    takeNodeOutofService(0, decommissionDNUuid, 0, null, maintenanceNodes, AdminStates.DECOMMISSIONED);
    // Out of the replicas returned, one is the decommissioned node.
    assertNull(checkWithRetry(ns, fileSys, file, repl, maintenanceDN));
    putNodeInService(0, maintenanceDN);
    assertNull(checkWithRetry(ns, fileSys, file, repl + 1, null));
    cleanupFile(fileSys, file);
    teardown();
}

Example 43 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestBalancer method waitForBalancer.

/**
   * Wait until balanced: each datanode gives utilization within
   * BALANCE_ALLOWED_VARIANCE of average
   * @throws IOException
   * @throws TimeoutException
   */
static void waitForBalancer(long totalUsedSpace, long totalCapacity, ClientProtocol client, MiniDFSCluster cluster, BalancerParameters p, int expectedExcludedNodes) throws IOException, TimeoutException {
    long timeout = TIMEOUT;
    long failtime = (timeout <= 0L) ? Long.MAX_VALUE : Time.monotonicNow() + timeout;
    if (!p.getIncludedNodes().isEmpty()) {
        totalCapacity = p.getIncludedNodes().size() * CAPACITY;
    }
    if (!p.getExcludedNodes().isEmpty()) {
        totalCapacity -= p.getExcludedNodes().size() * CAPACITY;
    }
    final double avgUtilization = ((double) totalUsedSpace) / totalCapacity;
    boolean balanced;
    do {
        DatanodeInfo[] datanodeReport = client.getDatanodeReport(DatanodeReportType.ALL);
        assertEquals(datanodeReport.length, cluster.getDataNodes().size());
        balanced = true;
        int actualExcludedNodeCount = 0;
        for (DatanodeInfo datanode : datanodeReport) {
            double nodeUtilization = ((double) datanode.getDfsUsed()) / datanode.getCapacity();
            if (Dispatcher.Util.isExcluded(p.getExcludedNodes(), datanode)) {
                assertTrue(nodeUtilization == 0);
                actualExcludedNodeCount++;
                continue;
            }
            if (!Dispatcher.Util.isIncluded(p.getIncludedNodes(), datanode)) {
                assertTrue(nodeUtilization == 0);
                actualExcludedNodeCount++;
                continue;
            }
            if (Math.abs(avgUtilization - nodeUtilization) > BALANCE_ALLOWED_VARIANCE) {
                balanced = false;
                if (Time.monotonicNow() > failtime) {
                    throw new TimeoutException("Rebalancing expected avg utilization to become " + avgUtilization + ", but on datanode " + datanode + " it remains at " + nodeUtilization + " after more than " + TIMEOUT + " msec.");
                }
                try {
                    Thread.sleep(100);
                } catch (InterruptedException ignored) {
                }
                break;
            }
        }
        assertEquals(expectedExcludedNodes, actualExcludedNodeCount);
    } while (!balanced);
}

Example 44 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDatanodeManager method HelperFunction.

/**
   * Helper function that tests the DatanodeManagers SortedBlock function
   * we invoke this function with and without topology scripts
   *
   * @param scriptFileName - Script Name or null
   *
   * @throws URISyntaxException
   * @throws IOException
   */
public void HelperFunction(String scriptFileName) throws URISyntaxException, IOException {
    // create the DatanodeManager which will be tested
    Configuration conf = new Configuration();
    FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
    Mockito.when(fsn.hasWriteLock()).thenReturn(true);
    if (scriptFileName != null && !scriptFileName.isEmpty()) {
        URL shellScript = getClass().getResource(scriptFileName);
        Path resourcePath = Paths.get(shellScript.toURI());
        FileUtil.setExecutable(resourcePath.toFile(), true);
        conf.set(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, resourcePath.toString());
    }
    DatanodeManager dm = mockDatanodeManager(fsn, conf);
    // register 5 datanodes, each with different storage ID and type
    DatanodeInfo[] locs = new DatanodeInfo[5];
    String[] storageIDs = new String[5];
    StorageType[] storageTypes = new StorageType[] { StorageType.ARCHIVE, StorageType.DEFAULT, StorageType.DISK, StorageType.RAM_DISK, StorageType.SSD };
    for (int i = 0; i < 5; i++) {
        // register new datanode
        String uuid = "UUID-" + i;
        String ip = "IP-" + i;
        DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class);
        Mockito.when(dr.getDatanodeUuid()).thenReturn(uuid);
        Mockito.when(dr.getIpAddr()).thenReturn(ip);
        Mockito.when(dr.getXferAddr()).thenReturn(ip + ":9000");
        Mockito.when(dr.getXferPort()).thenReturn(9000);
        Mockito.when(dr.getSoftwareVersion()).thenReturn("version1");
        dm.registerDatanode(dr);
        // get location and storage information
        locs[i] = dm.getDatanode(uuid);
        storageIDs[i] = "storageID-" + i;
    }
    // set first 2 locations as decomissioned
    locs[0].setDecommissioned();
    locs[1].setDecommissioned();
    // create LocatedBlock with above locations
    ExtendedBlock b = new ExtendedBlock("somePoolID", 1234);
    LocatedBlock block = new LocatedBlock(b, locs, storageIDs, storageTypes);
    List<LocatedBlock> blocks = new ArrayList<>();
    blocks.add(block);
    final String targetIp = locs[4].getIpAddr();
    // sort block locations
    dm.sortLocatedBlocks(targetIp, blocks);
    // check that storage IDs/types are aligned with datanode locs
    DatanodeInfo[] sortedLocs = block.getLocations();
    storageIDs = block.getStorageIDs();
    storageTypes = block.getStorageTypes();
    assertThat(sortedLocs.length, is(5));
    assertThat(storageIDs.length, is(5));
    assertThat(storageTypes.length, is(5));
    for (int i = 0; i < sortedLocs.length; i++) {
        assertThat(((DatanodeInfoWithStorage) sortedLocs[i]).getStorageID(), is(storageIDs[i]));
        assertThat(((DatanodeInfoWithStorage) sortedLocs[i]).getStorageType(), is(storageTypes[i]));
    }
    // Ensure the local node is first.
    assertThat(sortedLocs[0].getIpAddr(), is(targetIp));
    // Ensure the two decommissioned DNs were moved to the end.
    assertThat(sortedLocs[sortedLocs.length - 1].getAdminState(), is(DatanodeInfo.AdminStates.DECOMMISSIONED));
    assertThat(sortedLocs[sortedLocs.length - 2].getAdminState(), is(DatanodeInfo.AdminStates.DECOMMISSIONED));
}

Example 45 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDecommissioningStatus method checkDFSAdminDecommissionStatus.

private void checkDFSAdminDecommissionStatus(List<DatanodeDescriptor> expectedDecomm, DistributedFileSystem dfs, DFSAdmin admin) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream(baos);
    PrintStream oldOut = System.out;
    System.setOut(ps);
    try {
        // Parse DFSAdmin just to check the count
        admin.report(new String[] { "-decommissioning" }, 0);
        String[] lines = baos.toString().split("\n");
        Integer num = null;
        int count = 0;
        for (String line : lines) {
            if (line.startsWith("Decommissioning datanodes")) {
                // Pull out the "(num)" and parse it into an int
                String temp = line.split(" ")[2];
                num = Integer.parseInt((String) temp.subSequence(1, temp.length() - 2));
            }
            if (line.contains("Decommission in progress")) {
                count++;
            }
        }
        assertTrue("No decommissioning output", num != null);
        assertEquals("Unexpected number of decomming DNs", expectedDecomm.size(), num.intValue());
        assertEquals("Unexpected number of decomming DNs", expectedDecomm.size(), count);
        // Check Java API for correct contents
        List<DatanodeInfo> decomming = new ArrayList<DatanodeInfo>(Arrays.asList(dfs.getDataNodeStats(DatanodeReportType.DECOMMISSIONING)));
        assertEquals("Unexpected number of decomming DNs", expectedDecomm.size(), decomming.size());
        for (DatanodeID id : expectedDecomm) {
            assertTrue("Did not find expected decomming DN " + id, decomming.contains(id));
        }
    } finally {
        System.setOut(oldOut);
    }
}