Examples with FSNamesystem org.apache.hadoop.hdfs.server.namenode.FSNamesystem used on opensource projects

Example 41 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testTakeDeadNodeOutOfMaintenance.

/**
   * Verify the following scenario.
   * a. Put a live node to maintenance => 1 maintenance, 2 live.
   * b. The maintenance node becomes dead => block map still has 1 maintenance,
   *    2 live.
   * c. Take the node out of maintenance => NN should schedule the replication
   *    and end up with 3 live.
   */
@Test(timeout = 360000)
public void testTakeDeadNodeOutOfMaintenance() throws Exception {
    LOG.info("Starting testTakeDeadNodeOutOfMaintenance");
    final int numNamenodes = 1;
    final int numDatanodes = 4;
    startCluster(numNamenodes, numDatanodes);
    final Path file = new Path("/testTakeDeadNodeOutOfMaintenance.dat");
    final int replicas = 3;
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas, 1);
    final DatanodeInfo nodeOutofService = takeNodeOutofService(0, getFirstBlockFirstReplicaUuid(fileSys, file), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    assertNull(checkWithRetry(ns, fileSys, file, replicas - 1, nodeOutofService));
    final DFSClient client = getDfsClient(0);
    assertEquals("All datanodes must be alive", numDatanodes, client.datanodeReport(DatanodeReportType.LIVE).length);
    getCluster().stopDataNode(nodeOutofService.getXferAddr());
    DFSTestUtil.waitForDatanodeState(getCluster(), nodeOutofService.getDatanodeUuid(), false, 20000);
    assertEquals("maintenance node shouldn't be alive", numDatanodes - 1, client.datanodeReport(DatanodeReportType.LIVE).length);
    // Dead maintenance node's blocks should remain in block map.
    assertNull(checkWithRetry(ns, fileSys, file, replicas - 1, nodeOutofService));
    // When dead maintenance mode is transitioned to out of maintenance mode,
    // its blocks should be removed from block map.
    // This will then trigger replication to restore the live replicas back
    // to replication factor.
    putNodeInService(0, nodeOutofService.getDatanodeUuid());
    assertNull(checkWithRetry(ns, fileSys, file, replicas, nodeOutofService, null));
    cleanupFile(fileSys, file);
}

Example 42 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testPutDeadNodeToMaintenanceWithExpiration.

/**
   * When a dead node is put to maintenance, it transitions directly to
   * AdminStates.IN_MAINTENANCE. Then AdminStates.IN_MAINTENANCE expires and
   * transitions to AdminStates.NORMAL.
   */
@Test(timeout = 360000)
public void testPutDeadNodeToMaintenanceWithExpiration() throws Exception {
    LOG.info("Starting testPutDeadNodeToMaintenanceWithExpiration");
    final Path file = new Path("/testPutDeadNodeToMaintenanceWithExpiration.dat");
    startCluster(1, 1);
    FileSystem fileSys = getCluster().getFileSystem(0);
    FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, 1, 1);
    MiniDFSCluster.DataNodeProperties dnProp = getCluster().stopDataNode(0);
    DFSTestUtil.waitForDatanodeState(getCluster(), dnProp.datanode.getDatanodeUuid(), false, 20000);
    int deadInMaintenance = ns.getNumInMaintenanceDeadDataNodes();
    int liveInMaintenance = ns.getNumInMaintenanceLiveDataNodes();
    DatanodeInfo nodeOutofService = takeNodeOutofService(0, dnProp.datanode.getDatanodeUuid(), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    // Adjust the expiration.
    takeNodeOutofService(0, nodeOutofService.getDatanodeUuid(), Time.now() + EXPIRATION_IN_MS, null, AdminStates.NORMAL);
    // no change
    assertEquals(deadInMaintenance, ns.getNumInMaintenanceDeadDataNodes());
    assertEquals(liveInMaintenance, ns.getNumInMaintenanceLiveDataNodes());
    cleanupFile(fileSys, file);
}

Example 43 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testChangeReplicationFactor.

/**
   * After the change of replication factor, # of live replicas <=
   * the new replication factor.
   */
private void testChangeReplicationFactor(int oldFactor, int newFactor, int expectedLiveReplicas) throws IOException {
    setup();
    LOG.info("Starting testChangeReplicationFactor {} {} {}", oldFactor, newFactor, expectedLiveReplicas);
    startCluster(1, 5);
    final Path file = new Path("/testChangeReplicationFactor.dat");
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, oldFactor, 1);
    final DatanodeInfo nodeOutofService = takeNodeOutofService(0, getFirstBlockFirstReplicaUuid(fileSys, file), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    // Verify that the nodeOutofService remains in blocksMap and
    // # of live replicas For read operation is expected.
    assertNull(checkWithRetry(ns, fileSys, file, oldFactor - 1, nodeOutofService));
    final DFSClient client = getDfsClient(0);
    client.setReplication(file.toString(), (short) newFactor);
    // Verify that the nodeOutofService remains in blocksMap and
    // # of live replicas for read operation.
    assertNull(checkWithRetry(ns, fileSys, file, expectedLiveReplicas, nodeOutofService));
    putNodeInService(0, nodeOutofService.getDatanodeUuid());
    assertNull(checkWithRetry(ns, fileSys, file, newFactor, null));
    cleanupFile(fileSys, file);
    teardown();
}

Example 44 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testNodeDeadWhenInEnteringMaintenance.

/**
   * When a node is put to maintenance, it first transitions to
   * AdminStates.ENTERING_MAINTENANCE. It makes sure all blocks have minimal
   * replication before it can be transitioned to AdminStates.IN_MAINTENANCE.
   * If node becomes dead when it is in AdminStates.ENTERING_MAINTENANCE, it
   * should stay in AdminStates.ENTERING_MAINTENANCE state.
   */
@Test(timeout = 360000)
public void testNodeDeadWhenInEnteringMaintenance() throws Exception {
    LOG.info("Starting testNodeDeadWhenInEnteringMaintenance");
    final int numNamenodes = 1;
    final int numDatanodes = 1;
    final int replicas = 1;
    final Path file = new Path("/testNodeDeadWhenInEnteringMaintenance.dat");
    startCluster(numNamenodes, numDatanodes);
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas, 1);
    DatanodeInfo nodeOutofService = takeNodeOutofService(0, getFirstBlockFirstReplicaUuid(fileSys, file), Long.MAX_VALUE, null, AdminStates.ENTERING_MAINTENANCE);
    assertEquals(1, ns.getNumEnteringMaintenanceDataNodes());
    MiniDFSCluster.DataNodeProperties dnProp = getCluster().stopDataNode(nodeOutofService.getXferAddr());
    DFSTestUtil.waitForDatanodeState(getCluster(), nodeOutofService.getDatanodeUuid(), false, 20000);
    DFSClient client = getDfsClient(0);
    assertEquals("maintenance node shouldn't be live", numDatanodes - 1, client.datanodeReport(DatanodeReportType.LIVE).length);
    assertEquals(1, ns.getNumEnteringMaintenanceDataNodes());
    getCluster().restartDataNode(dnProp, true);
    getCluster().waitActive();
    waitNodeState(nodeOutofService, AdminStates.ENTERING_MAINTENANCE);
    assertEquals(1, ns.getNumEnteringMaintenanceDataNodes());
    assertEquals("maintenance node should be live", numDatanodes, client.datanodeReport(DatanodeReportType.LIVE).length);
    cleanupFile(fileSys, file);
}

Example 45 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testPutDeadNodeToMaintenance.

/**
   * When a dead node is put to maintenance, it transitions directly to
   * AdminStates.IN_MAINTENANCE.
   */
@Test(timeout = 360000)
public void testPutDeadNodeToMaintenance() throws Exception {
    LOG.info("Starting testPutDeadNodeToMaintenance");
    final int replicas = 1;
    final Path file = new Path("/testPutDeadNodeToMaintenance.dat");
    startCluster(1, 1);
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas, 1);
    final MiniDFSCluster.DataNodeProperties dnProp = getCluster().stopDataNode(0);
    DFSTestUtil.waitForDatanodeState(getCluster(), dnProp.datanode.getDatanodeUuid(), false, 20000);
    int deadInMaintenance = ns.getNumInMaintenanceDeadDataNodes();
    int liveInMaintenance = ns.getNumInMaintenanceLiveDataNodes();
    takeNodeOutofService(0, dnProp.datanode.getDatanodeUuid(), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    assertEquals(deadInMaintenance + 1, ns.getNumInMaintenanceDeadDataNodes());
    assertEquals(liveInMaintenance, ns.getNumInMaintenanceLiveDataNodes());
    cleanupFile(fileSys, file);
}