Examples with FSNamesystem org.apache.hadoop.hdfs.server.namenode.FSNamesystem used on opensource projects

Example 36 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class NamenodeWebHdfsMethods method chooseDatanode.

@VisibleForTesting
static DatanodeInfo chooseDatanode(final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, final long blocksize, final String excludeDatanodes, final String remoteAddr) throws IOException {
    FSNamesystem fsn = namenode.getNamesystem();
    if (fsn == null) {
        throw new IOException("Namesystem has not been intialized yet.");
    }
    final BlockManager bm = fsn.getBlockManager();
    HashSet<Node> excludes = new HashSet<Node>();
    if (excludeDatanodes != null) {
        for (String host : StringUtils.getTrimmedStringCollection(excludeDatanodes)) {
            int idx = host.indexOf(":");
            if (idx != -1) {
                excludes.add(bm.getDatanodeManager().getDatanodeByXferAddr(host.substring(0, idx), Integer.parseInt(host.substring(idx + 1))));
            } else {
                excludes.add(bm.getDatanodeManager().getDatanodeByHost(host));
            }
        }
    }
    if (op == PutOpParam.Op.CREATE) {
        //choose a datanode near to client 
        final DatanodeDescriptor clientNode = bm.getDatanodeManager().getDatanodeByHost(remoteAddr);
        if (clientNode != null) {
            final DatanodeStorageInfo[] storages = bm.chooseTarget4WebHDFS(path, clientNode, excludes, blocksize);
            if (storages.length > 0) {
                return storages[0].getDatanodeDescriptor();
            }
        }
    } else if (op == GetOpParam.Op.OPEN || op == GetOpParam.Op.GETFILECHECKSUM || op == PostOpParam.Op.APPEND) {
        //choose a datanode containing a replica 
        final NamenodeProtocols np = getRPCServer(namenode);
        final HdfsFileStatus status = np.getFileInfo(path);
        if (status == null) {
            throw new FileNotFoundException("File " + path + " not found.");
        }
        final long len = status.getLen();
        if (op == GetOpParam.Op.OPEN) {
            if (openOffset < 0L || (openOffset >= len && len > 0)) {
                throw new IOException("Offset=" + openOffset + " out of the range [0, " + len + "); " + op + ", path=" + path);
            }
        }
        if (len > 0) {
            final long offset = op == GetOpParam.Op.OPEN ? openOffset : len - 1;
            final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
            final int count = locations.locatedBlockCount();
            if (count > 0) {
                return bestNode(locations.get(0).getLocations(), excludes);
            }
        }
    }
    return (DatanodeDescriptor) bm.getDatanodeManager().getNetworkTopology().chooseRandom(NodeBase.ROOT, excludes);
}

Example 37 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testTransitionFromDecommissioning.

/**
   * Transition from decommissioning state to maintenance state.
   */
@Test(timeout = 360000)
public void testTransitionFromDecommissioning() throws IOException {
    LOG.info("Starting testTransitionFromDecommissioning");
    startCluster(1, 3);
    final Path file = new Path("/testTransitionFromDecommissioning.dat");
    final int replicas = 3;
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas);
    final DatanodeInfo nodeOutofService = takeNodeOutofService(0, null, 0, null, AdminStates.DECOMMISSION_INPROGRESS);
    takeNodeOutofService(0, nodeOutofService.getDatanodeUuid(), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    assertNull(checkWithRetry(ns, fileSys, file, replicas - 1, nodeOutofService));
    cleanupFile(fileSys, file);
}

Example 38 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testInvalidation.

/**
   * Machine under maintenance state won't be chosen for invalidation.
   */
@Test(timeout = 360000)
public void testInvalidation() throws IOException {
    LOG.info("Starting testInvalidation");
    int numNamenodes = 1;
    int numDatanodes = 3;
    startCluster(numNamenodes, numDatanodes);
    Path file = new Path("/testInvalidation.dat");
    int replicas = 3;
    FileSystem fileSys = getCluster().getFileSystem(0);
    FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas);
    DatanodeInfo nodeOutofService = takeNodeOutofService(0, null, Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    DFSClient client = getDfsClient(0);
    client.setReplication(file.toString(), (short) 1);
    // Verify the nodeOutofService remains in blocksMap.
    assertNull(checkWithRetry(ns, fileSys, file, 1, nodeOutofService));
    // Restart NN and verify the nodeOutofService remains in blocksMap.
    getCluster().restartNameNode(0);
    ns = getCluster().getNamesystem(0);
    assertNull(checkWithRetry(ns, fileSys, file, 1, nodeOutofService));
    cleanupFile(fileSys, file);
}

Example 39 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testWriteAfterMaintenance.

/**
   * Machine under maintenance state won't be chosen for new block allocation.
   */
@Test(timeout = 3600000)
public void testWriteAfterMaintenance() throws IOException {
    LOG.info("Starting testWriteAfterMaintenance");
    startCluster(1, 3);
    final Path file = new Path("/testWriteAfterMaintenance.dat");
    int replicas = 3;
    final FileSystem fileSys = getCluster().getFileSystem(0);
    FSNamesystem ns = getCluster().getNamesystem(0);
    final DatanodeInfo nodeOutofService = takeNodeOutofService(0, null, Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    writeFile(fileSys, file, replicas, 2);
    // Verify nodeOutofService wasn't chosen for write operation.
    assertNull(checkWithRetry(ns, fileSys, file, replicas - 1, nodeOutofService, null));
    // Put the node back to service, live replicas should be restored.
    putNodeInService(0, nodeOutofService.getDatanodeUuid());
    assertNull(checkWithRetry(ns, fileSys, file, replicas, null));
    cleanupFile(fileSys, file);
}

Example 40 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testTakeNodeOutOfEnteringMaintenance.

/**
   * Verify a node can transition from AdminStates.ENTERING_MAINTENANCE to
   * AdminStates.NORMAL.
   */
@Test(timeout = 360000)
public void testTakeNodeOutOfEnteringMaintenance() throws Exception {
    LOG.info("Starting testTakeNodeOutOfEnteringMaintenance");
    final int replicas = 1;
    final Path file = new Path("/testTakeNodeOutOfEnteringMaintenance.dat");
    startCluster(1, 1);
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas, 1);
    final DatanodeInfo nodeOutofService = takeNodeOutofService(0, null, Long.MAX_VALUE, null, AdminStates.ENTERING_MAINTENANCE);
    // When node is in ENTERING_MAINTENANCE state, it can still serve read
    // requests
    assertNull(checkWithRetry(ns, fileSys, file, replicas, null, nodeOutofService));
    putNodeInService(0, nodeOutofService.getDatanodeUuid());
    cleanupFile(fileSys, file);
}