Examples with LocatedFileStatus org.apache.hadoop.fs.LocatedFileStatus used on opensource projects

Example 16 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestDecommissioningStatus method testDecommissionStatusAfterDNRestart.

/**
   * Verify a DN remains in DECOMMISSION_INPROGRESS state if it is marked
   * as dead before decommission has completed. That will allow DN to resume
   * the replication process after it rejoins the cluster.
   */
@Test(timeout = 120000)
public void testDecommissionStatusAfterDNRestart() throws Exception {
    DistributedFileSystem fileSys = (DistributedFileSystem) cluster.getFileSystem();
    // Create a file with one block. That block has one replica.
    Path f = new Path("decommission.dat");
    DFSTestUtil.createFile(fileSys, f, fileSize, fileSize, fileSize, (short) 1, seed);
    // Find the DN that owns the only replica.
    RemoteIterator<LocatedFileStatus> fileList = fileSys.listLocatedStatus(f);
    BlockLocation[] blockLocations = fileList.next().getBlockLocations();
    String dnName = blockLocations[0].getNames()[0];
    // Decommission the DN.
    FSNamesystem fsn = cluster.getNamesystem();
    final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
    decommissionNode(dnName);
    dm.refreshNodes(conf);
    // Stop the DN when decommission is in progress.
    // Given DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY is to 1 and the size of
    // the block, it will take much longer time that test timeout value for
    // the decommission to complete. So when stopDataNode is called,
    // decommission should be in progress.
    DataNodeProperties dataNodeProperties = cluster.stopDataNode(dnName);
    final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
    while (true) {
        dm.fetchDatanodes(null, dead, false);
        if (dead.size() == 1) {
            break;
        }
        Thread.sleep(1000);
    }
    // Force removal of the dead node's blocks.
    BlockManagerTestUtil.checkHeartbeat(fsn.getBlockManager());
    // Force DatanodeManager to check decommission state.
    BlockManagerTestUtil.recheckDecommissionState(dm);
    // Verify that the DN remains in DECOMMISSION_INPROGRESS state.
    assertTrue("the node should be DECOMMISSION_IN_PROGRESSS", dead.get(0).isDecommissionInProgress());
    // Check DatanodeManager#getDecommissionNodes, make sure it returns
    // the node as decommissioning, even if it's dead
    List<DatanodeDescriptor> decomlist = dm.getDecommissioningNodes();
    assertTrue("The node should be be decommissioning", decomlist.size() == 1);
    // Delete the under-replicated file, which should let the 
    // DECOMMISSION_IN_PROGRESS node become DECOMMISSIONED
    AdminStatesBaseTest.cleanupFile(fileSys, f);
    BlockManagerTestUtil.recheckDecommissionState(dm);
    assertTrue("the node should be decommissioned", dead.get(0).isDecommissioned());
    // Add the node back
    cluster.restartDataNode(dataNodeProperties, true);
    cluster.waitActive();
    // Call refreshNodes on FSNamesystem with empty exclude file.
    // This will remove the datanodes from decommissioning list and
    // make them available again.
    hostsFileWriter.initExcludeHost("");
    dm.refreshNodes(conf);
}

Example 17 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project hadoop by apache.

the class TestINodeFile method checkEquals.

private static void checkEquals(RemoteIterator<LocatedFileStatus> i1, RemoteIterator<LocatedFileStatus> i2) throws IOException {
    while (i1.hasNext()) {
        assertTrue(i2.hasNext());
        // Compare all the fields but the path name, which is relative
        // to the original path from listFiles.
        LocatedFileStatus l1 = i1.next();
        LocatedFileStatus l2 = i2.next();
        assertEquals(l1.getAccessTime(), l2.getAccessTime());
        assertEquals(l1.getBlockSize(), l2.getBlockSize());
        assertEquals(l1.getGroup(), l2.getGroup());
        assertEquals(l1.getLen(), l2.getLen());
        assertEquals(l1.getModificationTime(), l2.getModificationTime());
        assertEquals(l1.getOwner(), l2.getOwner());
        assertEquals(l1.getPermission(), l2.getPermission());
        assertEquals(l1.getReplication(), l2.getReplication());
    }
    assertFalse(i2.hasNext());
}

Example 18 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project jena by apache.

the class AbstractNodeTupleOutputFormatTests method findOutputFile.

protected File findOutputFile(File dir, JobContext context) throws FileNotFoundException, IOException {
    Path outputPath = FileOutputFormat.getOutputPath(context);
    RemoteIterator<LocatedFileStatus> files = outputPath.getFileSystem(context.getConfiguration()).listFiles(outputPath, true);
    while (files.hasNext()) {
        LocatedFileStatus status = files.next();
        if (status.isFile() && !status.getPath().getName().startsWith("_")) {
            return new File(status.getPath().toUri());
        }
    }
    return null;
}

Example 19 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project presto by prestodb.

the class PrestoS3FileSystem method statusFromPrefixes.

private Iterator<LocatedFileStatus> statusFromPrefixes(List<String> prefixes) {
    List<LocatedFileStatus> list = new ArrayList<>();
    for (String prefix : prefixes) {
        Path path = qualifiedPath(new Path(PATH_SEPARATOR + prefix));
        FileStatus status = new FileStatus(0, true, 1, 0, 0, path);
        list.add(createLocatedFileStatus(status));
    }
    return list.iterator();
}

Example 20 with LocatedFileStatus

use of org.apache.hadoop.fs.LocatedFileStatus in project presto by prestodb.

the class BackgroundHiveSplitLoader method listAndSortBucketFiles.

private static List<LocatedFileStatus> listAndSortBucketFiles(HiveFileIterator hiveFileIterator, int bucketCount) {
    ArrayList<LocatedFileStatus> list = new ArrayList<>(bucketCount);
    while (hiveFileIterator.hasNext()) {
        LocatedFileStatus next = hiveFileIterator.next();
        if (isDirectory(next)) {
            // Fail here to be on the safe side. This seems to be the same as what Hive does
            throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("%s Found sub-directory in bucket directory for partition: %s", CORRUPT_BUCKETING, hiveFileIterator.getPartitionName()));
        }
        list.add(next);
    }
    if (list.size() != bucketCount) {
        throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format("%s The number of files in the directory (%s) does not match the declared bucket count (%s) for partition: %s", CORRUPT_BUCKETING, list.size(), bucketCount, hiveFileIterator.getPartitionName()));
    }
    // Sort FileStatus objects (instead of, e.g., fileStatus.getPath().toString). This matches org.apache.hadoop.hive.ql.metadata.Table.getSortedPaths
    list.sort(null);
    return list;
}