Examples with DirectoryListing org.apache.hadoop.hdfs.protocol.DirectoryListing used on opensource projects

Example 6 with DirectoryListing

use of org.apache.hadoop.hdfs.protocol.DirectoryListing in project hadoop by apache.

the class TestDFSUpgradeFromImage method recoverAllLeases.

static void recoverAllLeases(DFSClient dfs, Path path) throws IOException {
    String pathStr = path.toString();
    HdfsFileStatus status = dfs.getFileInfo(pathStr);
    if (!status.isDir()) {
        for (int retries = 10; retries > 0; retries--) {
            if (dfs.recoverLease(pathStr)) {
                return;
            } else {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException ignored) {
                }
            }
        }
        throw new IOException("Failed to recover lease of " + path);
    }
    byte[] prev = HdfsFileStatus.EMPTY_NAME;
    DirectoryListing dirList;
    do {
        dirList = dfs.listPaths(pathStr, prev);
        HdfsFileStatus[] files = dirList.getPartialListing();
        for (HdfsFileStatus f : files) {
            recoverAllLeases(dfs, f.getFullPath(path));
        }
        prev = dirList.getLastName();
    } while (dirList.hasMore());
}

Example 7 with DirectoryListing

use of org.apache.hadoop.hdfs.protocol.DirectoryListing in project hive by apache.

the class Hadoop23Shims method listLocatedHdfsStatus.

@Override
public List<HdfsFileStatusWithId> listLocatedHdfsStatus(FileSystem fs, Path p, PathFilter filter) throws IOException {
    DistributedFileSystem dfs = ensureDfs(fs);
    DFSClient dfsc = dfs.getClient();
    final String src = p.toUri().getPath();
    DirectoryListing current = dfsc.listPaths(src, org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true);
    if (current == null) {
        // the directory does not exist
        throw new FileNotFoundException("File " + p + " does not exist.");
    }
    final URI fsUri = fs.getUri();
    List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>(current.getPartialListing().length);
    while (current != null) {
        org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing();
        for (int i = 0; i < hfss.length; ++i) {
            HdfsLocatedFileStatus next = (HdfsLocatedFileStatus) (hfss[i]);
            if (filter != null) {
                Path filterPath = next.getFullPath(p).makeQualified(fsUri, null);
                if (!filter.accept(filterPath))
                    continue;
            }
            LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p);
            result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId()));
        }
        current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null;
    }
    return result;
}

Example 8 with DirectoryListing

use of org.apache.hadoop.hdfs.protocol.DirectoryListing in project hbase by apache.

the class TestBlockReorder method testHBaseCluster.

/**
   * Test that the hook works within HBase, including when there are multiple blocks.
   */
@Test()
public void testHBaseCluster() throws Exception {
    byte[] sb = "sb".getBytes();
    htu.startMiniZKCluster();
    MiniHBaseCluster hbm = htu.startMiniHBaseCluster(1, 1);
    hbm.waitForActiveAndReadyMaster();
    HRegionServer targetRs = hbm.getMaster();
    // We want to have a datanode with the same name as the region server, so
    //  we're going to get the regionservername, and start a new datanode with this name.
    String host4 = targetRs.getServerName().getHostname();
    LOG.info("Starting a new datanode with the name=" + host4);
    cluster.startDataNodes(conf, 1, true, null, new String[] { "/r4" }, new String[] { host4 }, null);
    cluster.waitClusterUp();
    final int repCount = 3;
    // We use the regionserver file system & conf as we expect it to have the hook.
    conf = targetRs.getConfiguration();
    HFileSystem rfs = (HFileSystem) targetRs.getFileSystem();
    Table h = htu.createTable(TableName.valueOf(name.getMethodName()), sb);
    // Now, we have 4 datanodes and a replication count of 3. So we don't know if the datanode
    // with the same node will be used. We can't really stop an existing datanode, this would
    // make us fall in nasty hdfs bugs/issues. So we're going to try multiple times.
    // Now we need to find the log file, its locations, and look at it
    String rootDir = new Path(FSUtils.getRootDir(conf) + "/" + HConstants.HREGION_LOGDIR_NAME + "/" + targetRs.getServerName().toString()).toUri().getPath();
    DistributedFileSystem mdfs = (DistributedFileSystem) hbm.getMaster().getMasterFileSystem().getFileSystem();
    int nbTest = 0;
    while (nbTest < 10) {
        final List<Region> regions = targetRs.getOnlineRegions(h.getName());
        final CountDownLatch latch = new CountDownLatch(regions.size());
        // listen for successful log rolls
        final WALActionsListener listener = new WALActionsListener.Base() {

            @Override
            public void postLogRoll(final Path oldPath, final Path newPath) throws IOException {
                latch.countDown();
            }
        };
        for (Region region : regions) {
            ((HRegion) region).getWAL().registerWALActionsListener(listener);
        }
        htu.getAdmin().rollWALWriter(targetRs.getServerName());
        // wait
        try {
            latch.await();
        } catch (InterruptedException exception) {
            LOG.warn("Interrupted while waiting for the wal of '" + targetRs + "' to roll. If later " + "tests fail, it's probably because we should still be waiting.");
            Thread.currentThread().interrupt();
        }
        for (Region region : regions) {
            ((HRegion) region).getWAL().unregisterWALActionsListener(listener);
        }
        // We need a sleep as the namenode is informed asynchronously
        Thread.sleep(100);
        // insert one put to ensure a minimal size
        Put p = new Put(sb);
        p.addColumn(sb, sb, sb);
        h.put(p);
        DirectoryListing dl = dfs.getClient().listPaths(rootDir, HdfsFileStatus.EMPTY_NAME);
        HdfsFileStatus[] hfs = dl.getPartialListing();
        // As we wrote a put, we should have at least one log file.
        Assert.assertTrue(hfs.length >= 1);
        for (HdfsFileStatus hf : hfs) {
            // Because this is a live cluster, log files might get archived while we're processing
            try {
                LOG.info("Log file found: " + hf.getLocalName() + " in " + rootDir);
                String logFile = rootDir + "/" + hf.getLocalName();
                FileStatus fsLog = rfs.getFileStatus(new Path(logFile));
                LOG.info("Checking log file: " + logFile);
                // Now checking that the hook is up and running
                // We can't call directly getBlockLocations, it's not available in HFileSystem
                // We're trying multiple times to be sure, as the order is random
                BlockLocation[] bls = rfs.getFileBlockLocations(fsLog, 0, 1);
                if (bls.length > 0) {
                    BlockLocation bl = bls[0];
                    LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " ");
                    for (int i = 0; i < bl.getHosts().length - 1; i++) {
                        LOG.info(bl.getHosts()[i] + "    " + logFile);
                        Assert.assertNotSame(bl.getHosts()[i], host4);
                    }
                    String last = bl.getHosts()[bl.getHosts().length - 1];
                    LOG.info(last + "    " + logFile);
                    if (host4.equals(last)) {
                        nbTest++;
                        LOG.info(logFile + " is on the new datanode and is ok");
                        if (bl.getHosts().length == 3) {
                            // We can test this case from the file system as well
                            // Checking the underlying file system. Multiple times as the order is random
                            testFromDFS(dfs, logFile, repCount, host4);
                            // now from the master
                            testFromDFS(mdfs, logFile, repCount, host4);
                        }
                    }
                }
            } catch (FileNotFoundException exception) {
                LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
            } catch (RemoteException exception) {
                if (exception.unwrapRemoteException() instanceof FileNotFoundException) {
                    LOG.debug("Failed to find log file '" + hf.getLocalName() + "'; it probably was " + "archived out from under us so we'll ignore and retry. If this test hangs " + "indefinitely you should treat this failure as a symptom.", exception);
                } else {
                    throw exception;
                }
            }
        }
    }
}

Example 9 with DirectoryListing

use of org.apache.hadoop.hdfs.protocol.DirectoryListing in project hadoop by apache.

the class DistributedFileSystem method listStatusInternal.

private FileStatus[] listStatusInternal(Path p) throws IOException {
    String src = getPathName(p);
    // fetch the first batch of entries in the directory
    DirectoryListing thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME);
    if (thisListing == null) {
        // the directory does not exist
        throw new FileNotFoundException("File " + p + " does not exist.");
    }
    HdfsFileStatus[] partialListing = thisListing.getPartialListing();
    if (!thisListing.hasMore()) {
        // got all entries of the directory
        FileStatus[] stats = new FileStatus[partialListing.length];
        for (int i = 0; i < partialListing.length; i++) {
            stats[i] = partialListing[i].makeQualified(getUri(), p);
        }
        statistics.incrementReadOps(1);
        storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
        return stats;
    }
    // The directory size is too big that it needs to fetch more
    // estimate the total number of entries in the directory
    int totalNumEntries = partialListing.length + thisListing.getRemainingEntries();
    ArrayList<FileStatus> listing = new ArrayList<>(totalNumEntries);
    // add the first batch of entries to the array list
    for (HdfsFileStatus fileStatus : partialListing) {
        listing.add(fileStatus.makeQualified(getUri(), p));
    }
    statistics.incrementLargeReadOps(1);
    storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
    // now fetch more entries
    do {
        thisListing = dfs.listPaths(src, thisListing.getLastName());
        if (thisListing == null) {
            // the directory is deleted
            throw new FileNotFoundException("File " + p + " does not exist.");
        }
        partialListing = thisListing.getPartialListing();
        for (HdfsFileStatus fileStatus : partialListing) {
            listing.add(fileStatus.makeQualified(getUri(), p));
        }
        statistics.incrementLargeReadOps(1);
        storageStatistics.incrementOpCounter(OpType.LIST_STATUS);
    } while (thisListing.hasMore());
    return listing.toArray(new FileStatus[listing.size()]);
}

Example 10 with DirectoryListing

use of org.apache.hadoop.hdfs.protocol.DirectoryListing in project hadoop by apache.

the class WebHdfsFileSystem method listStatusBatch.

@Override
public DirectoryEntries listStatusBatch(Path f, byte[] token) throws FileNotFoundException, IOException {
    byte[] prevKey = EMPTY_ARRAY;
    if (token != null) {
        prevKey = token;
    }
    DirectoryListing listing = new FsPathResponseRunner<DirectoryListing>(GetOpParam.Op.LISTSTATUS_BATCH, f, new StartAfterParam(new String(prevKey, Charsets.UTF_8))) {

        @Override
        DirectoryListing decodeResponse(Map<?, ?> json) throws IOException {
            return JsonUtilClient.toDirectoryListing(json);
        }
    }.run();
    // Qualify the returned FileStatus array
    final HdfsFileStatus[] statuses = listing.getPartialListing();
    FileStatus[] qualified = new FileStatus[statuses.length];
    for (int i = 0; i < statuses.length; i++) {
        qualified[i] = makeQualified(statuses[i], f);
    }
    return new DirectoryEntries(qualified, listing.getLastName(), listing.hasMore());
}