Examples with HdfsDataInputStream org.apache.hadoop.hdfs.client.HdfsDataInputStream used on opensource projects

Example 11 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestDecommission method checkFile.

/**
   * Verify that the number of replicas are as expected for each block in
   * the given file.
   * For blocks with a decommissioned node, verify that their replication
   * is 1 more than what is specified.
   * For blocks without decommissioned nodes, verify their replication is
   * equal to what is specified.
   * 
   * @param downnode - if null, there is no decommissioned node for this file.
   * @return - null if no failure found, else an error message string.
   */
private static String checkFile(FileSystem fileSys, Path name, int repl, String downnode, int numDatanodes) throws IOException {
    boolean isNodeDown = (downnode != null);
    // need a raw stream
    assertTrue("Not HDFS:" + fileSys.getUri(), fileSys instanceof DistributedFileSystem);
    HdfsDataInputStream dis = (HdfsDataInputStream) fileSys.open(name);
    Collection<LocatedBlock> dinfo = dis.getAllBlocks();
    for (LocatedBlock blk : dinfo) {
        // for each block
        int hasdown = 0;
        DatanodeInfo[] nodes = blk.getLocations();
        for (int j = 0; j < nodes.length; j++) {
            // for each replica
            if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
                hasdown++;
                //Downnode must actually be decommissioned
                if (!nodes[j].isDecommissioned()) {
                    return "For block " + blk.getBlock() + " replica on " + nodes[j] + " is given as downnode, " + "but is not decommissioned";
                }
                //Decommissioned node (if any) should only be last node in list.
                if (j != nodes.length - 1) {
                    return "For block " + blk.getBlock() + " decommissioned node " + nodes[j] + " was not last node in list: " + (j + 1) + " of " + nodes.length;
                }
                LOG.info("Block " + blk.getBlock() + " replica on " + nodes[j] + " is decommissioned.");
            } else {
                //Non-downnodes must not be decommissioned
                if (nodes[j].isDecommissioned()) {
                    return "For block " + blk.getBlock() + " replica on " + nodes[j] + " is unexpectedly decommissioned";
                }
            }
        }
        LOG.info("Block " + blk.getBlock() + " has " + hasdown + " decommissioned replica.");
        if (Math.min(numDatanodes, repl + hasdown) != nodes.length) {
            return "Wrong number of replicas for block " + blk.getBlock() + ": " + nodes.length + ", expected " + Math.min(numDatanodes, repl + hasdown);
        }
    }
    return null;
}

Example 12 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testZeroCopyReads.

@Test
public void testZeroCopyReads() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    FSDataInputStream fsIn = null;
    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, 7567L);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        byte[] original = new byte[TEST_FILE_LENGTH];
        IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
        fsIn.close();
        fsIn = fs.open(TEST_PATH);
        ByteBuffer result = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
        Assert.assertEquals(BLOCK_SIZE, result.remaining());
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
        Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE), byteBufferToArray(result));
        fsIn.releaseBuffer(result);
    } finally {
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
    }
}

Example 13 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class WebHdfsHandler method onOpen.

private void onOpen(ChannelHandlerContext ctx) throws IOException {
    final String nnId = params.namenodeId();
    final int bufferSize = params.bufferSize();
    final long offset = params.offset();
    final long length = params.length();
    resp = new DefaultHttpResponse(HTTP_1_1, OK);
    HttpHeaders headers = resp.headers();
    // Allow the UI to access the file
    headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
    headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
    headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
    headers.set(CONNECTION, CLOSE);
    final DFSClient dfsclient = newDfsClient(nnId, conf);
    HdfsDataInputStream in = dfsclient.createWrappedInputStream(dfsclient.open(path, bufferSize, true));
    in.seek(offset);
    long contentLength = in.getVisibleLength() - offset;
    if (length >= 0) {
        contentLength = Math.min(contentLength, length);
    }
    final InputStream data;
    if (contentLength >= 0) {
        headers.set(CONTENT_LENGTH, contentLength);
        data = new LimitInputStream(in, contentLength);
    } else {
        data = in;
    }
    ctx.write(resp);
    ctx.writeAndFlush(new ChunkedStream(data) {

        @Override
        public void close() throws Exception {
            super.close();
            dfsclient.close();
        }
    }).addListener(ChannelFutureListener.CLOSE);
}

Example 14 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testShortZeroCopyReads.

@Test
public void testShortZeroCopyReads() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    FSDataInputStream fsIn = null;
    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, 7567L);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        byte[] original = new byte[TEST_FILE_LENGTH];
        IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
        fsIn.close();
        fsIn = fs.open(TEST_PATH);
        // Try to read (2 * ${BLOCK_SIZE}), but only get ${BLOCK_SIZE} because of the block size.
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        ByteBuffer result = dfsIn.read(null, 2 * BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
        Assert.assertEquals(BLOCK_SIZE, result.remaining());
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
        Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE), byteBufferToArray(result));
        dfsIn.releaseBuffer(result);
        // Try to read (1 + ${BLOCK_SIZE}), but only get ${BLOCK_SIZE} because of the block size.
        result = dfsIn.read(null, 1 + BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
        Assert.assertEquals(BLOCK_SIZE, result.remaining());
        Assert.assertArrayEquals(Arrays.copyOfRange(original, BLOCK_SIZE, 2 * BLOCK_SIZE), byteBufferToArray(result));
        dfsIn.releaseBuffer(result);
    } finally {
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
    }
}

Example 15 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testZeroCopyReadsNoFallback.

@Test
public void testZeroCopyReadsNoFallback() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    FSDataInputStream fsIn = null;
    final int TEST_FILE_LENGTH = 3 * BLOCK_SIZE;
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, 7567L);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        byte[] original = new byte[TEST_FILE_LENGTH];
        IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
        fsIn.close();
        fsIn = fs.open(TEST_PATH);
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        ByteBuffer result;
        try {
            result = dfsIn.read(null, BLOCK_SIZE + 1, EnumSet.noneOf(ReadOption.class));
            Assert.fail("expected UnsupportedOperationException");
        } catch (UnsupportedOperationException e) {
        // expected
        }
        result = dfsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
        Assert.assertEquals(BLOCK_SIZE, result.remaining());
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BLOCK_SIZE, dfsIn.getReadStatistics().getTotalZeroCopyBytesRead());
        Assert.assertArrayEquals(Arrays.copyOfRange(original, 0, BLOCK_SIZE), byteBufferToArray(result));
    } finally {
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
    }
}