Examples with FSDataInputStream org.apache.hadoop.fs.FSDataInputStream used on opensource projects

Example 61 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestSequenceFileSync method testLowSyncpoint.

@Test
public void testLowSyncpoint() throws IOException {
    // Uses a smaller sync interval of 2000 bytes
    final Configuration conf = new Configuration();
    final FileSystem fs = FileSystem.getLocal(conf);
    final Path path = new Path(GenericTestUtils.getTempPath("sequencefile.sync.test"));
    final IntWritable input = new IntWritable();
    final Text val = new Text();
    SequenceFile.Writer writer = new SequenceFile.Writer(conf, SequenceFile.Writer.file(path), SequenceFile.Writer.compression(CompressionType.NONE), SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.valueClass(Text.class), SequenceFile.Writer.syncInterval(20 * 100));
    // Ensure the custom sync interval value is set
    assertEquals(writer.syncInterval, 20 * 100);
    try {
        writeSequenceFile(writer, NUMRECORDS);
        for (int i = 0; i < 5; i++) {
            final SequenceFile.Reader reader;
            //try different SequenceFile.Reader constructors
            if (i % 2 == 0) {
                final int bufferSize = conf.getInt("io.file.buffer.size", 4096);
                reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path), SequenceFile.Reader.bufferSize(bufferSize));
            } else {
                final FSDataInputStream in = fs.open(path);
                final long length = fs.getFileStatus(path).getLen();
                reader = new SequenceFile.Reader(conf, SequenceFile.Reader.stream(in), SequenceFile.Reader.start(0L), SequenceFile.Reader.length(length));
            }
            try {
                forOffset(reader, input, val, i, 0, 0);
                forOffset(reader, input, val, i, 65, 0);
                // There would be only a few records within
                // this sync interval
                forOffset(reader, input, val, i, 2000, 21);
                forOffset(reader, input, val, i, 0, 0);
            } finally {
                reader.close();
            }
        }
    } finally {
        fs.delete(path, false);
    }
}

Example 62 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class DistributedFileSystem method open.

@Override
public FSDataInputStream open(Path f, final int bufferSize) throws IOException {
    statistics.incrementReadOps(1);
    storageStatistics.incrementOpCounter(OpType.OPEN);
    Path absF = fixRelativePart(f);
    return new FileSystemLinkResolver<FSDataInputStream>() {

        @Override
        public FSDataInputStream doCall(final Path p) throws IOException {
            final DFSInputStream dfsis = dfs.open(getPathName(p), bufferSize, verifyChecksum);
            return dfs.createWrappedInputStream(dfsis);
        }

        @Override
        public FSDataInputStream next(final FileSystem fs, final Path p) throws IOException {
            return fs.open(p, bufferSize);
        }
    }.resolve(this, absF);
}

Example 63 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestEncryptedTransfer method testEncryptedAppendRequiringBlockTransfer.

@Test
public void testEncryptedAppendRequiringBlockTransfer() throws IOException {
    setEncryptionConfigKeys();
    // start up 4 DNs
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fs = getFileSystem(conf);
    // Create a file with replication 3, so its block is on 3 / 4 DNs.
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT, DFSTestUtil.readFile(fs, TEST_PATH));
    // Shut down one of the DNs holding a block replica.
    FSDataInputStream in = fs.open(TEST_PATH);
    List<LocatedBlock> locatedBlocks = DFSTestUtil.getAllBlocks(in);
    in.close();
    assertEquals(1, locatedBlocks.size());
    assertEquals(3, locatedBlocks.get(0).getLocations().length);
    DataNode dn = cluster.getDataNode(locatedBlocks.get(0).getLocations()[0].getIpcPort());
    dn.shutdown();
    // Reopen the file for append, which will need to add another DN to the
    // pipeline and in doing so trigger a block transfer.
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT + PLAIN_TEXT, DFSTestUtil.readFile(fs, TEST_PATH));
}

Example 64 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestDistributedFileSystem method testDFSClient.

@Test
public void testDFSClient() throws Exception {
    Configuration conf = getTestConfiguration();
    final long grace = 1000L;
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
        final String filepathstring = "/test/LeaseChecker/foo";
        final Path[] filepaths = new Path[4];
        for (int i = 0; i < filepaths.length; i++) {
            filepaths[i] = new Path(filepathstring + i);
        }
        final long millis = Time.now();
        {
            final DistributedFileSystem dfs = cluster.getFileSystem();
            Method setMethod = dfs.dfs.getLeaseRenewer().getClass().getDeclaredMethod("setGraceSleepPeriod", long.class);
            setMethod.setAccessible(true);
            setMethod.invoke(dfs.dfs.getLeaseRenewer(), grace);
            Method checkMethod = dfs.dfs.getLeaseRenewer().getClass().getDeclaredMethod("isRunning");
            checkMethod.setAccessible(true);
            assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            {
                //create a file
                final FSDataOutputStream out = dfs.create(filepaths[0]);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //write something
                out.writeLong(millis);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //close
                out.close();
                Thread.sleep(grace / 4 * 3);
                //within grace period
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                for (int i = 0; i < 3; i++) {
                    if ((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer())) {
                        Thread.sleep(grace / 2);
                    }
                }
                //passed grace period
                assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            }
            {
                //create file1
                final FSDataOutputStream out1 = dfs.create(filepaths[1]);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //create file2
                final FSDataOutputStream out2 = dfs.create(filepaths[2]);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //write something to file1
                out1.writeLong(millis);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //close file1
                out1.close();
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //write something to file2
                out2.writeLong(millis);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //close file2
                out2.close();
                Thread.sleep(grace / 4 * 3);
                //within grace period
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            }
            {
                //create file3
                final FSDataOutputStream out3 = dfs.create(filepaths[3]);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                Thread.sleep(grace / 4 * 3);
                //passed previous grace period, should still running
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //write something to file3
                out3.writeLong(millis);
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                //close file3
                out3.close();
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                Thread.sleep(grace / 4 * 3);
                //within grace period
                assertTrue((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
                for (int i = 0; i < 3; i++) {
                    if ((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer())) {
                        Thread.sleep(grace / 2);
                    }
                }
                //passed grace period
                assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            }
            dfs.close();
        }
        {
            // Check to see if opening a non-existent file triggers a FNF
            FileSystem fs = cluster.getFileSystem();
            Path dir = new Path("/wrwelkj");
            assertFalse("File should not exist for test.", fs.exists(dir));
            try {
                FSDataInputStream in = fs.open(dir);
                try {
                    in.close();
                    fs.close();
                } finally {
                    assertTrue("Did not get a FileNotFoundException for non-existing" + " file.", false);
                }
            } catch (FileNotFoundException fnf) {
            // This is the proper exception to catch; move on.
            }
        }
        {
            final DistributedFileSystem dfs = cluster.getFileSystem();
            Method checkMethod = dfs.dfs.getLeaseRenewer().getClass().getDeclaredMethod("isRunning");
            checkMethod.setAccessible(true);
            assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            //open and check the file
            FSDataInputStream in = dfs.open(filepaths[0]);
            assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            assertEquals(millis, in.readLong());
            assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            in.close();
            assertFalse((boolean) checkMethod.invoke(dfs.dfs.getLeaseRenewer()));
            dfs.close();
        }
        {
            // test accessing DFS with ip address. should work with any hostname
            // alias or ip address that points to the interface that NameNode
            // is listening on. In this case, it is localhost.
            String uri = "hdfs://127.0.0.1:" + cluster.getNameNodePort() + "/test/ipAddress/file";
            Path path = new Path(uri);
            FileSystem fs = FileSystem.get(path.toUri(), conf);
            FSDataOutputStream out = fs.create(path);
            byte[] buf = new byte[1024];
            out.write(buf);
            out.close();
            FSDataInputStream in = fs.open(path);
            in.readFully(buf);
            in.close();
            fs.close();
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 65 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestDistributedFileSystem method testStatistics.

@Test
public void testStatistics() throws IOException {
    FileSystem.getStatistics(HdfsConstants.HDFS_URI_SCHEME, DistributedFileSystem.class).reset();
    @SuppressWarnings("unchecked") ThreadLocal<StatisticsData> data = (ThreadLocal<StatisticsData>) Whitebox.getInternalState(FileSystem.getStatistics(HdfsConstants.HDFS_URI_SCHEME, DistributedFileSystem.class), "threadData");
    data.set(null);
    int lsLimit = 2;
    final Configuration conf = getTestConfiguration();
    conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, lsLimit);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        cluster.waitActive();
        final FileSystem fs = cluster.getFileSystem();
        Path dir = new Path("/test");
        Path file = new Path(dir, "file");
        int readOps = 0;
        int writeOps = 0;
        int largeReadOps = 0;
        long opCount = getOpStatistics(OpType.MKDIRS);
        fs.mkdirs(dir);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.MKDIRS, opCount + 1);
        opCount = getOpStatistics(OpType.CREATE);
        FSDataOutputStream out = fs.create(file, (short) 1);
        out.close();
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.CREATE, opCount + 1);
        opCount = getOpStatistics(OpType.GET_FILE_STATUS);
        FileStatus status = fs.getFileStatus(file);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_FILE_STATUS, opCount + 1);
        opCount = getOpStatistics(OpType.GET_FILE_BLOCK_LOCATIONS);
        fs.getFileBlockLocations(file, 0, 0);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_FILE_BLOCK_LOCATIONS, opCount + 1);
        fs.getFileBlockLocations(status, 0, 0);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_FILE_BLOCK_LOCATIONS, opCount + 2);
        opCount = getOpStatistics(OpType.OPEN);
        FSDataInputStream in = fs.open(file);
        in.close();
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.OPEN, opCount + 1);
        opCount = getOpStatistics(OpType.SET_REPLICATION);
        fs.setReplication(file, (short) 2);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.SET_REPLICATION, opCount + 1);
        opCount = getOpStatistics(OpType.RENAME);
        Path file1 = new Path(dir, "file1");
        fs.rename(file, file1);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.RENAME, opCount + 1);
        opCount = getOpStatistics(OpType.GET_CONTENT_SUMMARY);
        fs.getContentSummary(file1);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_CONTENT_SUMMARY, opCount + 1);
        // Iterative ls test
        long mkdirOp = getOpStatistics(OpType.MKDIRS);
        long listStatusOp = getOpStatistics(OpType.LIST_STATUS);
        for (int i = 0; i < 10; i++) {
            Path p = new Path(dir, Integer.toString(i));
            fs.mkdirs(p);
            mkdirOp++;
            FileStatus[] list = fs.listStatus(dir);
            if (list.length > lsLimit) {
                // if large directory, then count readOps and largeReadOps by 
                // number times listStatus iterates
                int iterations = (int) Math.ceil((double) list.length / lsLimit);
                largeReadOps += iterations;
                readOps += iterations;
                listStatusOp += iterations;
            } else {
                // Single iteration in listStatus - no large read operation done
                readOps++;
                listStatusOp++;
            }
            // writeOps incremented by 1 for mkdirs
            // readOps and largeReadOps incremented by 1 or more
            checkStatistics(fs, readOps, ++writeOps, largeReadOps);
            checkOpStatistics(OpType.MKDIRS, mkdirOp);
            checkOpStatistics(OpType.LIST_STATUS, listStatusOp);
        }
        opCount = getOpStatistics(OpType.GET_STATUS);
        fs.getStatus(file1);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_STATUS, opCount + 1);
        opCount = getOpStatistics(OpType.GET_FILE_CHECKSUM);
        fs.getFileChecksum(file1);
        checkStatistics(fs, ++readOps, writeOps, largeReadOps);
        checkOpStatistics(OpType.GET_FILE_CHECKSUM, opCount + 1);
        opCount = getOpStatistics(OpType.SET_PERMISSION);
        fs.setPermission(file1, new FsPermission((short) 0777));
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.SET_PERMISSION, opCount + 1);
        opCount = getOpStatistics(OpType.SET_TIMES);
        fs.setTimes(file1, 0L, 0L);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.SET_TIMES, opCount + 1);
        opCount = getOpStatistics(OpType.SET_OWNER);
        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
        fs.setOwner(file1, ugi.getUserName(), ugi.getGroupNames()[0]);
        checkOpStatistics(OpType.SET_OWNER, opCount + 1);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        opCount = getOpStatistics(OpType.DELETE);
        fs.delete(dir, true);
        checkStatistics(fs, readOps, ++writeOps, largeReadOps);
        checkOpStatistics(OpType.DELETE, opCount + 1);
    } finally {
        if (cluster != null)
            cluster.shutdown();
    }
}