Examples with FSDataInputStream org.apache.hadoop.fs.FSDataInputStream used on opensource projects

Example 16 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryForLastBlock.

/** Test whether corrupt replicas are detected correctly during pipeline
   * recoveries.
   */
@Test
public void testPipelineRecoveryForLastBlock() throws IOException {
    DFSClientFaultInjector faultInjector = Mockito.mock(DFSClientFaultInjector.class);
    DFSClientFaultInjector oldInjector = DFSClientFaultInjector.get();
    DFSClientFaultInjector.set(faultInjector);
    Configuration conf = new HdfsConfiguration();
    conf.setInt(HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 3);
    MiniDFSCluster cluster = null;
    try {
        int numDataNodes = 3;
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
        cluster.waitActive();
        FileSystem fileSys = cluster.getFileSystem();
        Path file = new Path("dataprotocol1.dat");
        Mockito.when(faultInjector.failPacket()).thenReturn(true);
        DFSTestUtil.createFile(fileSys, file, 68000000L, (short) numDataNodes, 0L);
        // At this point, NN should have accepted only valid replicas.
        // Read should succeed.
        FSDataInputStream in = fileSys.open(file);
        try {
            in.read();
        // Test will fail with BlockMissingException if NN does not update the
        // replica state based on the latest report.
        } catch (org.apache.hadoop.hdfs.BlockMissingException bme) {
            Assert.fail("Block is missing because the file was closed with" + " corrupt replicas.");
        }
    } finally {
        DFSClientFaultInjector.set(oldInjector);
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 17 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestCrcCorruption method testCorruptionDuringWrt.

/** 
   * Test case for data corruption during data transmission for
   * create/write. To recover from corruption while writing, at
   * least two replicas are needed.
   */
@Test(timeout = 50000)
public void testCorruptionDuringWrt() throws Exception {
    Configuration conf = new HdfsConfiguration();
    // Set short retry timeouts so this test runs faster
    conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(10).build();
        cluster.waitActive();
        FileSystem fs = cluster.getFileSystem();
        Path file = new Path("/test_corruption_file");
        FSDataOutputStream out = fs.create(file, true, 8192, (short) 3, (long) (128 * 1024 * 1024));
        byte[] data = new byte[65536];
        for (int i = 0; i < 65536; i++) {
            data[i] = (byte) (i % 256);
        }
        for (int i = 0; i < 5; i++) {
            out.write(data, 0, 65535);
        }
        out.hflush();
        // corrupt the packet once
        Mockito.when(faultInjector.corruptPacket()).thenReturn(true, false);
        Mockito.when(faultInjector.uncorruptPacket()).thenReturn(true, false);
        for (int i = 0; i < 5; i++) {
            out.write(data, 0, 65535);
        }
        out.close();
        // read should succeed
        FSDataInputStream in = fs.open(file);
        for (int c; (c = in.read()) != -1; ) ;
        in.close();
        // test the retry limit
        out = fs.create(file, true, 8192, (short) 3, (long) (128 * 1024 * 1024));
        // corrupt the packet once and never fix it.
        Mockito.when(faultInjector.corruptPacket()).thenReturn(true, false);
        Mockito.when(faultInjector.uncorruptPacket()).thenReturn(false);
        // the client should give up pipeline reconstruction after retries.
        try {
            for (int i = 0; i < 5; i++) {
                out.write(data, 0, 65535);
            }
            out.close();
            fail("Write did not fail");
        } catch (IOException ioe) {
            // we should get an ioe
            DFSClient.LOG.info("Got expected exception", ioe);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
        Mockito.when(faultInjector.corruptPacket()).thenReturn(false);
        Mockito.when(faultInjector.uncorruptPacket()).thenReturn(false);
    }
}

Example 18 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestBlockMissingException method validateFile.

//
// validates that file encounters BlockMissingException
//
private void validateFile(FileSystem fileSys, Path name) throws IOException {
    FSDataInputStream stm = fileSys.open(name);
    final byte[] b = new byte[4192];
    int num = 0;
    boolean gotException = false;
    try {
        while (num >= 0) {
            num = stm.read(b);
            if (num < 0) {
                break;
            }
        }
    } catch (BlockMissingException e) {
        gotException = true;
    }
    stm.close();
    assertTrue("Expected BlockMissingException ", gotException);
}

Example 19 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestReplaceDatanodeOnFailure method testReplaceDatanodeOnFailure.

/** Test replace datanode on failure. */
@Test
public void testReplaceDatanodeOnFailure() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // do not consider load factor when selecting a data node
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false);
    //always replace a datanode
    ReplaceDatanodeOnFailure.write(Policy.ALWAYS, true, conf);
    final String[] racks = new String[REPLICATION];
    Arrays.fill(racks, RACK0);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).racks(racks).numDataNodes(REPLICATION).build();
    try {
        cluster.waitActive();
        final DistributedFileSystem fs = cluster.getFileSystem();
        final Path dir = new Path(DIR);
        final int NUM_WRITERS = 10;
        final int FIRST_BATCH = 5;
        final SlowWriter[] slowwriters = new SlowWriter[NUM_WRITERS];
        for (int i = 1; i <= slowwriters.length; i++) {
            //create slow writers in different speed
            slowwriters[i - 1] = new SlowWriter(fs, new Path(dir, "file" + i), i * 200L);
        }
        for (int i = 0; i < FIRST_BATCH; i++) {
            slowwriters[i].start();
        }
        // Let slow writers write something.
        // Some of them are too slow and will be not yet started.
        sleepSeconds(3);
        //start new datanodes
        cluster.startDataNodes(conf, 2, true, null, new String[] { RACK1, RACK1 });
        cluster.waitActive();
        // wait for first block reports for up to 10 seconds
        cluster.waitFirstBRCompleted(0, 10000);
        //stop an old datanode
        MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(AppendTestUtil.nextInt(REPLICATION));
        for (int i = FIRST_BATCH; i < slowwriters.length; i++) {
            slowwriters[i].start();
        }
        waitForBlockReplication(slowwriters);
        //check replication and interrupt.
        for (SlowWriter s : slowwriters) {
            s.checkReplication();
            s.interruptRunning();
        }
        //close files
        for (SlowWriter s : slowwriters) {
            s.joinAndClose();
        }
        //Verify the file
        LOG.info("Verify the file");
        for (int i = 0; i < slowwriters.length; i++) {
            LOG.info(slowwriters[i].filepath + ": length=" + fs.getFileStatus(slowwriters[i].filepath).getLen());
            FSDataInputStream in = null;
            try {
                in = fs.open(slowwriters[i].filepath);
                for (int j = 0, x; (x = in.read()) != -1; j++) {
                    Assert.assertEquals(j, x);
                }
            } finally {
                IOUtils.closeStream(in);
            }
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 20 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestBlockReaderLocal method testStatistics.

private void testStatistics(boolean isShortCircuit) throws Exception {
    Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
    HdfsConfiguration conf = new HdfsConfiguration();
    TemporarySocketDirectory sockDir = null;
    if (isShortCircuit) {
        DFSInputStream.tcpReadsDisabledForTesting = true;
        sockDir = new TemporarySocketDirectory();
        conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestStatisticsForLocalRead.%d.sock").getAbsolutePath());
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
        DomainSocket.disableBindPathValidation();
    } else {
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, false);
    }
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final long RANDOM_SEED = 4567L;
    FSDataInputStream fsIn = null;
    byte[] original = new byte[BlockReaderLocalTest.TEST_LENGTH];
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).hosts(new String[] { NetUtils.getLocalHostname() }).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, BlockReaderLocalTest.TEST_LENGTH, (short) 1, RANDOM_SEED);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        IOUtils.readFully(fsIn, original, 0, BlockReaderLocalTest.TEST_LENGTH);
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalLocalBytesRead());
        if (isShortCircuit) {
            Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        } else {
            Assert.assertEquals(0, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        }
        fsIn.close();
        fsIn = null;
    } finally {
        DFSInputStream.tcpReadsDisabledForTesting = false;
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
        if (sockDir != null)
            sockDir.close();
    }
}