Search in sources :

Example 1 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestBlockReaderLocal method testStatistics.

private void testStatistics(boolean isShortCircuit) throws Exception {
    Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
    HdfsConfiguration conf = new HdfsConfiguration();
    TemporarySocketDirectory sockDir = null;
    if (isShortCircuit) {
        DFSInputStream.tcpReadsDisabledForTesting = true;
        sockDir = new TemporarySocketDirectory();
        conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestStatisticsForLocalRead.%d.sock").getAbsolutePath());
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
        DomainSocket.disableBindPathValidation();
    } else {
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, false);
    }
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final long RANDOM_SEED = 4567L;
    FSDataInputStream fsIn = null;
    byte[] original = new byte[BlockReaderLocalTest.TEST_LENGTH];
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).hosts(new String[] { NetUtils.getLocalHostname() }).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, BlockReaderLocalTest.TEST_LENGTH, (short) 1, RANDOM_SEED);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        IOUtils.readFully(fsIn, original, 0, BlockReaderLocalTest.TEST_LENGTH);
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalLocalBytesRead());
        if (isShortCircuit) {
            Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        } else {
            Assert.assertEquals(0, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        }
        fsIn.close();
        fsIn = null;
    } finally {
        DFSInputStream.tcpReadsDisabledForTesting = false;
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
        if (sockDir != null)
            sockDir.close();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) TemporarySocketDirectory(org.apache.hadoop.net.unix.TemporarySocketDirectory) FileSystem(org.apache.hadoop.fs.FileSystem) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) HdfsDataInputStream(org.apache.hadoop.hdfs.client.HdfsDataInputStream) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestWriteRead method readData.

/**
   * Open the file to read from begin to end. Then close the file. 
   * Return number of bytes read. 
   * Support both sequential read and position read.
   */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition) throws IOException {
    long totalByteRead = 0;
    Path path = getFullyQualifiedPath(fname);
    FSDataInputStream in = null;
    try {
        in = openInputStream(path);
        long visibleLenFromReadStream = ((HdfsDataInputStream) in).getVisibleLength();
        if (visibleLenFromReadStream < byteExpected) {
            throw new IOException(visibleLenFromReadStream + " = visibleLenFromReadStream < bytesExpected= " + byteExpected);
        }
        totalByteRead = readUntilEnd(in, buffer, buffer.length, fname, beginPosition, visibleLenFromReadStream, positionReadOption);
        in.close();
        // reading more data than visibleLeng is OK, but not less
        if (totalByteRead + beginPosition < byteExpected) {
            throw new IOException("readData mismatch in byte read: expected=" + byteExpected + " ; got " + (totalByteRead + beginPosition));
        }
        return totalByteRead + beginPosition;
    } catch (IOException e) {
        throw new IOException("##### Caught Exception in readData. " + "Total Byte Read so far = " + totalByteRead + " beginPosition = " + beginPosition, e);
    } finally {
        if (in != null)
            in.close();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) FSDataInputStream(org.apache.hadoop.fs.FSDataInputStream) IOException(java.io.IOException) HdfsDataInputStream(org.apache.hadoop.hdfs.client.HdfsDataInputStream)

Example 3 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestFileLengthOnClusterRestart method testFileLengthWithHSyncAndClusterRestartWithOutDNsRegister.

/**
   * Tests the fileLength when we sync the file and restart the cluster and
   * Datanodes not report to Namenode yet.
   */
@Test(timeout = 60000)
public void testFileLengthWithHSyncAndClusterRestartWithOutDNsRegister() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // create cluster
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    HdfsDataInputStream in = null;
    try {
        Path path = new Path("/tmp/TestFileLengthOnClusterRestart", "test");
        DistributedFileSystem dfs = cluster.getFileSystem();
        FSDataOutputStream out = dfs.create(path);
        int fileLength = 1030;
        out.write(new byte[fileLength]);
        out.hsync();
        cluster.restartNameNode();
        cluster.waitActive();
        in = (HdfsDataInputStream) dfs.open(path, 1024);
        // Verify the length when we just restart NN. DNs will register
        // immediately.
        Assert.assertEquals(fileLength, in.getVisibleLength());
        cluster.shutdownDataNodes();
        cluster.restartNameNode(false);
        // This is just for ensuring NN started.
        verifyNNIsInSafeMode(dfs);
        try {
            in = (HdfsDataInputStream) dfs.open(path);
            Assert.fail("Expected IOException");
        } catch (IOException e) {
            Assert.assertTrue(e.getLocalizedMessage().indexOf("Name node is in safe mode") >= 0);
        }
    } finally {
        if (null != in) {
            in.close();
        }
        cluster.shutdown();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Configuration(org.apache.hadoop.conf.Configuration) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) IOException(java.io.IOException) HdfsDataInputStream(org.apache.hadoop.hdfs.client.HdfsDataInputStream) Test(org.junit.Test)

Example 4 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestScrLazyPersistFiles method tesScrDuringEviction.

/**
   * Eviction of lazy persisted blocks with Short Circuit Read handle open
   * Note: the test uses faked RAM_DISK from physical disk.
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void tesScrDuringEviction() throws Exception {
    getClusterBuilder().setUseScr(true).build();
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
    // Create a file and wait till it is persisted.
    makeTestFile(path1, BLOCK_SIZE, true);
    ensureFileReplicasOnStorageType(path1, RAM_DISK);
    waitForMetric("RamDiskBlocksLazyPersisted", 1);
    HdfsDataInputStream fis = (HdfsDataInputStream) fs.open(path1);
    try {
        // Keep and open read handle to path1 while creating path2
        byte[] buf = new byte[BUFFER_LENGTH];
        fis.read(0, buf, 0, BUFFER_LENGTH);
        triggerEviction(cluster.getDataNodes().get(0));
        // Ensure path1 is still readable from the open SCR handle.
        fis.read(0, buf, 0, BUFFER_LENGTH);
        assertThat(fis.getReadStatistics().getTotalBytesRead(), is((long) 2 * BUFFER_LENGTH));
        assertThat(fis.getReadStatistics().getTotalShortCircuitBytesRead(), is((long) 2 * BUFFER_LENGTH));
    } finally {
        IOUtils.closeQuietly(fis);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) HdfsDataInputStream(org.apache.hadoop.hdfs.client.HdfsDataInputStream) Test(org.junit.Test)

Example 5 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestLeaseRecovery2 method hardLeaseRecoveryRestartHelper.

public void hardLeaseRecoveryRestartHelper(boolean doRename, int size) throws Exception {
    if (size < 0) {
        size = AppendTestUtil.nextInt(FILE_SIZE + 1);
    }
    //create a file
    String fileStr = "/hardLeaseRecovery";
    AppendTestUtil.LOG.info("filestr=" + fileStr);
    Path filePath = new Path(fileStr);
    FSDataOutputStream stm = dfs.create(filePath, true, BUF_SIZE, REPLICATION_NUM, BLOCK_SIZE);
    assertTrue(dfs.dfs.exists(fileStr));
    // write bytes into the file.
    AppendTestUtil.LOG.info("size=" + size);
    stm.write(buffer, 0, size);
    String originalLeaseHolder = NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), fileStr);
    assertFalse("original lease holder should not be the NN", originalLeaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
    // hflush file
    AppendTestUtil.LOG.info("hflush");
    stm.hflush();
    // check visible length
    final HdfsDataInputStream in = (HdfsDataInputStream) dfs.open(filePath);
    Assert.assertEquals(size, in.getVisibleLength());
    in.close();
    if (doRename) {
        fileStr += ".renamed";
        Path renamedPath = new Path(fileStr);
        assertTrue(dfs.rename(filePath, renamedPath));
        filePath = renamedPath;
    }
    // kill the lease renewal thread
    AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
    dfs.dfs.getLeaseRenewer().interruptAndJoin();
    // won't actually get completed during lease recovery.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    // set the hard limit to be 1 second 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // Make sure lease recovery begins.
    final String path = fileStr;
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return HdfsServerConstants.NAMENODE_LEASE_HOLDER.equals(NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path));
        }
    }, (int) SHORT_LEASE_PERIOD, (int) SHORT_LEASE_PERIOD * 10);
    // Normally, the in-progress edit log would be finalized by
    // FSEditLog#endCurrentLogSegment.  For testing purposes, we
    // disable that here.
    FSEditLog spyLog = spy(cluster.getNameNode().getFSImage().getEditLog());
    doNothing().when(spyLog).endCurrentLogSegment(Mockito.anyBoolean());
    DFSTestUtil.setEditLogForTesting(cluster.getNamesystem(), spyLog);
    cluster.restartNameNode(false);
    checkLease(fileStr, size);
    // Let the DNs send heartbeats again.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
    }
    cluster.waitActive();
    // set the hard limit to be 1 second, to initiate lease recovery. 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // wait for lease recovery to complete
    LocatedBlocks locatedBlocks;
    do {
        Thread.sleep(SHORT_LEASE_PERIOD);
        locatedBlocks = dfs.dfs.getLocatedBlocks(fileStr, 0L, size);
    } while (locatedBlocks.isUnderConstruction());
    assertEquals(size, locatedBlocks.getFileLength());
    // make sure that the client can't write data anymore.
    try {
        stm.write('b');
        stm.hflush();
        fail("Should not be able to flush after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expceted exception on write/hflush", e);
    }
    try {
        stm.close();
        fail("Should not be able to close after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expected exception on close", e);
    }
    // verify data
    AppendTestUtil.LOG.info("File size is good. Now validating sizes from datanodes...");
    AppendTestUtil.checkFullFile(dfs, filePath, size, buffer, fileStr);
}
Also used : Path(org.apache.hadoop.fs.Path) FSEditLog(org.apache.hadoop.hdfs.server.namenode.FSEditLog) DataNode(org.apache.hadoop.hdfs.server.datanode.DataNode) LocatedBlocks(org.apache.hadoop.hdfs.protocol.LocatedBlocks) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) IOException(java.io.IOException) HdfsDataInputStream(org.apache.hadoop.hdfs.client.HdfsDataInputStream)

Aggregations

HdfsDataInputStream (org.apache.hadoop.hdfs.client.HdfsDataInputStream)20 Path (org.apache.hadoop.fs.Path)8 Test (org.junit.Test)7 ByteBuffer (java.nio.ByteBuffer)5 LocatedBlock (org.apache.hadoop.hdfs.protocol.LocatedBlock)5 TimeoutException (java.util.concurrent.TimeoutException)4 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)4 HdfsConfiguration (org.apache.hadoop.hdfs.HdfsConfiguration)4 MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)4 DatanodeInfo (org.apache.hadoop.hdfs.protocol.DatanodeInfo)4 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)2 Configuration (org.apache.hadoop.conf.Configuration)2 FSDataInputStream (org.apache.hadoop.fs.FSDataInputStream)2 FSDataOutputStream (org.apache.hadoop.fs.FSDataOutputStream)2 FileSystem (org.apache.hadoop.fs.FileSystem)2 DefaultHttpResponse (io.netty.handler.codec.http.DefaultHttpResponse)1 HttpHeaders (io.netty.handler.codec.http.HttpHeaders)1 ChunkedStream (io.netty.handler.stream.ChunkedStream)1 EOFException (java.io.EOFException)1