Examples with HdfsDataInputStream org.apache.hadoop.hdfs.client.HdfsDataInputStream used on opensource projects

Example 1 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestBlockReaderLocal method testStatistics.

private void testStatistics(boolean isShortCircuit) throws Exception {
    Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
    HdfsConfiguration conf = new HdfsConfiguration();
    TemporarySocketDirectory sockDir = null;
    if (isShortCircuit) {
        DFSInputStream.tcpReadsDisabledForTesting = true;
        sockDir = new TemporarySocketDirectory();
        conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), "TestStatisticsForLocalRead.%d.sock").getAbsolutePath());
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
        DomainSocket.disableBindPathValidation();
    } else {
        conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, false);
    }
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final long RANDOM_SEED = 4567L;
    FSDataInputStream fsIn = null;
    byte[] original = new byte[BlockReaderLocalTest.TEST_LENGTH];
    FileSystem fs = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).hosts(new String[] { NetUtils.getLocalHostname() }).build();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, TEST_PATH, BlockReaderLocalTest.TEST_LENGTH, (short) 1, RANDOM_SEED);
        try {
            DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
        } catch (InterruptedException e) {
            Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
        } catch (TimeoutException e) {
            Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
        }
        fsIn = fs.open(TEST_PATH);
        IOUtils.readFully(fsIn, original, 0, BlockReaderLocalTest.TEST_LENGTH);
        HdfsDataInputStream dfsIn = (HdfsDataInputStream) fsIn;
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalBytesRead());
        Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalLocalBytesRead());
        if (isShortCircuit) {
            Assert.assertEquals(BlockReaderLocalTest.TEST_LENGTH, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        } else {
            Assert.assertEquals(0, dfsIn.getReadStatistics().getTotalShortCircuitBytesRead());
        }
        fsIn.close();
        fsIn = null;
    } finally {
        DFSInputStream.tcpReadsDisabledForTesting = false;
        if (fsIn != null)
            fsIn.close();
        if (fs != null)
            fs.close();
        if (cluster != null)
            cluster.shutdown();
        if (sockDir != null)
            sockDir.close();
    }
}

Example 2 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestWriteRead method readData.

/**
   * Open the file to read from begin to end. Then close the file. 
   * Return number of bytes read. 
   * Support both sequential read and position read.
   */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition) throws IOException {
    long totalByteRead = 0;
    Path path = getFullyQualifiedPath(fname);
    FSDataInputStream in = null;
    try {
        in = openInputStream(path);
        long visibleLenFromReadStream = ((HdfsDataInputStream) in).getVisibleLength();
        if (visibleLenFromReadStream < byteExpected) {
            throw new IOException(visibleLenFromReadStream + " = visibleLenFromReadStream < bytesExpected= " + byteExpected);
        }
        totalByteRead = readUntilEnd(in, buffer, buffer.length, fname, beginPosition, visibleLenFromReadStream, positionReadOption);
        in.close();
        // reading more data than visibleLeng is OK, but not less
        if (totalByteRead + beginPosition < byteExpected) {
            throw new IOException("readData mismatch in byte read: expected=" + byteExpected + " ; got " + (totalByteRead + beginPosition));
        }
        return totalByteRead + beginPosition;
    } catch (IOException e) {
        throw new IOException("##### Caught Exception in readData. " + "Total Byte Read so far = " + totalByteRead + " beginPosition = " + beginPosition, e);
    } finally {
        if (in != null)
            in.close();
    }
}

Example 3 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestFileLengthOnClusterRestart method testFileLengthWithHSyncAndClusterRestartWithOutDNsRegister.

/**
   * Tests the fileLength when we sync the file and restart the cluster and
   * Datanodes not report to Namenode yet.
   */
@Test(timeout = 60000)
public void testFileLengthWithHSyncAndClusterRestartWithOutDNsRegister() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // create cluster
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    HdfsDataInputStream in = null;
    try {
        Path path = new Path("/tmp/TestFileLengthOnClusterRestart", "test");
        DistributedFileSystem dfs = cluster.getFileSystem();
        FSDataOutputStream out = dfs.create(path);
        int fileLength = 1030;
        out.write(new byte[fileLength]);
        out.hsync();
        cluster.restartNameNode();
        cluster.waitActive();
        in = (HdfsDataInputStream) dfs.open(path, 1024);
        // Verify the length when we just restart NN. DNs will register
        // immediately.
        Assert.assertEquals(fileLength, in.getVisibleLength());
        cluster.shutdownDataNodes();
        cluster.restartNameNode(false);
        // This is just for ensuring NN started.
        verifyNNIsInSafeMode(dfs);
        try {
            in = (HdfsDataInputStream) dfs.open(path);
            Assert.fail("Expected IOException");
        } catch (IOException e) {
            Assert.assertTrue(e.getLocalizedMessage().indexOf("Name node is in safe mode") >= 0);
        }
    } finally {
        if (null != in) {
            in.close();
        }
        cluster.shutdown();
    }
}

Example 4 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestScrLazyPersistFiles method tesScrDuringEviction.

/**
   * Eviction of lazy persisted blocks with Short Circuit Read handle open
   * Note: the test uses faked RAM_DISK from physical disk.
   * @throws IOException
   * @throws InterruptedException
   */
@Test
public void tesScrDuringEviction() throws Exception {
    getClusterBuilder().setUseScr(true).build();
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
    // Create a file and wait till it is persisted.
    makeTestFile(path1, BLOCK_SIZE, true);
    ensureFileReplicasOnStorageType(path1, RAM_DISK);
    waitForMetric("RamDiskBlocksLazyPersisted", 1);
    HdfsDataInputStream fis = (HdfsDataInputStream) fs.open(path1);
    try {
        // Keep and open read handle to path1 while creating path2
        byte[] buf = new byte[BUFFER_LENGTH];
        fis.read(0, buf, 0, BUFFER_LENGTH);
        triggerEviction(cluster.getDataNodes().get(0));
        // Ensure path1 is still readable from the open SCR handle.
        fis.read(0, buf, 0, BUFFER_LENGTH);
        assertThat(fis.getReadStatistics().getTotalBytesRead(), is((long) 2 * BUFFER_LENGTH));
        assertThat(fis.getReadStatistics().getTotalShortCircuitBytesRead(), is((long) 2 * BUFFER_LENGTH));
    } finally {
        IOUtils.closeQuietly(fis);
    }
}

Example 5 with HdfsDataInputStream

use of org.apache.hadoop.hdfs.client.HdfsDataInputStream in project hadoop by apache.

the class TestLeaseRecovery2 method hardLeaseRecoveryRestartHelper.

public void hardLeaseRecoveryRestartHelper(boolean doRename, int size) throws Exception {
    if (size < 0) {
        size = AppendTestUtil.nextInt(FILE_SIZE + 1);
    }
    //create a file
    String fileStr = "/hardLeaseRecovery";
    AppendTestUtil.LOG.info("filestr=" + fileStr);
    Path filePath = new Path(fileStr);
    FSDataOutputStream stm = dfs.create(filePath, true, BUF_SIZE, REPLICATION_NUM, BLOCK_SIZE);
    assertTrue(dfs.dfs.exists(fileStr));
    // write bytes into the file.
    AppendTestUtil.LOG.info("size=" + size);
    stm.write(buffer, 0, size);
    String originalLeaseHolder = NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), fileStr);
    assertFalse("original lease holder should not be the NN", originalLeaseHolder.equals(HdfsServerConstants.NAMENODE_LEASE_HOLDER));
    // hflush file
    AppendTestUtil.LOG.info("hflush");
    stm.hflush();
    // check visible length
    final HdfsDataInputStream in = (HdfsDataInputStream) dfs.open(filePath);
    Assert.assertEquals(size, in.getVisibleLength());
    in.close();
    if (doRename) {
        fileStr += ".renamed";
        Path renamedPath = new Path(fileStr);
        assertTrue(dfs.rename(filePath, renamedPath));
        filePath = renamedPath;
    }
    // kill the lease renewal thread
    AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
    dfs.dfs.getLeaseRenewer().interruptAndJoin();
    // won't actually get completed during lease recovery.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    // set the hard limit to be 1 second 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // Make sure lease recovery begins.
    final String path = fileStr;
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return HdfsServerConstants.NAMENODE_LEASE_HOLDER.equals(NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), path));
        }
    }, (int) SHORT_LEASE_PERIOD, (int) SHORT_LEASE_PERIOD * 10);
    // Normally, the in-progress edit log would be finalized by
    // FSEditLog#endCurrentLogSegment.  For testing purposes, we
    // disable that here.
    FSEditLog spyLog = spy(cluster.getNameNode().getFSImage().getEditLog());
    doNothing().when(spyLog).endCurrentLogSegment(Mockito.anyBoolean());
    DFSTestUtil.setEditLogForTesting(cluster.getNamesystem(), spyLog);
    cluster.restartNameNode(false);
    checkLease(fileStr, size);
    // Let the DNs send heartbeats again.
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
    }
    cluster.waitActive();
    // set the hard limit to be 1 second, to initiate lease recovery. 
    cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
    // wait for lease recovery to complete
    LocatedBlocks locatedBlocks;
    do {
        Thread.sleep(SHORT_LEASE_PERIOD);
        locatedBlocks = dfs.dfs.getLocatedBlocks(fileStr, 0L, size);
    } while (locatedBlocks.isUnderConstruction());
    assertEquals(size, locatedBlocks.getFileLength());
    // make sure that the client can't write data anymore.
    try {
        stm.write('b');
        stm.hflush();
        fail("Should not be able to flush after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expceted exception on write/hflush", e);
    }
    try {
        stm.close();
        fail("Should not be able to close after we've lost the lease");
    } catch (IOException e) {
        LOG.info("Expected exception on close", e);
    }
    // verify data
    AppendTestUtil.LOG.info("File size is good. Now validating sizes from datanodes...");
    AppendTestUtil.checkFullFile(dfs, filePath, size, buffer, fileStr);
}