Examples with FSDataOutputStream org.apache.hadoop.fs.FSDataOutputStream used on opensource projects

Example 36 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestWriteRead method testWriteAndRead.

/**
   * Common routine to do position read while open the file for write. 
   * After each iteration of write, do a read of the file from begin to end. 
   * Return 0 on success, else number of failure.
   */
private int testWriteAndRead(String fname, int loopN, int chunkSize, long readBeginPosition) throws IOException {
    int countOfFailures = 0;
    long byteVisibleToRead = 0;
    FSDataOutputStream out = null;
    byte[] outBuffer = new byte[BUFFER_SIZE];
    byte[] inBuffer = new byte[BUFFER_SIZE];
    for (int i = 0; i < BUFFER_SIZE; i++) {
        outBuffer[i] = (byte) (i & 0x00ff);
    }
    try {
        Path path = getFullyQualifiedPath(fname);
        long fileLengthBeforeOpen = 0;
        if (ifExists(path)) {
            if (truncateOption) {
                out = useFCOption ? mfc.create(path, EnumSet.of(CreateFlag.OVERWRITE)) : mfs.create(path, truncateOption);
                LOG.info("File already exists. File open with Truncate mode: " + path);
            } else {
                out = useFCOption ? mfc.create(path, EnumSet.of(CreateFlag.APPEND)) : mfs.append(path);
                fileLengthBeforeOpen = getFileLengthFromNN(path);
                LOG.info("File already exists of size " + fileLengthBeforeOpen + " File open for Append mode: " + path);
            }
        } else {
            out = useFCOption ? mfc.create(path, EnumSet.of(CreateFlag.CREATE)) : mfs.create(path);
        }
        long totalByteWritten = fileLengthBeforeOpen;
        long totalByteVisible = fileLengthBeforeOpen;
        long totalByteWrittenButNotVisible = 0;
        boolean toFlush;
        for (int i = 0; i < loopN; i++) {
            toFlush = (i % 2) == 0;
            writeData(out, outBuffer, chunkSize);
            totalByteWritten += chunkSize;
            if (toFlush) {
                out.hflush();
                totalByteVisible += chunkSize + totalByteWrittenButNotVisible;
                totalByteWrittenButNotVisible = 0;
            } else {
                totalByteWrittenButNotVisible += chunkSize;
            }
            if (verboseOption) {
                LOG.info("TestReadWrite - Written " + chunkSize + ". Total written = " + totalByteWritten + ". TotalByteVisible = " + totalByteVisible + " to file " + fname);
            }
            byteVisibleToRead = readData(fname, inBuffer, totalByteVisible, readBeginPosition);
            String readmsg = "Written=" + totalByteWritten + " ; Expected Visible=" + totalByteVisible + " ; Got Visible=" + byteVisibleToRead + " of file " + fname;
            if (byteVisibleToRead >= totalByteVisible && byteVisibleToRead <= totalByteWritten) {
                readmsg = "pass: reader sees expected number of visible byte. " + readmsg + " [pass]";
            } else {
                countOfFailures++;
                readmsg = "fail: reader see different number of visible byte. " + readmsg + " [fail]";
                if (abortTestOnFailure) {
                    throw new IOException(readmsg);
                }
            }
            LOG.info(readmsg);
        }
        // test the automatic flush after close
        writeData(out, outBuffer, chunkSize);
        totalByteWritten += chunkSize;
        totalByteVisible += chunkSize + totalByteWrittenButNotVisible;
        totalByteWrittenButNotVisible += 0;
        out.close();
        byteVisibleToRead = readData(fname, inBuffer, totalByteVisible, readBeginPosition);
        String readmsg2 = "Written=" + totalByteWritten + " ; Expected Visible=" + totalByteVisible + " ; Got Visible=" + byteVisibleToRead + " of file " + fname;
        String readmsg;
        if (byteVisibleToRead >= totalByteVisible && byteVisibleToRead <= totalByteWritten) {
            readmsg = "pass: reader sees expected number of visible byte on close. " + readmsg2 + " [pass]";
        } else {
            countOfFailures++;
            readmsg = "fail: reader sees different number of visible byte on close. " + readmsg2 + " [fail]";
            LOG.info(readmsg);
            if (abortTestOnFailure)
                throw new IOException(readmsg);
        }
        // now check if NN got the same length 
        long lenFromFc = getFileLengthFromNN(path);
        if (lenFromFc != byteVisibleToRead) {
            readmsg = "fail: reader sees different number of visible byte from NN " + readmsg2 + " [fail]";
            throw new IOException(readmsg);
        }
    } catch (IOException e) {
        throw new IOException("##### Caught Exception in testAppendWriteAndRead. Close file. " + "Total Byte Read so far = " + byteVisibleToRead, e);
    } finally {
        if (out != null)
            out.close();
    }
    return -countOfFailures;
}

Example 37 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestWriteStripedFileWithFailure method writeFileWithDNFailure.

/**
   * Test writing a file with shutting down some DNs(data DNs or parity DNs or both).
   * @param fileLength file length
   * @param dataDNFailureNum the shutdown number of data DNs
   * @param parityDNFailureNum the shutdown number of parity DNs
   * @throws IOException
   */
private void writeFileWithDNFailure(int fileLength, int dataDNFailureNum, int parityDNFailureNum) throws IOException {
    String fileType = fileLength < (blockSize * dataBlocks) ? "smallFile" : "largeFile";
    String src = "/dnFailure_" + dataDNFailureNum + "_" + parityDNFailureNum + "_" + fileType;
    LOG.info("writeFileWithDNFailure: file = " + src + ", fileType = " + fileType + ", dataDNFailureNum = " + dataDNFailureNum + ", parityDNFailureNum = " + parityDNFailureNum);
    Path srcPath = new Path(src);
    final AtomicInteger pos = new AtomicInteger();
    final FSDataOutputStream out = fs.create(srcPath);
    final DFSStripedOutputStream stripedOut = (DFSStripedOutputStream) out.getWrappedStream();
    int[] dataDNFailureIndices = StripedFileTestUtil.randomArray(0, dataBlocks, dataDNFailureNum);
    Assert.assertNotNull(dataDNFailureIndices);
    int[] parityDNFailureIndices = StripedFileTestUtil.randomArray(dataBlocks, dataBlocks + parityBlocks, parityDNFailureNum);
    Assert.assertNotNull(parityDNFailureIndices);
    int[] failedDataNodes = new int[dataDNFailureNum + parityDNFailureNum];
    System.arraycopy(dataDNFailureIndices, 0, failedDataNodes, 0, dataDNFailureIndices.length);
    System.arraycopy(parityDNFailureIndices, 0, failedDataNodes, dataDNFailureIndices.length, parityDNFailureIndices.length);
    final int killPos = fileLength / 2;
    for (; pos.get() < fileLength; ) {
        final int i = pos.getAndIncrement();
        if (i == killPos) {
            for (int failedDn : failedDataNodes) {
                StripedFileTestUtil.killDatanode(cluster, stripedOut, failedDn, pos);
            }
        }
        write(out, i);
    }
    out.close();
    // make sure the expected number of Datanode have been killed
    int dnFailureNum = dataDNFailureNum + parityDNFailureNum;
    Assert.assertEquals(cluster.getDataNodes().size(), numDNs - dnFailureNum);
    byte[] smallBuf = new byte[1024];
    byte[] largeBuf = new byte[fileLength + 100];
    final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
    StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
    StripedFileTestUtil.verifySeek(fs, srcPath, fileLength, ecPolicy, blockSize * dataBlocks);
    StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf);
    StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
    // delete the file
    fs.delete(srcPath, true);
}

Example 38 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestHSync method testHSyncWithReplication.

/** Test that syncBlock is correctly performed at replicas */
@Test
public void testHSyncWithReplication() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    final FileSystem fs = cluster.getFileSystem();
    final Path p = new Path("/testHSyncWithReplication/foo");
    final int len = 1 << 16;
    FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), 4096, (short) 3, len, null);
    out.write(1);
    out.hflush();
    checkSyncMetric(cluster, 0, 0);
    checkSyncMetric(cluster, 1, 0);
    checkSyncMetric(cluster, 2, 0);
    out.hsync();
    checkSyncMetric(cluster, 0, 1);
    checkSyncMetric(cluster, 1, 1);
    checkSyncMetric(cluster, 2, 1);
    out.hsync();
    checkSyncMetric(cluster, 0, 2);
    checkSyncMetric(cluster, 1, 2);
    checkSyncMetric(cluster, 2, 2);
    cluster.shutdown();
}

Example 39 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestHSync method testHSyncOperation.

private void testHSyncOperation(boolean testWithAppend) throws IOException {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    final DistributedFileSystem fs = cluster.getFileSystem();
    final Path p = new Path("/testHSync/foo");
    final int len = 1 << 16;
    FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), 4096, (short) 1, len, null);
    if (testWithAppend) {
        // re-open the file with append call
        out.close();
        out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK), 4096, null);
    }
    out.hflush();
    // hflush does not sync
    checkSyncMetric(cluster, 0);
    out.hsync();
    // hsync on empty file does nothing
    checkSyncMetric(cluster, 0);
    out.write(1);
    checkSyncMetric(cluster, 0);
    out.hsync();
    checkSyncMetric(cluster, 1);
    // avoiding repeated hsyncs is a potential future optimization
    out.hsync();
    checkSyncMetric(cluster, 2);
    out.hflush();
    // hflush still does not sync
    checkSyncMetric(cluster, 2);
    out.close();
    // close is sync'ing
    checkSyncMetric(cluster, 3);
    // same with a file created with out SYNC_BLOCK
    out = fs.create(p, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), 4096, (short) 1, len, null);
    out.hsync();
    checkSyncMetric(cluster, 3);
    out.write(1);
    checkSyncMetric(cluster, 3);
    out.hsync();
    checkSyncMetric(cluster, 4);
    // repeated hsyncs
    out.hsync();
    checkSyncMetric(cluster, 5);
    out.close();
    // close does not sync (not opened with SYNC_BLOCK)
    checkSyncMetric(cluster, 5);
    cluster.shutdown();
}

Example 40 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestQuotasWithHA method testQuotasTrackedOnStandby.

/**
   * Test that quotas are properly tracked by the standby through
   * create, append, delete.
   */
@Test(timeout = 60000)
public void testQuotasTrackedOnStandby() throws Exception {
    fs.mkdirs(TEST_DIR);
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.setQuota(TEST_DIR, NS_QUOTA, DS_QUOTA);
    long expectedSize = 3 * BLOCK_SIZE + BLOCK_SIZE / 2;
    DFSTestUtil.createFile(fs, TEST_FILE, expectedSize, (short) 1, 1L);
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    ContentSummary cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(1, cs.getFileCount());
    // Append to the file and make sure quota is updated correctly.
    FSDataOutputStream stm = fs.append(TEST_FILE);
    try {
        byte[] data = new byte[(int) (BLOCK_SIZE * 3 / 2)];
        stm.write(data);
        expectedSize += data.length;
    } finally {
        IOUtils.closeStream(stm);
    }
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(1, cs.getFileCount());
    fs.delete(TEST_FILE, true);
    expectedSize = 0;
    HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
    cs = nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
    assertEquals(NS_QUOTA, cs.getQuota());
    assertEquals(DS_QUOTA, cs.getSpaceQuota());
    assertEquals(expectedSize, cs.getSpaceConsumed());
    assertEquals(1, cs.getDirectoryCount());
    assertEquals(0, cs.getFileCount());
}