Examples with FSDataInputStreamWrapper org.apache.hadoop.hbase.io.FSDataInputStreamWrapper used on opensource projects

Example 1 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFile method createReader.

/**
   *
   * @param fs filesystem
   * @param path Path to file to read
   * @param cacheConf This must not be null.  @see {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
   * @return an active Reader instance
   * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile is corrupt/invalid.
   */
public static Reader createReader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    return pickReaderVersion(path, stream, fs.getFileStatus(path).getLen(), cacheConf, stream.getHfs(), conf);
}

Example 2 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestChecksum method testAllChecksumTypes.

/**
   * Test all checksum types by writing and reading back blocks.
   */
@Test
public void testAllChecksumTypes() throws IOException {
    List<ChecksumType> cktypes = new ArrayList<>(Arrays.asList(ChecksumType.values()));
    for (Iterator<ChecksumType> itr = cktypes.iterator(); itr.hasNext(); ) {
        ChecksumType cktype = itr.next();
        Path path = new Path(TEST_UTIL.getDataTestDir(), "checksum" + cktype.getName());
        FSDataOutputStream os = fs.create(path);
        HFileContext meta = new HFileContextBuilder().withChecksumType(cktype).build();
        HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
        DataOutputStream dos = hbw.startWriting(BlockType.DATA);
        for (int i = 0; i < 1000; ++i) {
            dos.writeInt(i);
        }
        hbw.writeHeaderAndData(os);
        int totalSize = hbw.getOnDiskSizeWithHeader();
        os.close();
        // Use hbase checksums.
        assertEquals(true, hfs.useHBaseChecksum());
        FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
        meta = new HFileContextBuilder().withHBaseCheckSum(true).build();
        HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, totalSize, (HFileSystem) fs, path, meta);
        HFileBlock b = hbr.readBlockData(0, -1, false);
        ByteBuff data = b.getBufferWithoutHeader();
        for (int i = 0; i < 1000; i++) {
            assertEquals(i, data.getInt());
        }
        boolean exception_thrown = false;
        try {
            data.getInt();
        } catch (BufferUnderflowException e) {
            exception_thrown = true;
        }
        assertTrue(exception_thrown);
        assertEquals(0, HFile.getChecksumFailuresCount());
    }
}

Example 3 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class StoreFileInfo method open.

/**
   * Open a Reader for the StoreFile
   * @param fs The current file system to use.
   * @param cacheConf The cache configuration and block cache reference.
   * @return The StoreFile.Reader for the file
   */
public StoreFileReader open(final FileSystem fs, final CacheConfig cacheConf, final boolean canUseDropBehind) throws IOException {
    FSDataInputStreamWrapper in;
    FileStatus status;
    final boolean doDropBehind = canUseDropBehind && cacheConf.shouldDropBehindCompaction();
    if (this.link != null) {
        // HFileLink
        in = new FSDataInputStreamWrapper(fs, this.link, doDropBehind);
        status = this.link.getFileStatus(fs);
    } else if (this.reference != null) {
        // HFile Reference
        Path referencePath = getReferredToFile(this.getPath());
        in = new FSDataInputStreamWrapper(fs, referencePath, doDropBehind);
        status = fs.getFileStatus(referencePath);
    } else {
        in = new FSDataInputStreamWrapper(fs, this.getPath(), doDropBehind);
        status = fs.getFileStatus(initialPath);
    }
    long length = status.getLen();
    hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
    StoreFileReader reader = null;
    if (this.coprocessorHost != null) {
        reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length, cacheConf, reference);
    }
    if (reader == null) {
        if (this.reference != null) {
            reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference, conf);
        } else {
            reader = new StoreFileReader(fs, status.getPath(), in, length, cacheConf, conf);
        }
    }
    if (this.coprocessorHost != null) {
        reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length, cacheConf, reference, reader);
    }
    return reader;
}

Example 4 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFile method createReader.

/**
 * @param fs filesystem
 * @param path Path to file to read
 * @param cacheConf This must not be null. @see
 *          {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)}
 * @param primaryReplicaReader true if this is a reader for primary replica
 * @param conf Configuration
 * @return an active Reader instance
 * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile
 *           is corrupt/invalid.
 */
public static Reader createReader(FileSystem fs, Path path, CacheConfig cacheConf, boolean primaryReplicaReader, Configuration conf) throws IOException {
    Preconditions.checkNotNull(cacheConf, "Cannot create Reader with null CacheConf");
    FSDataInputStreamWrapper stream = new FSDataInputStreamWrapper(fs, path);
    ReaderContext context = new ReaderContextBuilder().withFilePath(path).withInputStreamWrapper(stream).withFileSize(fs.getFileStatus(path).getLen()).withFileSystem(stream.getHfs()).withPrimaryReplicaReader(primaryReplicaReader).withReaderType(ReaderType.PREAD).build();
    HFileInfo fileInfo = new HFileInfo(context, conf);
    Reader reader = createReader(context, fileInfo, cacheConf, conf);
    fileInfo.initMetaAndIndex(reader);
    return reader;
}

Example 5 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestChecksum method testChecksumCorruptionInternals.

protected void testChecksumCorruptionInternals(boolean useTags) throws IOException {
    for (Compression.Algorithm algo : COMPRESSION_ALGORITHMS) {
        for (boolean pread : new boolean[] { false, true }) {
            LOG.info("testChecksumCorruption: Compression algorithm: " + algo + ", pread=" + pread);
            Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + algo);
            FSDataOutputStream os = fs.create(path);
            HFileContext meta = new HFileContextBuilder().withCompression(algo).withIncludesMvcc(true).withIncludesTags(useTags).withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM).build();
            HFileBlock.Writer hbw = new HFileBlock.Writer(TEST_UTIL.getConfiguration(), null, meta);
            long totalSize = 0;
            for (int blockId = 0; blockId < 2; ++blockId) {
                DataOutputStream dos = hbw.startWriting(BlockType.DATA);
                for (int i = 0; i < 1234; ++i) dos.writeInt(i);
                hbw.writeHeaderAndData(os);
                totalSize += hbw.getOnDiskSizeWithHeader();
            }
            os.close();
            // Use hbase checksums.
            assertEquals(true, hfs.useHBaseChecksum());
            // Do a read that purposely introduces checksum verification failures.
            FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
            meta = new HFileContextBuilder().withCompression(algo).withIncludesMvcc(true).withIncludesTags(useTags).withHBaseCheckSum(true).build();
            ReaderContext context = new ReaderContextBuilder().withInputStreamWrapper(is).withFileSize(totalSize).withFileSystem(fs).withFilePath(path).build();
            HFileBlock.FSReader hbr = new CorruptedFSReaderImpl(context, meta, TEST_UTIL.getConfiguration());
            HFileBlock b = hbr.readBlockData(0, -1, pread, false, true);
            b.sanityCheck();
            assertEquals(4936, b.getUncompressedSizeWithoutHeader());
            assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
            // read data back from the hfile, exclude header and checksum
            // read back data
            ByteBuff bb = b.unpack(meta, hbr).getBufferWithoutHeader();
            DataInputStream in = new DataInputStream(new ByteArrayInputStream(bb.array(), bb.arrayOffset(), bb.limit()));
            // assert that we encountered hbase checksum verification failures
            // but still used hdfs checksums and read data successfully.
            assertEquals(1, HFile.getAndResetChecksumFailuresCount());
            validateData(in);
            // requests. Verify that this is correct.
            for (int i = 0; i < HFileBlock.CHECKSUM_VERIFICATION_NUM_IO_THRESHOLD + 1; i++) {
                b = hbr.readBlockData(0, -1, pread, false, true);
                assertTrue(b.getBufferReadOnly() instanceof SingleByteBuff);
                assertEquals(0, HFile.getAndResetChecksumFailuresCount());
            }
            // The next read should have hbase checksum verification reanabled,
            // we verify this by assertng that there was a hbase-checksum failure.
            b = hbr.readBlockData(0, -1, pread, false, true);
            assertTrue(b.getBufferReadOnly() instanceof SingleByteBuff);
            assertEquals(1, HFile.getAndResetChecksumFailuresCount());
            // Since the above encountered a checksum failure, we switch
            // back to not checking hbase checksums.
            b = hbr.readBlockData(0, -1, pread, false, true);
            assertTrue(b.getBufferReadOnly() instanceof SingleByteBuff);
            assertEquals(0, HFile.getAndResetChecksumFailuresCount());
            is.close();
            // Now, use a completely new reader. Switch off hbase checksums in
            // the configuration. In this case, we should not detect
            // any retries within hbase.
            Configuration conf = TEST_UTIL.getConfiguration();
            HFileSystem newfs = new HFileSystem(conf, false);
            assertEquals(false, newfs.useHBaseChecksum());
            is = new FSDataInputStreamWrapper(newfs, path);
            context = new ReaderContextBuilder().withInputStreamWrapper(is).withFileSize(totalSize).withFileSystem(newfs).withFilePath(path).build();
            hbr = new CorruptedFSReaderImpl(context, meta, conf);
            b = hbr.readBlockData(0, -1, pread, false, true);
            is.close();
            b.sanityCheck();
            b = b.unpack(meta, hbr);
            assertEquals(4936, b.getUncompressedSizeWithoutHeader());
            assertEquals(algo == GZ ? 2173 : 4936, b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes());
            // read data back from the hfile, exclude header and checksum
            // read back data
            bb = b.getBufferWithoutHeader();
            in = new DataInputStream(new ByteArrayInputStream(bb.array(), bb.arrayOffset(), bb.limit()));
            // assert that we did not encounter hbase checksum verification failures
            // but still used hdfs checksums and read data successfully.
            assertEquals(0, HFile.getAndResetChecksumFailuresCount());
            validateData(in);
        }
    }
}