Examples with FSDataInputStreamWrapper org.apache.hadoop.hbase.io.FSDataInputStreamWrapper used on opensource projects

Example 6 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestChecksum method testNewBlocksHaveDefaultChecksum.

@Test
public void testNewBlocksHaveDefaultChecksum() throws IOException {
    Path path = new Path(TEST_UTIL.getDataTestDir(), "default_checksum");
    FSDataOutputStream os = fs.create(path);
    HFileContext meta = new HFileContextBuilder().build();
    HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
    DataOutputStream dos = hbw.startWriting(BlockType.DATA);
    for (int i = 0; i < 1000; ++i) dos.writeInt(i);
    hbw.writeHeaderAndData(os);
    int totalSize = hbw.getOnDiskSizeWithHeader();
    os.close();
    // Use hbase checksums.
    assertEquals(true, hfs.useHBaseChecksum());
    FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path);
    meta = new HFileContextBuilder().withHBaseCheckSum(true).build();
    HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(is, totalSize, (HFileSystem) fs, path, meta);
    HFileBlock b = hbr.readBlockData(0, -1, false);
    assertEquals(b.getChecksumType(), ChecksumType.getDefaultChecksumType().getCode());
}

Example 7 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class TestChecksum method testChecksumInternals.

protected void testChecksumInternals(boolean useTags) throws IOException {
    Compression.Algorithm algo = NONE;
    for (boolean pread : new boolean[] { false, true }) {
        for (int bytesPerChecksum : BYTES_PER_CHECKSUM) {
            Path path = new Path(TEST_UTIL.getDataTestDir(), "checksumChunk_" + algo + bytesPerChecksum);
            FSDataOutputStream os = fs.create(path);
            HFileContext meta = new HFileContextBuilder().withCompression(algo).withIncludesMvcc(true).withIncludesTags(useTags).withHBaseCheckSum(true).withBytesPerCheckSum(bytesPerChecksum).build();
            HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
            // write one block. The block has data
            // that is at least 6 times more than the checksum chunk size
            long dataSize = 0;
            DataOutputStream dos = hbw.startWriting(BlockType.DATA);
            for (; dataSize < 6 * bytesPerChecksum; ) {
                for (int i = 0; i < 1234; ++i) {
                    dos.writeInt(i);
                    dataSize += 4;
                }
            }
            hbw.writeHeaderAndData(os);
            long totalSize = hbw.getOnDiskSizeWithHeader();
            os.close();
            long expectedChunks = ChecksumUtil.numChunks(dataSize + HConstants.HFILEBLOCK_HEADER_SIZE, bytesPerChecksum);
            LOG.info("testChecksumChunks: pread=" + pread + ", bytesPerChecksum=" + bytesPerChecksum + ", fileSize=" + totalSize + ", dataSize=" + dataSize + ", expectedChunks=" + expectedChunks);
            // Verify hbase checksums. 
            assertEquals(true, hfs.useHBaseChecksum());
            // Read data back from file.
            FSDataInputStream is = fs.open(path);
            FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);
            meta = new HFileContextBuilder().withCompression(algo).withIncludesMvcc(true).withIncludesTags(useTags).withHBaseCheckSum(true).withBytesPerCheckSum(bytesPerChecksum).build();
            HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(is, nochecksum), totalSize, hfs, path, meta);
            HFileBlock b = hbr.readBlockData(0, -1, pread);
            is.close();
            b.sanityCheck();
            assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());
            // verify that we have the expected number of checksum chunks
            assertEquals(totalSize, HConstants.HFILEBLOCK_HEADER_SIZE + dataSize + expectedChunks * HFileBlock.CHECKSUM_SIZE);
            // assert that we did not encounter hbase checksum verification failures
            assertEquals(0, HFile.getChecksumFailuresCount());
        }
    }
}

Example 8 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFilePrettyPrinter method processFile.

public int processFile(Path file) throws IOException {
    if (verbose)
        out.println("Scanning -> " + file);
    Path rootPath = FSUtils.getRootDir(getConf());
    String rootString = rootPath + rootPath.SEPARATOR;
    if (!file.toString().startsWith(rootString)) {
        // First we see if fully-qualified URI matches the root dir. It might
        // also be an absolute path in the same filesystem, so we prepend the FS
        // of the root dir and see if that fully-qualified URI matches.
        FileSystem rootFS = rootPath.getFileSystem(getConf());
        String qualifiedFile = rootFS.getUri().toString() + file.toString();
        if (!qualifiedFile.startsWith(rootString)) {
            err.println("ERROR, file (" + file + ") is not in HBase's root directory (" + rootString + ")");
            return -2;
        }
    }
    FileSystem fs = file.getFileSystem(getConf());
    if (!fs.exists(file)) {
        err.println("ERROR, file doesnt exist: " + file);
        return -2;
    }
    HFile.Reader reader = HFile.createReader(fs, file, new CacheConfig(getConf()), getConf());
    Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
    KeyValueStatsCollector fileStats = null;
    if (verbose || printKey || checkRow || checkFamily || printStats || checkMobIntegrity) {
        // scan over file and read key/value's and check if requested
        HFileScanner scanner = reader.getScanner(false, false, false);
        fileStats = new KeyValueStatsCollector();
        boolean shouldScanKeysValues = false;
        if (this.isSeekToRow) {
            // seek to the first kv on this row
            shouldScanKeysValues = (scanner.seekTo(CellUtil.createFirstOnRow(this.row)) != -1);
        } else {
            shouldScanKeysValues = scanner.seekTo();
        }
        if (shouldScanKeysValues)
            scanKeysValues(file, fileStats, scanner, row);
    }
    // print meta data
    if (shouldPrintMeta) {
        printMeta(reader, fileInfo);
    }
    if (printBlockIndex) {
        out.println("Block Index:");
        out.println(reader.getDataBlockIndexReader());
    }
    if (printBlockHeaders) {
        out.println("Block Headers:");
        /*
       * TODO: this same/similar block iteration logic is used in HFileBlock#blockRange and
       * TestLazyDataBlockDecompression. Refactor?
       */
        FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, file);
        long fileSize = fs.getFileStatus(file).getLen();
        FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize);
        long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset();
        HFileBlock block;
        while (offset <= max) {
            block = reader.readBlock(offset, -1, /* cacheBlock */
            false, /* pread */
            false, /* isCompaction */
            false, /* updateCacheMetrics */
            false, null, null);
            offset += block.getOnDiskSizeWithHeader();
            out.println(block);
        }
    }
    if (printStats) {
        fileStats.finish();
        out.println("Stats:\n" + fileStats);
    }
    reader.close();
    return 0;
}

Example 9 with FSDataInputStreamWrapper

use of org.apache.hadoop.hbase.io.FSDataInputStreamWrapper in project hbase by apache.

the class HFile method isHFileFormat.

/**
   * Returns true if the specified file has a valid HFile Trailer.
   * @param fs filesystem
   * @param fileStatus the file to verify
   * @return true if the file has a valid HFile Trailer, otherwise false
   * @throws IOException if failed to read from the underlying stream
   */
public static boolean isHFileFormat(final FileSystem fs, final FileStatus fileStatus) throws IOException {
    final Path path = fileStatus.getPath();
    final long size = fileStatus.getLen();
    FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path);
    try {
        boolean isHBaseChecksum = fsdis.shouldUseHBaseChecksum();
        // Initially we must read with FS checksum.
        assert !isHBaseChecksum;
        FixedFileTrailer.readFromStream(fsdis.getStream(isHBaseChecksum), size);
        return true;
    } catch (IllegalArgumentException e) {
        return false;
    } catch (IOException e) {
        throw e;
    } finally {
        try {
            fsdis.close();
        } catch (Throwable t) {
            LOG.warn("Error closing fsdis FSDataInputStreamWrapper: " + path, t);
        }
    }
}